From 191c4b052924e11db20f61061892205a42b71a78 Mon Sep 17 00:00:00 2001
From: Jan Mach <jan.mach@cesnet.cz>
Date: Wed, 6 Nov 2019 16:13:57 +0100
Subject: [PATCH] Moved JSON serialized event from events.event column to
events_json table.
By this move we attempt to speed up the searching process and lower the usage of system resources, because the amount of data the database engine needs to load and process will be significantly smaller. (Redmine issue: #4274,#4275)
---
doc/sphinx/_doclib/development.rst | 11 +-
lib/hawat/base.py | 16 +--
lib/hawat/blueprints/events/__init__.py | 7 ++
lib/mentat/__init__.py | 2 +-
lib/mentat/services/eventstorage.py | 97 ++++++++++-----
lib/mentat/services/test_eventstorage.py | 110 +++++++++---------
...a714_split_event_column_from_events_to_.py | 46 ++++++++
7 files changed, 190 insertions(+), 99 deletions(-)
create mode 100644 migrations-events/versions/0f31c168a714_split_event_column_from_events_to_.py
diff --git a/doc/sphinx/_doclib/development.rst b/doc/sphinx/_doclib/development.rst
index b9e9aea1d..e40883f73 100644
--- a/doc/sphinx/_doclib/development.rst
+++ b/doc/sphinx/_doclib/development.rst
@@ -496,6 +496,11 @@ following resources as reference:
* `Operation Reference <https://alembic.sqlalchemy.org/en/latest/ops.html>`__
* `Cookbook <https://alembic.sqlalchemy.org/en/latest/cookbook.html>`__
+Do not forget to update the :py:mod:`mentat.services.eventstorage` module according
+to the migration. There should always be current database schema to allow simple
+and straightforward installation (instead of installing schema, that needs to be
+migrated right away).
+
Migration can be then invoked locally from within the migration environment directory::
cd migrations-events
@@ -503,10 +508,10 @@ Migration can be then invoked locally from within the migration environment dire
alembic history
To enable execution of database migrations on target systems after installation
-from package there is a simple wrapper script ``/etc/mentat/scripts/sqldb-migrate-e.sh``::
+from package there is a simple wrapper script ``/etc/mentat/scripts/sqldb-migrate.sh``::
- /etc/mentat/scripts/sqldb-migrate-e.sh upgrade head
- /etc/mentat/scripts/sqldb-migrate-e.sh history
+ /etc/mentat/scripts/sqldb-migrate.sh upgrade head
+ /etc/mentat/scripts/sqldb-migrate.sh history
Important resources:
diff --git a/lib/hawat/base.py b/lib/hawat/base.py
index 52bb72bfb..302a06166 100644
--- a/lib/hawat/base.py
+++ b/lib/hawat/base.py
@@ -888,18 +888,11 @@ class PsycopgMixin:
return hawat.events.db_get()
@staticmethod
- def get_event_factory():
+ def get_qtype():
"""
- Get reference to a item factory for converting database records into objects.
+ Get type of the event select query.
"""
- return mentat.services.eventstorage.record_to_idea
-
- @staticmethod
- def get_event_columns():
- """
- Get list of event database table column names.
- """
- return mentat.services.eventstorage.EVENT_COLUMNS
+ return mentat.services.eventstorage.QTYPE_SELECT
def search(self, form_args):
"""
@@ -911,8 +904,7 @@ class PsycopgMixin:
"""
items_count_total, items = hawat.events.db_get().search_events(
form_args,
- event_factory = self.get_event_factory(),
- columns = self.get_event_columns()
+ qtype = self.get_qtype(),
)
self.response_context.update(
sqlquery = hawat.events.db_get().cursor.lastquery.decode('utf-8')
diff --git a/lib/hawat/blueprints/events/__init__.py b/lib/hawat/blueprints/events/__init__.py
index e8073b884..d46ea1c2a 100644
--- a/lib/hawat/blueprints/events/__init__.py
+++ b/lib/hawat/blueprints/events/__init__.py
@@ -132,6 +132,13 @@ class SearchView(HTMLMixin, AbstractSearchView): # pylint: disable=locally-disa
has_help = True
+ @staticmethod
+ def get_qtype():
+ """
+ Get type of the event select query.
+ """
+ return mentat.services.eventstorage.QTYPE_SELECT_GHOST
+
@classmethod
def get_breadcrumbs_menu(cls):
"""
diff --git a/lib/mentat/__init__.py b/lib/mentat/__init__.py
index a9f44acae..82c82df49 100644
--- a/lib/mentat/__init__.py
+++ b/lib/mentat/__init__.py
@@ -20,4 +20,4 @@ open-source project.
__author__ = "Jan Mach <jan.mach@cesnet.cz>"
__credits__ = "Pavel Kácha <pavel.kacha@cesnet.cz>, Andrea Kropáčová <andrea.kropacova@cesnet.cz>"
-__version__ = "2.5.6"
+__version__ = "2.5.7"
diff --git a/lib/mentat/services/eventstorage.py b/lib/mentat/services/eventstorage.py
index 89e6960b0..fcbc3d06a 100644
--- a/lib/mentat/services/eventstorage.py
+++ b/lib/mentat/services/eventstorage.py
@@ -36,8 +36,8 @@ References
"""
-__author__ = "Jan Mach <jan.mach@cesnet.cz>"
-__credits__ = "Radko Krkoš <radko.krkos@cesnet.cz>, Pavel Kácha <pavel.kacha@cesnet.cz>, Andrea Kropáčová <andrea.kropacova@cesnet.cz>"
+__author__ = "Jan Mach <jan.mach@cesnet.cz>, Radko Krkoš <radko.krkos@cesnet.cz>"
+__credits__ = "Pavel Kácha <pavel.kacha@cesnet.cz>, Andrea Kropáčová <andrea.kropacova@cesnet.cz>"
import copy
@@ -56,9 +56,10 @@ from mentat.const import CKEY_CORE_DATABASE, CKEY_CORE_DATABASE_EVENTSTORAGE
_MANAGER = None
-QTYPE_SELECT = 'select'
-QTYPE_COUNT = 'count'
-QTYPE_DELETE = 'delete'
+QTYPE_SELECT = 'select'
+QTYPE_SELECT_GHOST = 'select_ghost'
+QTYPE_COUNT = 'count'
+QTYPE_DELETE = 'delete'
ENUM_TABLES = (
"category",
@@ -73,7 +74,7 @@ ENUM_TABLES = (
"cesnet_inspectionerrors"
)
-EVENT_COLUMNS = (
+EVENTS_COLUMNS = (
"id",
"detecttime",
"category",
@@ -96,7 +97,6 @@ EVENT_COLUMNS = (
"cesnet_eventclass",
"cesnet_eventseverity",
"cesnet_inspectionerrors",
- "event"
)
class EventStorageException(Exception):
@@ -191,7 +191,35 @@ def _bq_searchby_addr(chunks, params, idents, items):
items_exp.append(i)
params.extend(items_exp)
-def build_query(parameters = None, qtype = QTYPE_SELECT, columns = EVENT_COLUMNS): # pylint: disable=locally-disabled,too-many-branches
+def _bq_qbase_select_full():
+ return psycopg2.sql.SQL('SELECT {}, {}.{} FROM events INNER JOIN events_json ON events.id = events_json.id').format(
+ psycopg2.sql.SQL(', ').join([
+ psycopg2.sql.SQL('{}.{}').format(
+ psycopg2.sql.Identifier('events'),
+ psycopg2.sql.Identifier(x)
+ ) for x in EVENTS_COLUMNS
+ ]),
+ psycopg2.sql.Identifier('events_json'),
+ psycopg2.sql.Identifier('event')
+ )
+
+def _bq_qbase_select_ghost():
+ return psycopg2.sql.SQL('SELECT {} FROM events').format(
+ psycopg2.sql.SQL(', ').join([
+ psycopg2.sql.SQL('{}.{}').format(
+ psycopg2.sql.Identifier('events'),
+ psycopg2.sql.Identifier(x)
+ ) for x in EVENTS_COLUMNS
+ ])
+ )
+
+def _bq_qbase_count():
+ return psycopg2.sql.SQL('SELECT count(id) FROM events')
+
+def _bq_qbase_delete():
+ return psycopg2.sql.SQL('DELETE FROM events')
+
+def build_query(parameters = None, qtype = QTYPE_SELECT): # pylint: disable=locally-disabled,too-many-branches
"""
Build SQL database query according to given parameters.
@@ -206,15 +234,13 @@ def build_query(parameters = None, qtype = QTYPE_SELECT, columns = EVENT_COLUMNS
# Prepare query base.
if qtype == QTYPE_SELECT:
- query = psycopg2.sql.SQL('SELECT {} FROM events').format(
- psycopg2.sql.SQL(', ').join(
- [psycopg2.sql.Identifier(x) for x in columns]
- )
- )
+ query = _bq_qbase_select_full()
+ elif qtype == QTYPE_SELECT_GHOST:
+ query = _bq_qbase_select_ghost()
elif qtype == QTYPE_COUNT:
- query = psycopg2.sql.SQL('SELECT count(id) FROM events')
+ query = _bq_qbase_count()
elif qtype == QTYPE_DELETE:
- query = psycopg2.sql.SQL('DELETE FROM events')
+ query = _bq_qbase_delete()
elif isinstance(qtype, psycopg2.sql.Composed):
query = qtype
else:
@@ -384,6 +410,12 @@ def record_to_idea_ghost(val):
return mentat.idea.internal.IdeaGhost.from_record(val)
+_OBJECT_TYPES = {
+ QTYPE_SELECT: record_to_idea,
+ QTYPE_SELECT_GHOST: record_to_idea_ghost
+}
+
+
class EventStorageCursor:
"""
Encapsulation of :py:class:`psycopg2.cursor` class.
@@ -418,9 +450,14 @@ class EventStorageCursor:
"""
idea_pgsql = mentat.idea.sqldb.Idea(idea_event)
record = idea_pgsql.get_record()
+
self.cursor.execute(
- "INSERT INTO events (id, detecttime, category, description, source_ip, target_ip, source_ip_aggr_ip4, source_ip_aggr_ip6, target_ip_aggr_ip4, target_ip_aggr_ip6, source_port, target_port, source_type, target_type, protocol, node_name, node_type, cesnet_resolvedabuses, cesnet_storagetime, cesnet_eventclass, cesnet_eventseverity, cesnet_inspectionerrors, event) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)",
- record
+ "INSERT INTO events (id, detecttime, category, description, source_ip, target_ip, source_ip_aggr_ip4, source_ip_aggr_ip6, target_ip_aggr_ip4, target_ip_aggr_ip6, source_port, target_port, source_type, target_type, protocol, node_name, node_type, cesnet_resolvedabuses, cesnet_storagetime, cesnet_eventclass, cesnet_eventseverity, cesnet_inspectionerrors) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)",
+ record[0:-1]
+ )
+ self.cursor.execute(
+ "INSERT INTO events_json (id, event) VALUES (%s, %s)",
+ (record[0], record[-1])
)
def fetch_event(self, eventid):
@@ -432,7 +469,7 @@ class EventStorageCursor:
:rtype: mentat.idea.internal
"""
self.cursor.execute(
- "SELECT id, event FROM events WHERE id = %s",
+ "SELECT id, event FROM events_json WHERE id = %s",
(eventid,)
)
record = self.cursor.fetchone()
@@ -472,16 +509,18 @@ class EventStorageCursor:
return record[0]
return None
- def search_events(self, parameters = None, event_factory = record_to_idea, columns = EVENT_COLUMNS):
+ def search_events(self, parameters = None, qtype = QTYPE_SELECT):
"""
Search IDEA messages in database according to given parameters. The
parameters will be passed down to the :py:func:`mentat.services.eventstorage.build_query`
function to generate proper SQL query.
:param dict parameters: Search query parameters, see :py:func:`mentat.services.eventstorage.build_query` for details.
- :param callable event_factory: Conversion callback method to use to convert each record.
+ :param string qtype: Type of the select query.
"""
- query, params = build_query(parameters, columns = columns)
+ event_factory = _OBJECT_TYPES[qtype]
+
+ query, params = build_query(parameters, qtype = qtype)
self.lastquery = self.cursor.mogrify(query, params)
self.cursor.execute(query, params)
@@ -656,7 +695,7 @@ class EventStorageCursor:
:rtype: list
"""
self.cursor.execute(
- "SELECT * FROM events WHERE id IN (SELECT DISTINCT eventid FROM events_thresholded WHERE keyid IN (SELECT keyid FROM events_thresholded INNER JOIN thresholds ON (events_thresholded.keyid = thresholds.id) WHERE events_thresholded.groupname = %s AND events_thresholded.eventseverity = %s AND events_thresholded.createtime >= thresholds.relapsetime AND thresholds.ttltime <= %s))",
+ "SELECT events.*, events_json.event FROM events INNER JOIN events_json ON events.id = events_json.id WHERE events.id IN (SELECT DISTINCT eventid FROM events_thresholded WHERE keyid IN (SELECT keyid FROM events_thresholded INNER JOIN thresholds ON (events_thresholded.keyid = thresholds.id) WHERE events_thresholded.groupname = %s AND events_thresholded.eventseverity = %s AND events_thresholded.createtime >= thresholds.relapsetime AND thresholds.ttltime <= %s))",
(group_name, severity, ttl)
)
events_raw = self.cursor.fetchall()
@@ -804,7 +843,8 @@ class EventStorageService:
"""
# Base list of CREATE TABLE SQLs.
create_table_sqls = [
- "CREATE TABLE IF NOT EXISTS events(id text PRIMARY KEY, detecttime timestamp NOT NULL, category text[] NOT NULL, description text, source_ip iprange[], target_ip iprange[], source_ip_aggr_ip4 ip4r, source_ip_aggr_ip6 ip6r, target_ip_aggr_ip4 ip4r, target_ip_aggr_ip6 ip6r, source_port integer[], target_port integer[], source_type text[], target_type text[], protocol text[], node_name text[] NOT NULL, node_type text[], cesnet_storagetime timestamp NOT NULL, cesnet_resolvedabuses text[], cesnet_eventclass text, cesnet_eventseverity text, cesnet_inspectionerrors text[], event bytea)",
+ "CREATE TABLE IF NOT EXISTS events(id text PRIMARY KEY, detecttime timestamp NOT NULL, category text[] NOT NULL, description text, source_ip iprange[], target_ip iprange[], source_ip_aggr_ip4 ip4r, source_ip_aggr_ip6 ip6r, target_ip_aggr_ip4 ip4r, target_ip_aggr_ip6 ip6r, source_port integer[], target_port integer[], source_type text[], target_type text[], protocol text[], node_name text[] NOT NULL, node_type text[], cesnet_storagetime timestamp NOT NULL, cesnet_resolvedabuses text[], cesnet_eventclass text, cesnet_eventseverity text, cesnet_inspectionerrors text[])",
+ "CREATE TABLE IF NOT EXISTS events_json(id text PRIMARY KEY REFERENCES events(id) ON DELETE CASCADE, event bytea NOT NULL)",
"CREATE TABLE IF NOT EXISTS thresholds(id text PRIMARY KEY, thresholdtime timestamp NOT NULL, relapsetime timestamp NOT NULL, ttltime timestamp NOT NULL)",
"CREATE TABLE IF NOT EXISTS events_thresholded(eventid text NOT NULL, keyid text NOT NULL, groupname text NOT NULL, eventseverity text NOT NULL, createtime timestamp NOT NULL, PRIMARY KEY(eventid, keyid))"
]
@@ -869,9 +909,10 @@ class EventStorageService:
"""
# Base list of DROP TABLE SQLs.
drop_table_sqls = [
- "DROP TABLE IF EXISTS events",
- "DROP TABLE IF EXISTS thresholds",
- "DROP TABLE IF EXISTS events_thresholded"
+ "DROP TABLE IF EXISTS events_json CASCADE",
+ "DROP TABLE IF EXISTS events CASCADE",
+ "DROP TABLE IF EXISTS thresholds CASCADE",
+ "DROP TABLE IF EXISTS events_thresholded CASCADE"
]
# Generate list of CREATE INDEX SQLs for column value enumeration tables.
@@ -981,7 +1022,7 @@ class EventStorageService:
return result
@handle_db_exceptions
- def search_events(self, parameters = None, event_factory = record_to_idea, columns = EVENT_COLUMNS):
+ def search_events(self, parameters = None, qtype = QTYPE_SELECT):
"""
This method is a convenience wrapper for underlying
:py:func:`mentat.services.eventstorage.EventStorageCursor.search_events`
@@ -990,7 +1031,7 @@ class EventStorageService:
It will automatically commit transaction for successfull database operation
and rollback the invalid one.
"""
- count, result = self.cursor.search_events(parameters, event_factory, columns)
+ count, result = self.cursor.search_events(parameters, qtype)
self.commit()
return count, result
diff --git a/lib/mentat/services/test_eventstorage.py b/lib/mentat/services/test_eventstorage.py
index 0fc92a502..774a7d6f1 100644
--- a/lib/mentat/services/test_eventstorage.py
+++ b/lib/mentat/services/test_eventstorage.py
@@ -324,7 +324,7 @@ class TestMentatStorage(unittest.TestCase):
{
'parameters': {}
},
- b'SELECT "id","detecttime","category","description","source_ip","target_ip","source_ip_aggr_ip4","source_ip_aggr_ip6","target_ip_aggr_ip4","target_ip_aggr_ip6","source_port","target_port","source_type","target_type","protocol","node_name","node_type","cesnet_resolvedabuses","cesnet_storagetime","cesnet_eventclass","cesnet_eventseverity","cesnet_inspectionerrors","event" FROM events'
+ b'SELECT "events"."id","events"."detecttime","events"."category","events"."description","events"."source_ip","events"."target_ip","events"."source_ip_aggr_ip4","events"."source_ip_aggr_ip6","events"."target_ip_aggr_ip4","events"."target_ip_aggr_ip6","events"."source_port","events"."target_port","events"."source_type","events"."target_type","events"."protocol","events"."node_name","events"."node_type","events"."cesnet_resolvedabuses","events"."cesnet_storagetime","events"."cesnet_eventclass","events"."cesnet_eventseverity","events"."cesnet_inspectionerrors","events_json"."event" FROM events INNER JOIN events_json ON events.id = events_json.id'
),
(
{
@@ -332,7 +332,7 @@ class TestMentatStorage(unittest.TestCase):
'dt_from': datetime.datetime(2012, 11, 3, 10, 0, 7)
}
},
- b'SELECT "id","detecttime","category","description","source_ip","target_ip","source_ip_aggr_ip4","source_ip_aggr_ip6","target_ip_aggr_ip4","target_ip_aggr_ip6","source_port","target_port","source_type","target_type","protocol","node_name","node_type","cesnet_resolvedabuses","cesnet_storagetime","cesnet_eventclass","cesnet_eventseverity","cesnet_inspectionerrors","event" FROM events WHERE "detecttime" >= \'2012-11-03T10:00:07\'::timestamp'
+ b'SELECT "events"."id","events"."detecttime","events"."category","events"."description","events"."source_ip","events"."target_ip","events"."source_ip_aggr_ip4","events"."source_ip_aggr_ip6","events"."target_ip_aggr_ip4","events"."target_ip_aggr_ip6","events"."source_port","events"."target_port","events"."source_type","events"."target_type","events"."protocol","events"."node_name","events"."node_type","events"."cesnet_resolvedabuses","events"."cesnet_storagetime","events"."cesnet_eventclass","events"."cesnet_eventseverity","events"."cesnet_inspectionerrors","events_json"."event" FROM events INNER JOIN events_json ON events.id = events_json.id WHERE "detecttime" >= \'2012-11-03T10:00:07\'::timestamp'
),
(
{
@@ -340,7 +340,7 @@ class TestMentatStorage(unittest.TestCase):
'dt_to': datetime.datetime(2012, 11, 3, 10, 0, 7),
}
},
- b'SELECT "id","detecttime","category","description","source_ip","target_ip","source_ip_aggr_ip4","source_ip_aggr_ip6","target_ip_aggr_ip4","target_ip_aggr_ip6","source_port","target_port","source_type","target_type","protocol","node_name","node_type","cesnet_resolvedabuses","cesnet_storagetime","cesnet_eventclass","cesnet_eventseverity","cesnet_inspectionerrors","event" FROM events WHERE "detecttime" <= \'2012-11-03T10:00:07\'::timestamp'
+ b'SELECT "events"."id","events"."detecttime","events"."category","events"."description","events"."source_ip","events"."target_ip","events"."source_ip_aggr_ip4","events"."source_ip_aggr_ip6","events"."target_ip_aggr_ip4","events"."target_ip_aggr_ip6","events"."source_port","events"."target_port","events"."source_type","events"."target_type","events"."protocol","events"."node_name","events"."node_type","events"."cesnet_resolvedabuses","events"."cesnet_storagetime","events"."cesnet_eventclass","events"."cesnet_eventseverity","events"."cesnet_inspectionerrors","events_json"."event" FROM events INNER JOIN events_json ON events.id = events_json.id WHERE "detecttime" <= \'2012-11-03T10:00:07\'::timestamp'
),
(
{
@@ -349,7 +349,7 @@ class TestMentatStorage(unittest.TestCase):
'dt_to': datetime.datetime(2012, 11, 3, 10, 0, 7),
}
},
- b'SELECT "id","detecttime","category","description","source_ip","target_ip","source_ip_aggr_ip4","source_ip_aggr_ip6","target_ip_aggr_ip4","target_ip_aggr_ip6","source_port","target_port","source_type","target_type","protocol","node_name","node_type","cesnet_resolvedabuses","cesnet_storagetime","cesnet_eventclass","cesnet_eventseverity","cesnet_inspectionerrors","event" FROM events WHERE "detecttime" >= \'2012-11-03T10:00:07\'::timestamp AND "detecttime" <= \'2012-11-03T10:00:07\'::timestamp'
+ b'SELECT "events"."id","events"."detecttime","events"."category","events"."description","events"."source_ip","events"."target_ip","events"."source_ip_aggr_ip4","events"."source_ip_aggr_ip6","events"."target_ip_aggr_ip4","events"."target_ip_aggr_ip6","events"."source_port","events"."target_port","events"."source_type","events"."target_type","events"."protocol","events"."node_name","events"."node_type","events"."cesnet_resolvedabuses","events"."cesnet_storagetime","events"."cesnet_eventclass","events"."cesnet_eventseverity","events"."cesnet_inspectionerrors","events_json"."event" FROM events INNER JOIN events_json ON events.id = events_json.id WHERE "detecttime" >= \'2012-11-03T10:00:07\'::timestamp AND "detecttime" <= \'2012-11-03T10:00:07\'::timestamp'
),
(
{
@@ -358,7 +358,7 @@ class TestMentatStorage(unittest.TestCase):
'st_to': datetime.datetime(2012, 11, 3, 10, 0, 7)
}
},
- b'SELECT "id","detecttime","category","description","source_ip","target_ip","source_ip_aggr_ip4","source_ip_aggr_ip6","target_ip_aggr_ip4","target_ip_aggr_ip6","source_port","target_port","source_type","target_type","protocol","node_name","node_type","cesnet_resolvedabuses","cesnet_storagetime","cesnet_eventclass","cesnet_eventseverity","cesnet_inspectionerrors","event" FROM events WHERE "cesnet_storagetime" >= \'2012-11-03T10:00:07\'::timestamp AND "cesnet_storagetime" <= \'2012-11-03T10:00:07\'::timestamp'
+ b'SELECT "events"."id","events"."detecttime","events"."category","events"."description","events"."source_ip","events"."target_ip","events"."source_ip_aggr_ip4","events"."source_ip_aggr_ip6","events"."target_ip_aggr_ip4","events"."target_ip_aggr_ip6","events"."source_port","events"."target_port","events"."source_type","events"."target_type","events"."protocol","events"."node_name","events"."node_type","events"."cesnet_resolvedabuses","events"."cesnet_storagetime","events"."cesnet_eventclass","events"."cesnet_eventseverity","events"."cesnet_inspectionerrors","events_json"."event" FROM events INNER JOIN events_json ON events.id = events_json.id WHERE "cesnet_storagetime" >= \'2012-11-03T10:00:07\'::timestamp AND "cesnet_storagetime" <= \'2012-11-03T10:00:07\'::timestamp'
),
(
{
@@ -386,7 +386,7 @@ class TestMentatStorage(unittest.TestCase):
'source_addrs': ['192.168.1.0/24']
}
},
- b'SELECT "id","detecttime","category","description","source_ip","target_ip","source_ip_aggr_ip4","source_ip_aggr_ip6","target_ip_aggr_ip4","target_ip_aggr_ip6","source_port","target_port","source_type","target_type","protocol","node_name","node_type","cesnet_resolvedabuses","cesnet_storagetime","cesnet_eventclass","cesnet_eventseverity","cesnet_inspectionerrors","event" FROM events WHERE ("source_ip_aggr_ip4" && \'192.168.1.0/24\' AND \'192.168.1.0/24\' && ANY("source_ip"))'
+ b'SELECT "events"."id","events"."detecttime","events"."category","events"."description","events"."source_ip","events"."target_ip","events"."source_ip_aggr_ip4","events"."source_ip_aggr_ip6","events"."target_ip_aggr_ip4","events"."target_ip_aggr_ip6","events"."source_port","events"."target_port","events"."source_type","events"."target_type","events"."protocol","events"."node_name","events"."node_type","events"."cesnet_resolvedabuses","events"."cesnet_storagetime","events"."cesnet_eventclass","events"."cesnet_eventseverity","events"."cesnet_inspectionerrors","events_json"."event" FROM events INNER JOIN events_json ON events.id = events_json.id WHERE ("source_ip_aggr_ip4" && \'192.168.1.0/24\' AND \'192.168.1.0/24\' && ANY("source_ip"))'
),
(
{
@@ -394,7 +394,7 @@ class TestMentatStorage(unittest.TestCase):
'source_addrs': ['2001::/54']
}
},
- b'SELECT "id","detecttime","category","description","source_ip","target_ip","source_ip_aggr_ip4","source_ip_aggr_ip6","target_ip_aggr_ip4","target_ip_aggr_ip6","source_port","target_port","source_type","target_type","protocol","node_name","node_type","cesnet_resolvedabuses","cesnet_storagetime","cesnet_eventclass","cesnet_eventseverity","cesnet_inspectionerrors","event" FROM events WHERE ("source_ip_aggr_ip6" && \'2001::/54\' AND \'2001::/54\' && ANY("source_ip"))'
+ b'SELECT "events"."id","events"."detecttime","events"."category","events"."description","events"."source_ip","events"."target_ip","events"."source_ip_aggr_ip4","events"."source_ip_aggr_ip6","events"."target_ip_aggr_ip4","events"."target_ip_aggr_ip6","events"."source_port","events"."target_port","events"."source_type","events"."target_type","events"."protocol","events"."node_name","events"."node_type","events"."cesnet_resolvedabuses","events"."cesnet_storagetime","events"."cesnet_eventclass","events"."cesnet_eventseverity","events"."cesnet_inspectionerrors","events_json"."event" FROM events INNER JOIN events_json ON events.id = events_json.id WHERE ("source_ip_aggr_ip6" && \'2001::/54\' AND \'2001::/54\' && ANY("source_ip"))'
),
(
{
@@ -402,7 +402,7 @@ class TestMentatStorage(unittest.TestCase):
'source_addrs': ['192.168.1.0/24', '2001::/54']
}
},
- b'SELECT "id","detecttime","category","description","source_ip","target_ip","source_ip_aggr_ip4","source_ip_aggr_ip6","target_ip_aggr_ip4","target_ip_aggr_ip6","source_port","target_port","source_type","target_type","protocol","node_name","node_type","cesnet_resolvedabuses","cesnet_storagetime","cesnet_eventclass","cesnet_eventseverity","cesnet_inspectionerrors","event" FROM events WHERE (("source_ip_aggr_ip4" && \'192.168.1.0/24\' AND \'192.168.1.0/24\' && ANY("source_ip")) OR ("source_ip_aggr_ip6" && \'2001::/54\' AND \'2001::/54\' && ANY("source_ip")))'
+ b'SELECT "events"."id","events"."detecttime","events"."category","events"."description","events"."source_ip","events"."target_ip","events"."source_ip_aggr_ip4","events"."source_ip_aggr_ip6","events"."target_ip_aggr_ip4","events"."target_ip_aggr_ip6","events"."source_port","events"."target_port","events"."source_type","events"."target_type","events"."protocol","events"."node_name","events"."node_type","events"."cesnet_resolvedabuses","events"."cesnet_storagetime","events"."cesnet_eventclass","events"."cesnet_eventseverity","events"."cesnet_inspectionerrors","events_json"."event" FROM events INNER JOIN events_json ON events.id = events_json.id WHERE (("source_ip_aggr_ip4" && \'192.168.1.0/24\' AND \'192.168.1.0/24\' && ANY("source_ip")) OR ("source_ip_aggr_ip6" && \'2001::/54\' AND \'2001::/54\' && ANY("source_ip")))'
),
(
{
@@ -410,7 +410,7 @@ class TestMentatStorage(unittest.TestCase):
'target_addrs': ['192.168.1.0/24']
}
},
- b'SELECT "id","detecttime","category","description","source_ip","target_ip","source_ip_aggr_ip4","source_ip_aggr_ip6","target_ip_aggr_ip4","target_ip_aggr_ip6","source_port","target_port","source_type","target_type","protocol","node_name","node_type","cesnet_resolvedabuses","cesnet_storagetime","cesnet_eventclass","cesnet_eventseverity","cesnet_inspectionerrors","event" FROM events WHERE ("target_ip_aggr_ip4" && \'192.168.1.0/24\' AND \'192.168.1.0/24\' && ANY("target_ip"))'
+ b'SELECT "events"."id","events"."detecttime","events"."category","events"."description","events"."source_ip","events"."target_ip","events"."source_ip_aggr_ip4","events"."source_ip_aggr_ip6","events"."target_ip_aggr_ip4","events"."target_ip_aggr_ip6","events"."source_port","events"."target_port","events"."source_type","events"."target_type","events"."protocol","events"."node_name","events"."node_type","events"."cesnet_resolvedabuses","events"."cesnet_storagetime","events"."cesnet_eventclass","events"."cesnet_eventseverity","events"."cesnet_inspectionerrors","events_json"."event" FROM events INNER JOIN events_json ON events.id = events_json.id WHERE ("target_ip_aggr_ip4" && \'192.168.1.0/24\' AND \'192.168.1.0/24\' && ANY("target_ip"))'
),
(
{
@@ -418,7 +418,7 @@ class TestMentatStorage(unittest.TestCase):
'target_addrs': ['2001::/54']
}
},
- b'SELECT "id","detecttime","category","description","source_ip","target_ip","source_ip_aggr_ip4","source_ip_aggr_ip6","target_ip_aggr_ip4","target_ip_aggr_ip6","source_port","target_port","source_type","target_type","protocol","node_name","node_type","cesnet_resolvedabuses","cesnet_storagetime","cesnet_eventclass","cesnet_eventseverity","cesnet_inspectionerrors","event" FROM events WHERE ("target_ip_aggr_ip6" && \'2001::/54\' AND \'2001::/54\' && ANY("target_ip"))'
+ b'SELECT "events"."id","events"."detecttime","events"."category","events"."description","events"."source_ip","events"."target_ip","events"."source_ip_aggr_ip4","events"."source_ip_aggr_ip6","events"."target_ip_aggr_ip4","events"."target_ip_aggr_ip6","events"."source_port","events"."target_port","events"."source_type","events"."target_type","events"."protocol","events"."node_name","events"."node_type","events"."cesnet_resolvedabuses","events"."cesnet_storagetime","events"."cesnet_eventclass","events"."cesnet_eventseverity","events"."cesnet_inspectionerrors","events_json"."event" FROM events INNER JOIN events_json ON events.id = events_json.id WHERE ("target_ip_aggr_ip6" && \'2001::/54\' AND \'2001::/54\' && ANY("target_ip"))'
),
(
{
@@ -426,7 +426,7 @@ class TestMentatStorage(unittest.TestCase):
'target_addrs': ['192.168.1.0/24', '2001::/54']
}
},
- b'SELECT "id","detecttime","category","description","source_ip","target_ip","source_ip_aggr_ip4","source_ip_aggr_ip6","target_ip_aggr_ip4","target_ip_aggr_ip6","source_port","target_port","source_type","target_type","protocol","node_name","node_type","cesnet_resolvedabuses","cesnet_storagetime","cesnet_eventclass","cesnet_eventseverity","cesnet_inspectionerrors","event" FROM events WHERE (("target_ip_aggr_ip4" && \'192.168.1.0/24\' AND \'192.168.1.0/24\' && ANY("target_ip")) OR ("target_ip_aggr_ip6" && \'2001::/54\' AND \'2001::/54\' && ANY("target_ip")))'
+ b'SELECT "events"."id","events"."detecttime","events"."category","events"."description","events"."source_ip","events"."target_ip","events"."source_ip_aggr_ip4","events"."source_ip_aggr_ip6","events"."target_ip_aggr_ip4","events"."target_ip_aggr_ip6","events"."source_port","events"."target_port","events"."source_type","events"."target_type","events"."protocol","events"."node_name","events"."node_type","events"."cesnet_resolvedabuses","events"."cesnet_storagetime","events"."cesnet_eventclass","events"."cesnet_eventseverity","events"."cesnet_inspectionerrors","events_json"."event" FROM events INNER JOIN events_json ON events.id = events_json.id WHERE (("target_ip_aggr_ip4" && \'192.168.1.0/24\' AND \'192.168.1.0/24\' && ANY("target_ip")) OR ("target_ip_aggr_ip6" && \'2001::/54\' AND \'2001::/54\' && ANY("target_ip")))'
),
(
{
@@ -434,7 +434,7 @@ class TestMentatStorage(unittest.TestCase):
'host_addrs': ['192.168.1.0/24']
}
},
- b'SELECT "id","detecttime","category","description","source_ip","target_ip","source_ip_aggr_ip4","source_ip_aggr_ip6","target_ip_aggr_ip4","target_ip_aggr_ip6","source_port","target_port","source_type","target_type","protocol","node_name","node_type","cesnet_resolvedabuses","cesnet_storagetime","cesnet_eventclass","cesnet_eventseverity","cesnet_inspectionerrors","event" FROM events WHERE (("source_ip_aggr_ip4" && \'192.168.1.0/24\' AND \'192.168.1.0/24\' && ANY("source_ip")) OR ("target_ip_aggr_ip4" && \'192.168.1.0/24\' AND \'192.168.1.0/24\' && ANY("target_ip")))'
+ b'SELECT "events"."id","events"."detecttime","events"."category","events"."description","events"."source_ip","events"."target_ip","events"."source_ip_aggr_ip4","events"."source_ip_aggr_ip6","events"."target_ip_aggr_ip4","events"."target_ip_aggr_ip6","events"."source_port","events"."target_port","events"."source_type","events"."target_type","events"."protocol","events"."node_name","events"."node_type","events"."cesnet_resolvedabuses","events"."cesnet_storagetime","events"."cesnet_eventclass","events"."cesnet_eventseverity","events"."cesnet_inspectionerrors","events_json"."event" FROM events INNER JOIN events_json ON events.id = events_json.id WHERE (("source_ip_aggr_ip4" && \'192.168.1.0/24\' AND \'192.168.1.0/24\' && ANY("source_ip")) OR ("target_ip_aggr_ip4" && \'192.168.1.0/24\' AND \'192.168.1.0/24\' && ANY("target_ip")))'
),
(
{
@@ -442,7 +442,7 @@ class TestMentatStorage(unittest.TestCase):
'source_ports': [22,443]
}
},
- b'SELECT "id","detecttime","category","description","source_ip","target_ip","source_ip_aggr_ip4","source_ip_aggr_ip6","target_ip_aggr_ip4","target_ip_aggr_ip6","source_port","target_port","source_type","target_type","protocol","node_name","node_type","cesnet_resolvedabuses","cesnet_storagetime","cesnet_eventclass","cesnet_eventseverity","cesnet_inspectionerrors","event" FROM events WHERE "source_port" && ARRAY[22,443]'
+ b'SELECT "events"."id","events"."detecttime","events"."category","events"."description","events"."source_ip","events"."target_ip","events"."source_ip_aggr_ip4","events"."source_ip_aggr_ip6","events"."target_ip_aggr_ip4","events"."target_ip_aggr_ip6","events"."source_port","events"."target_port","events"."source_type","events"."target_type","events"."protocol","events"."node_name","events"."node_type","events"."cesnet_resolvedabuses","events"."cesnet_storagetime","events"."cesnet_eventclass","events"."cesnet_eventseverity","events"."cesnet_inspectionerrors","events_json"."event" FROM events INNER JOIN events_json ON events.id = events_json.id WHERE "source_port" && ARRAY[22,443]'
),
(
{
@@ -450,7 +450,7 @@ class TestMentatStorage(unittest.TestCase):
'target_ports': [22,443]
}
},
- b'SELECT "id","detecttime","category","description","source_ip","target_ip","source_ip_aggr_ip4","source_ip_aggr_ip6","target_ip_aggr_ip4","target_ip_aggr_ip6","source_port","target_port","source_type","target_type","protocol","node_name","node_type","cesnet_resolvedabuses","cesnet_storagetime","cesnet_eventclass","cesnet_eventseverity","cesnet_inspectionerrors","event" FROM events WHERE "target_port" && ARRAY[22,443]'
+ b'SELECT "events"."id","events"."detecttime","events"."category","events"."description","events"."source_ip","events"."target_ip","events"."source_ip_aggr_ip4","events"."source_ip_aggr_ip6","events"."target_ip_aggr_ip4","events"."target_ip_aggr_ip6","events"."source_port","events"."target_port","events"."source_type","events"."target_type","events"."protocol","events"."node_name","events"."node_type","events"."cesnet_resolvedabuses","events"."cesnet_storagetime","events"."cesnet_eventclass","events"."cesnet_eventseverity","events"."cesnet_inspectionerrors","events_json"."event" FROM events INNER JOIN events_json ON events.id = events_json.id WHERE "target_port" && ARRAY[22,443]'
),
(
{
@@ -459,7 +459,7 @@ class TestMentatStorage(unittest.TestCase):
'source_ports': [22,443]
}
},
- b'SELECT "id","detecttime","category","description","source_ip","target_ip","source_ip_aggr_ip4","source_ip_aggr_ip6","target_ip_aggr_ip4","target_ip_aggr_ip6","source_port","target_port","source_type","target_type","protocol","node_name","node_type","cesnet_resolvedabuses","cesnet_storagetime","cesnet_eventclass","cesnet_eventseverity","cesnet_inspectionerrors","event" FROM events WHERE ("source_port" && ARRAY[22,443] OR "target_port" && ARRAY[22,443])'
+ b'SELECT "events"."id","events"."detecttime","events"."category","events"."description","events"."source_ip","events"."target_ip","events"."source_ip_aggr_ip4","events"."source_ip_aggr_ip6","events"."target_ip_aggr_ip4","events"."target_ip_aggr_ip6","events"."source_port","events"."target_port","events"."source_type","events"."target_type","events"."protocol","events"."node_name","events"."node_type","events"."cesnet_resolvedabuses","events"."cesnet_storagetime","events"."cesnet_eventclass","events"."cesnet_eventseverity","events"."cesnet_inspectionerrors","events_json"."event" FROM events INNER JOIN events_json ON events.id = events_json.id WHERE ("source_port" && ARRAY[22,443] OR "target_port" && ARRAY[22,443])'
),
(
{
@@ -467,7 +467,7 @@ class TestMentatStorage(unittest.TestCase):
'source_types': ['Test','Tag']
}
},
- b'SELECT "id","detecttime","category","description","source_ip","target_ip","source_ip_aggr_ip4","source_ip_aggr_ip6","target_ip_aggr_ip4","target_ip_aggr_ip6","source_port","target_port","source_type","target_type","protocol","node_name","node_type","cesnet_resolvedabuses","cesnet_storagetime","cesnet_eventclass","cesnet_eventseverity","cesnet_inspectionerrors","event" FROM events WHERE "source_type" && ARRAY[\'Test\',\'Tag\']'
+ b'SELECT "events"."id","events"."detecttime","events"."category","events"."description","events"."source_ip","events"."target_ip","events"."source_ip_aggr_ip4","events"."source_ip_aggr_ip6","events"."target_ip_aggr_ip4","events"."target_ip_aggr_ip6","events"."source_port","events"."target_port","events"."source_type","events"."target_type","events"."protocol","events"."node_name","events"."node_type","events"."cesnet_resolvedabuses","events"."cesnet_storagetime","events"."cesnet_eventclass","events"."cesnet_eventseverity","events"."cesnet_inspectionerrors","events_json"."event" FROM events INNER JOIN events_json ON events.id = events_json.id WHERE "source_type" && ARRAY[\'Test\',\'Tag\']'
),
(
{
@@ -475,7 +475,7 @@ class TestMentatStorage(unittest.TestCase):
'target_types': ['Test','Tag']
}
},
- b'SELECT "id","detecttime","category","description","source_ip","target_ip","source_ip_aggr_ip4","source_ip_aggr_ip6","target_ip_aggr_ip4","target_ip_aggr_ip6","source_port","target_port","source_type","target_type","protocol","node_name","node_type","cesnet_resolvedabuses","cesnet_storagetime","cesnet_eventclass","cesnet_eventseverity","cesnet_inspectionerrors","event" FROM events WHERE "target_type" && ARRAY[\'Test\',\'Tag\']'
+ b'SELECT "events"."id","events"."detecttime","events"."category","events"."description","events"."source_ip","events"."target_ip","events"."source_ip_aggr_ip4","events"."source_ip_aggr_ip6","events"."target_ip_aggr_ip4","events"."target_ip_aggr_ip6","events"."source_port","events"."target_port","events"."source_type","events"."target_type","events"."protocol","events"."node_name","events"."node_type","events"."cesnet_resolvedabuses","events"."cesnet_storagetime","events"."cesnet_eventclass","events"."cesnet_eventseverity","events"."cesnet_inspectionerrors","events_json"."event" FROM events INNER JOIN events_json ON events.id = events_json.id WHERE "target_type" && ARRAY[\'Test\',\'Tag\']'
),
(
{
@@ -484,7 +484,7 @@ class TestMentatStorage(unittest.TestCase):
'target_types': ['Test','Tag']
}
},
- b'SELECT "id","detecttime","category","description","source_ip","target_ip","source_ip_aggr_ip4","source_ip_aggr_ip6","target_ip_aggr_ip4","target_ip_aggr_ip6","source_port","target_port","source_type","target_type","protocol","node_name","node_type","cesnet_resolvedabuses","cesnet_storagetime","cesnet_eventclass","cesnet_eventseverity","cesnet_inspectionerrors","event" FROM events WHERE ("source_type" && ARRAY[\'Test\',\'Tag\'] OR "target_type" && ARRAY[\'Test\',\'Tag\'])'
+ b'SELECT "events"."id","events"."detecttime","events"."category","events"."description","events"."source_ip","events"."target_ip","events"."source_ip_aggr_ip4","events"."source_ip_aggr_ip6","events"."target_ip_aggr_ip4","events"."target_ip_aggr_ip6","events"."source_port","events"."target_port","events"."source_type","events"."target_type","events"."protocol","events"."node_name","events"."node_type","events"."cesnet_resolvedabuses","events"."cesnet_storagetime","events"."cesnet_eventclass","events"."cesnet_eventseverity","events"."cesnet_inspectionerrors","events_json"."event" FROM events INNER JOIN events_json ON events.id = events_json.id WHERE ("source_type" && ARRAY[\'Test\',\'Tag\'] OR "target_type" && ARRAY[\'Test\',\'Tag\'])'
),
(
{
@@ -492,7 +492,7 @@ class TestMentatStorage(unittest.TestCase):
'protocols': ['tcp', 'ssh']
}
},
- b'SELECT "id","detecttime","category","description","source_ip","target_ip","source_ip_aggr_ip4","source_ip_aggr_ip6","target_ip_aggr_ip4","target_ip_aggr_ip6","source_port","target_port","source_type","target_type","protocol","node_name","node_type","cesnet_resolvedabuses","cesnet_storagetime","cesnet_eventclass","cesnet_eventseverity","cesnet_inspectionerrors","event" FROM events WHERE "protocol" && ARRAY[\'tcp\',\'ssh\']'
+ b'SELECT "events"."id","events"."detecttime","events"."category","events"."description","events"."source_ip","events"."target_ip","events"."source_ip_aggr_ip4","events"."source_ip_aggr_ip6","events"."target_ip_aggr_ip4","events"."target_ip_aggr_ip6","events"."source_port","events"."target_port","events"."source_type","events"."target_type","events"."protocol","events"."node_name","events"."node_type","events"."cesnet_resolvedabuses","events"."cesnet_storagetime","events"."cesnet_eventclass","events"."cesnet_eventseverity","events"."cesnet_inspectionerrors","events_json"."event" FROM events INNER JOIN events_json ON events.id = events_json.id WHERE "protocol" && ARRAY[\'tcp\',\'ssh\']'
),
(
{
@@ -501,7 +501,7 @@ class TestMentatStorage(unittest.TestCase):
'not_protocols': True
}
},
- b'SELECT "id","detecttime","category","description","source_ip","target_ip","source_ip_aggr_ip4","source_ip_aggr_ip6","target_ip_aggr_ip4","target_ip_aggr_ip6","source_port","target_port","source_type","target_type","protocol","node_name","node_type","cesnet_resolvedabuses","cesnet_storagetime","cesnet_eventclass","cesnet_eventseverity","cesnet_inspectionerrors","event" FROM events WHERE NOT ("protocol" && ARRAY[\'tcp\',\'ssh\'])'
+ b'SELECT "events"."id","events"."detecttime","events"."category","events"."description","events"."source_ip","events"."target_ip","events"."source_ip_aggr_ip4","events"."source_ip_aggr_ip6","events"."target_ip_aggr_ip4","events"."target_ip_aggr_ip6","events"."source_port","events"."target_port","events"."source_type","events"."target_type","events"."protocol","events"."node_name","events"."node_type","events"."cesnet_resolvedabuses","events"."cesnet_storagetime","events"."cesnet_eventclass","events"."cesnet_eventseverity","events"."cesnet_inspectionerrors","events_json"."event" FROM events INNER JOIN events_json ON events.id = events_json.id WHERE NOT ("protocol" && ARRAY[\'tcp\',\'ssh\'])'
),
(
{
@@ -509,7 +509,7 @@ class TestMentatStorage(unittest.TestCase):
'protocols': ['__EMPTY__']
}
},
- b'SELECT "id","detecttime","category","description","source_ip","target_ip","source_ip_aggr_ip4","source_ip_aggr_ip6","target_ip_aggr_ip4","target_ip_aggr_ip6","source_port","target_port","source_type","target_type","protocol","node_name","node_type","cesnet_resolvedabuses","cesnet_storagetime","cesnet_eventclass","cesnet_eventseverity","cesnet_inspectionerrors","event" FROM events WHERE "protocol" = \'{}\''
+ b'SELECT "events"."id","events"."detecttime","events"."category","events"."description","events"."source_ip","events"."target_ip","events"."source_ip_aggr_ip4","events"."source_ip_aggr_ip6","events"."target_ip_aggr_ip4","events"."target_ip_aggr_ip6","events"."source_port","events"."target_port","events"."source_type","events"."target_type","events"."protocol","events"."node_name","events"."node_type","events"."cesnet_resolvedabuses","events"."cesnet_storagetime","events"."cesnet_eventclass","events"."cesnet_eventseverity","events"."cesnet_inspectionerrors","events_json"."event" FROM events INNER JOIN events_json ON events.id = events_json.id WHERE "protocol" = \'{}\''
),
(
{
@@ -517,7 +517,7 @@ class TestMentatStorage(unittest.TestCase):
'protocols': ['__ANY__']
}
},
- b'SELECT "id","detecttime","category","description","source_ip","target_ip","source_ip_aggr_ip4","source_ip_aggr_ip6","target_ip_aggr_ip4","target_ip_aggr_ip6","source_port","target_port","source_type","target_type","protocol","node_name","node_type","cesnet_resolvedabuses","cesnet_storagetime","cesnet_eventclass","cesnet_eventseverity","cesnet_inspectionerrors","event" FROM events WHERE "protocol" != \'{}\''
+ b'SELECT "events"."id","events"."detecttime","events"."category","events"."description","events"."source_ip","events"."target_ip","events"."source_ip_aggr_ip4","events"."source_ip_aggr_ip6","events"."target_ip_aggr_ip4","events"."target_ip_aggr_ip6","events"."source_port","events"."target_port","events"."source_type","events"."target_type","events"."protocol","events"."node_name","events"."node_type","events"."cesnet_resolvedabuses","events"."cesnet_storagetime","events"."cesnet_eventclass","events"."cesnet_eventseverity","events"."cesnet_inspectionerrors","events_json"."event" FROM events INNER JOIN events_json ON events.id = events_json.id WHERE "protocol" != \'{}\''
),
(
{
@@ -525,7 +525,7 @@ class TestMentatStorage(unittest.TestCase):
'categories': ['Test', 'Category']
}
},
- b'SELECT "id","detecttime","category","description","source_ip","target_ip","source_ip_aggr_ip4","source_ip_aggr_ip6","target_ip_aggr_ip4","target_ip_aggr_ip6","source_port","target_port","source_type","target_type","protocol","node_name","node_type","cesnet_resolvedabuses","cesnet_storagetime","cesnet_eventclass","cesnet_eventseverity","cesnet_inspectionerrors","event" FROM events WHERE "category" && ARRAY[\'Test\',\'Category\']'
+ b'SELECT "events"."id","events"."detecttime","events"."category","events"."description","events"."source_ip","events"."target_ip","events"."source_ip_aggr_ip4","events"."source_ip_aggr_ip6","events"."target_ip_aggr_ip4","events"."target_ip_aggr_ip6","events"."source_port","events"."target_port","events"."source_type","events"."target_type","events"."protocol","events"."node_name","events"."node_type","events"."cesnet_resolvedabuses","events"."cesnet_storagetime","events"."cesnet_eventclass","events"."cesnet_eventseverity","events"."cesnet_inspectionerrors","events_json"."event" FROM events INNER JOIN events_json ON events.id = events_json.id WHERE "category" && ARRAY[\'Test\',\'Category\']'
),
(
{
@@ -534,7 +534,7 @@ class TestMentatStorage(unittest.TestCase):
'not_categories': True
}
},
- b'SELECT "id","detecttime","category","description","source_ip","target_ip","source_ip_aggr_ip4","source_ip_aggr_ip6","target_ip_aggr_ip4","target_ip_aggr_ip6","source_port","target_port","source_type","target_type","protocol","node_name","node_type","cesnet_resolvedabuses","cesnet_storagetime","cesnet_eventclass","cesnet_eventseverity","cesnet_inspectionerrors","event" FROM events WHERE NOT ("category" && ARRAY[\'Test\',\'Category\'])'
+ b'SELECT "events"."id","events"."detecttime","events"."category","events"."description","events"."source_ip","events"."target_ip","events"."source_ip_aggr_ip4","events"."source_ip_aggr_ip6","events"."target_ip_aggr_ip4","events"."target_ip_aggr_ip6","events"."source_port","events"."target_port","events"."source_type","events"."target_type","events"."protocol","events"."node_name","events"."node_type","events"."cesnet_resolvedabuses","events"."cesnet_storagetime","events"."cesnet_eventclass","events"."cesnet_eventseverity","events"."cesnet_inspectionerrors","events_json"."event" FROM events INNER JOIN events_json ON events.id = events_json.id WHERE NOT ("category" && ARRAY[\'Test\',\'Category\'])'
),
(
{
@@ -542,7 +542,7 @@ class TestMentatStorage(unittest.TestCase):
'categories': ['__EMPTY__']
}
},
- b'SELECT "id","detecttime","category","description","source_ip","target_ip","source_ip_aggr_ip4","source_ip_aggr_ip6","target_ip_aggr_ip4","target_ip_aggr_ip6","source_port","target_port","source_type","target_type","protocol","node_name","node_type","cesnet_resolvedabuses","cesnet_storagetime","cesnet_eventclass","cesnet_eventseverity","cesnet_inspectionerrors","event" FROM events WHERE "category" = \'{}\''
+ b'SELECT "events"."id","events"."detecttime","events"."category","events"."description","events"."source_ip","events"."target_ip","events"."source_ip_aggr_ip4","events"."source_ip_aggr_ip6","events"."target_ip_aggr_ip4","events"."target_ip_aggr_ip6","events"."source_port","events"."target_port","events"."source_type","events"."target_type","events"."protocol","events"."node_name","events"."node_type","events"."cesnet_resolvedabuses","events"."cesnet_storagetime","events"."cesnet_eventclass","events"."cesnet_eventseverity","events"."cesnet_inspectionerrors","events_json"."event" FROM events INNER JOIN events_json ON events.id = events_json.id WHERE "category" = \'{}\''
),
(
{
@@ -550,7 +550,7 @@ class TestMentatStorage(unittest.TestCase):
'categories': ['__ANY__']
}
},
- b'SELECT "id","detecttime","category","description","source_ip","target_ip","source_ip_aggr_ip4","source_ip_aggr_ip6","target_ip_aggr_ip4","target_ip_aggr_ip6","source_port","target_port","source_type","target_type","protocol","node_name","node_type","cesnet_resolvedabuses","cesnet_storagetime","cesnet_eventclass","cesnet_eventseverity","cesnet_inspectionerrors","event" FROM events WHERE "category" != \'{}\''
+ b'SELECT "events"."id","events"."detecttime","events"."category","events"."description","events"."source_ip","events"."target_ip","events"."source_ip_aggr_ip4","events"."source_ip_aggr_ip6","events"."target_ip_aggr_ip4","events"."target_ip_aggr_ip6","events"."source_port","events"."target_port","events"."source_type","events"."target_type","events"."protocol","events"."node_name","events"."node_type","events"."cesnet_resolvedabuses","events"."cesnet_storagetime","events"."cesnet_eventclass","events"."cesnet_eventseverity","events"."cesnet_inspectionerrors","events_json"."event" FROM events INNER JOIN events_json ON events.id = events_json.id WHERE "category" != \'{}\''
),
(
{
@@ -558,7 +558,7 @@ class TestMentatStorage(unittest.TestCase):
'classes': ['test', 'vulnerable-config-ssdp']
}
},
- b'SELECT "id","detecttime","category","description","source_ip","target_ip","source_ip_aggr_ip4","source_ip_aggr_ip6","target_ip_aggr_ip4","target_ip_aggr_ip6","source_port","target_port","source_type","target_type","protocol","node_name","node_type","cesnet_resolvedabuses","cesnet_storagetime","cesnet_eventclass","cesnet_eventseverity","cesnet_inspectionerrors","event" FROM events WHERE "cesnet_eventclass" = ANY(ARRAY[\'test\',\'vulnerable-config-ssdp\'])'
+ b'SELECT "events"."id","events"."detecttime","events"."category","events"."description","events"."source_ip","events"."target_ip","events"."source_ip_aggr_ip4","events"."source_ip_aggr_ip6","events"."target_ip_aggr_ip4","events"."target_ip_aggr_ip6","events"."source_port","events"."target_port","events"."source_type","events"."target_type","events"."protocol","events"."node_name","events"."node_type","events"."cesnet_resolvedabuses","events"."cesnet_storagetime","events"."cesnet_eventclass","events"."cesnet_eventseverity","events"."cesnet_inspectionerrors","events_json"."event" FROM events INNER JOIN events_json ON events.id = events_json.id WHERE "cesnet_eventclass" = ANY(ARRAY[\'test\',\'vulnerable-config-ssdp\'])'
),
(
{
@@ -567,7 +567,7 @@ class TestMentatStorage(unittest.TestCase):
'not_classes': True
}
},
- b'SELECT "id","detecttime","category","description","source_ip","target_ip","source_ip_aggr_ip4","source_ip_aggr_ip6","target_ip_aggr_ip4","target_ip_aggr_ip6","source_port","target_port","source_type","target_type","protocol","node_name","node_type","cesnet_resolvedabuses","cesnet_storagetime","cesnet_eventclass","cesnet_eventseverity","cesnet_inspectionerrors","event" FROM events WHERE NOT ("cesnet_eventclass" = ANY(ARRAY[\'test\',\'vulnerable-config-ssdp\']))'
+ b'SELECT "events"."id","events"."detecttime","events"."category","events"."description","events"."source_ip","events"."target_ip","events"."source_ip_aggr_ip4","events"."source_ip_aggr_ip6","events"."target_ip_aggr_ip4","events"."target_ip_aggr_ip6","events"."source_port","events"."target_port","events"."source_type","events"."target_type","events"."protocol","events"."node_name","events"."node_type","events"."cesnet_resolvedabuses","events"."cesnet_storagetime","events"."cesnet_eventclass","events"."cesnet_eventseverity","events"."cesnet_inspectionerrors","events_json"."event" FROM events INNER JOIN events_json ON events.id = events_json.id WHERE NOT ("cesnet_eventclass" = ANY(ARRAY[\'test\',\'vulnerable-config-ssdp\']))'
),
(
{
@@ -575,7 +575,7 @@ class TestMentatStorage(unittest.TestCase):
'classes': ['__EMPTY__']
}
},
- b'SELECT "id","detecttime","category","description","source_ip","target_ip","source_ip_aggr_ip4","source_ip_aggr_ip6","target_ip_aggr_ip4","target_ip_aggr_ip6","source_port","target_port","source_type","target_type","protocol","node_name","node_type","cesnet_resolvedabuses","cesnet_storagetime","cesnet_eventclass","cesnet_eventseverity","cesnet_inspectionerrors","event" FROM events WHERE COALESCE("cesnet_eventclass",\'\') = \'\''
+ b'SELECT "events"."id","events"."detecttime","events"."category","events"."description","events"."source_ip","events"."target_ip","events"."source_ip_aggr_ip4","events"."source_ip_aggr_ip6","events"."target_ip_aggr_ip4","events"."target_ip_aggr_ip6","events"."source_port","events"."target_port","events"."source_type","events"."target_type","events"."protocol","events"."node_name","events"."node_type","events"."cesnet_resolvedabuses","events"."cesnet_storagetime","events"."cesnet_eventclass","events"."cesnet_eventseverity","events"."cesnet_inspectionerrors","events_json"."event" FROM events INNER JOIN events_json ON events.id = events_json.id WHERE COALESCE("cesnet_eventclass",\'\') = \'\''
),
(
{
@@ -583,7 +583,7 @@ class TestMentatStorage(unittest.TestCase):
'classes': ['__ANY__']
}
},
- b'SELECT "id","detecttime","category","description","source_ip","target_ip","source_ip_aggr_ip4","source_ip_aggr_ip6","target_ip_aggr_ip4","target_ip_aggr_ip6","source_port","target_port","source_type","target_type","protocol","node_name","node_type","cesnet_resolvedabuses","cesnet_storagetime","cesnet_eventclass","cesnet_eventseverity","cesnet_inspectionerrors","event" FROM events WHERE COALESCE("cesnet_eventclass",\'\') != \'\''
+ b'SELECT "events"."id","events"."detecttime","events"."category","events"."description","events"."source_ip","events"."target_ip","events"."source_ip_aggr_ip4","events"."source_ip_aggr_ip6","events"."target_ip_aggr_ip4","events"."target_ip_aggr_ip6","events"."source_port","events"."target_port","events"."source_type","events"."target_type","events"."protocol","events"."node_name","events"."node_type","events"."cesnet_resolvedabuses","events"."cesnet_storagetime","events"."cesnet_eventclass","events"."cesnet_eventseverity","events"."cesnet_inspectionerrors","events_json"."event" FROM events INNER JOIN events_json ON events.id = events_json.id WHERE COALESCE("cesnet_eventclass",\'\') != \'\''
),
(
{
@@ -591,7 +591,7 @@ class TestMentatStorage(unittest.TestCase):
'severities': ['test', 'low']
}
},
- b'SELECT "id","detecttime","category","description","source_ip","target_ip","source_ip_aggr_ip4","source_ip_aggr_ip6","target_ip_aggr_ip4","target_ip_aggr_ip6","source_port","target_port","source_type","target_type","protocol","node_name","node_type","cesnet_resolvedabuses","cesnet_storagetime","cesnet_eventclass","cesnet_eventseverity","cesnet_inspectionerrors","event" FROM events WHERE "cesnet_eventseverity" = ANY(ARRAY[\'test\',\'low\'])'
+ b'SELECT "events"."id","events"."detecttime","events"."category","events"."description","events"."source_ip","events"."target_ip","events"."source_ip_aggr_ip4","events"."source_ip_aggr_ip6","events"."target_ip_aggr_ip4","events"."target_ip_aggr_ip6","events"."source_port","events"."target_port","events"."source_type","events"."target_type","events"."protocol","events"."node_name","events"."node_type","events"."cesnet_resolvedabuses","events"."cesnet_storagetime","events"."cesnet_eventclass","events"."cesnet_eventseverity","events"."cesnet_inspectionerrors","events_json"."event" FROM events INNER JOIN events_json ON events.id = events_json.id WHERE "cesnet_eventseverity" = ANY(ARRAY[\'test\',\'low\'])'
),
(
{
@@ -600,7 +600,7 @@ class TestMentatStorage(unittest.TestCase):
'not_severities': True
}
},
- b'SELECT "id","detecttime","category","description","source_ip","target_ip","source_ip_aggr_ip4","source_ip_aggr_ip6","target_ip_aggr_ip4","target_ip_aggr_ip6","source_port","target_port","source_type","target_type","protocol","node_name","node_type","cesnet_resolvedabuses","cesnet_storagetime","cesnet_eventclass","cesnet_eventseverity","cesnet_inspectionerrors","event" FROM events WHERE NOT ("cesnet_eventseverity" = ANY(ARRAY[\'test\',\'low\']))'
+ b'SELECT "events"."id","events"."detecttime","events"."category","events"."description","events"."source_ip","events"."target_ip","events"."source_ip_aggr_ip4","events"."source_ip_aggr_ip6","events"."target_ip_aggr_ip4","events"."target_ip_aggr_ip6","events"."source_port","events"."target_port","events"."source_type","events"."target_type","events"."protocol","events"."node_name","events"."node_type","events"."cesnet_resolvedabuses","events"."cesnet_storagetime","events"."cesnet_eventclass","events"."cesnet_eventseverity","events"."cesnet_inspectionerrors","events_json"."event" FROM events INNER JOIN events_json ON events.id = events_json.id WHERE NOT ("cesnet_eventseverity" = ANY(ARRAY[\'test\',\'low\']))'
),
(
{
@@ -608,7 +608,7 @@ class TestMentatStorage(unittest.TestCase):
'severities': ['__EMPTY__']
}
},
- b'SELECT "id","detecttime","category","description","source_ip","target_ip","source_ip_aggr_ip4","source_ip_aggr_ip6","target_ip_aggr_ip4","target_ip_aggr_ip6","source_port","target_port","source_type","target_type","protocol","node_name","node_type","cesnet_resolvedabuses","cesnet_storagetime","cesnet_eventclass","cesnet_eventseverity","cesnet_inspectionerrors","event" FROM events WHERE COALESCE("cesnet_eventseverity",\'\') = \'\''
+ b'SELECT "events"."id","events"."detecttime","events"."category","events"."description","events"."source_ip","events"."target_ip","events"."source_ip_aggr_ip4","events"."source_ip_aggr_ip6","events"."target_ip_aggr_ip4","events"."target_ip_aggr_ip6","events"."source_port","events"."target_port","events"."source_type","events"."target_type","events"."protocol","events"."node_name","events"."node_type","events"."cesnet_resolvedabuses","events"."cesnet_storagetime","events"."cesnet_eventclass","events"."cesnet_eventseverity","events"."cesnet_inspectionerrors","events_json"."event" FROM events INNER JOIN events_json ON events.id = events_json.id WHERE COALESCE("cesnet_eventseverity",\'\') = \'\''
),
(
{
@@ -616,7 +616,7 @@ class TestMentatStorage(unittest.TestCase):
'severities': ['__ANY__']
}
},
- b'SELECT "id","detecttime","category","description","source_ip","target_ip","source_ip_aggr_ip4","source_ip_aggr_ip6","target_ip_aggr_ip4","target_ip_aggr_ip6","source_port","target_port","source_type","target_type","protocol","node_name","node_type","cesnet_resolvedabuses","cesnet_storagetime","cesnet_eventclass","cesnet_eventseverity","cesnet_inspectionerrors","event" FROM events WHERE COALESCE("cesnet_eventseverity",\'\') != \'\''
+ b'SELECT "events"."id","events"."detecttime","events"."category","events"."description","events"."source_ip","events"."target_ip","events"."source_ip_aggr_ip4","events"."source_ip_aggr_ip6","events"."target_ip_aggr_ip4","events"."target_ip_aggr_ip6","events"."source_port","events"."target_port","events"."source_type","events"."target_type","events"."protocol","events"."node_name","events"."node_type","events"."cesnet_resolvedabuses","events"."cesnet_storagetime","events"."cesnet_eventclass","events"."cesnet_eventseverity","events"."cesnet_inspectionerrors","events_json"."event" FROM events INNER JOIN events_json ON events.id = events_json.id WHERE COALESCE("cesnet_eventseverity",\'\') != \'\''
),
(
{
@@ -624,7 +624,7 @@ class TestMentatStorage(unittest.TestCase):
'detectors': ['cz.cesnet.kippo', 'cz.cesnet.dionaea']
}
},
- b'SELECT "id","detecttime","category","description","source_ip","target_ip","source_ip_aggr_ip4","source_ip_aggr_ip6","target_ip_aggr_ip4","target_ip_aggr_ip6","source_port","target_port","source_type","target_type","protocol","node_name","node_type","cesnet_resolvedabuses","cesnet_storagetime","cesnet_eventclass","cesnet_eventseverity","cesnet_inspectionerrors","event" FROM events WHERE "node_name" && ARRAY[\'cz.cesnet.kippo\',\'cz.cesnet.dionaea\']'
+ b'SELECT "events"."id","events"."detecttime","events"."category","events"."description","events"."source_ip","events"."target_ip","events"."source_ip_aggr_ip4","events"."source_ip_aggr_ip6","events"."target_ip_aggr_ip4","events"."target_ip_aggr_ip6","events"."source_port","events"."target_port","events"."source_type","events"."target_type","events"."protocol","events"."node_name","events"."node_type","events"."cesnet_resolvedabuses","events"."cesnet_storagetime","events"."cesnet_eventclass","events"."cesnet_eventseverity","events"."cesnet_inspectionerrors","events_json"."event" FROM events INNER JOIN events_json ON events.id = events_json.id WHERE "node_name" && ARRAY[\'cz.cesnet.kippo\',\'cz.cesnet.dionaea\']'
),
(
{
@@ -633,7 +633,7 @@ class TestMentatStorage(unittest.TestCase):
'not_detectors': True
}
},
- b'SELECT "id","detecttime","category","description","source_ip","target_ip","source_ip_aggr_ip4","source_ip_aggr_ip6","target_ip_aggr_ip4","target_ip_aggr_ip6","source_port","target_port","source_type","target_type","protocol","node_name","node_type","cesnet_resolvedabuses","cesnet_storagetime","cesnet_eventclass","cesnet_eventseverity","cesnet_inspectionerrors","event" FROM events WHERE NOT ("node_name" && ARRAY[\'cz.cesnet.kippo\',\'cz.cesnet.dionaea\'])'
+ b'SELECT "events"."id","events"."detecttime","events"."category","events"."description","events"."source_ip","events"."target_ip","events"."source_ip_aggr_ip4","events"."source_ip_aggr_ip6","events"."target_ip_aggr_ip4","events"."target_ip_aggr_ip6","events"."source_port","events"."target_port","events"."source_type","events"."target_type","events"."protocol","events"."node_name","events"."node_type","events"."cesnet_resolvedabuses","events"."cesnet_storagetime","events"."cesnet_eventclass","events"."cesnet_eventseverity","events"."cesnet_inspectionerrors","events_json"."event" FROM events INNER JOIN events_json ON events.id = events_json.id WHERE NOT ("node_name" && ARRAY[\'cz.cesnet.kippo\',\'cz.cesnet.dionaea\'])'
),
(
{
@@ -641,7 +641,7 @@ class TestMentatStorage(unittest.TestCase):
'detectors': ['__EMPTY__']
}
},
- b'SELECT "id","detecttime","category","description","source_ip","target_ip","source_ip_aggr_ip4","source_ip_aggr_ip6","target_ip_aggr_ip4","target_ip_aggr_ip6","source_port","target_port","source_type","target_type","protocol","node_name","node_type","cesnet_resolvedabuses","cesnet_storagetime","cesnet_eventclass","cesnet_eventseverity","cesnet_inspectionerrors","event" FROM events WHERE "node_name" = \'{}\''
+ b'SELECT "events"."id","events"."detecttime","events"."category","events"."description","events"."source_ip","events"."target_ip","events"."source_ip_aggr_ip4","events"."source_ip_aggr_ip6","events"."target_ip_aggr_ip4","events"."target_ip_aggr_ip6","events"."source_port","events"."target_port","events"."source_type","events"."target_type","events"."protocol","events"."node_name","events"."node_type","events"."cesnet_resolvedabuses","events"."cesnet_storagetime","events"."cesnet_eventclass","events"."cesnet_eventseverity","events"."cesnet_inspectionerrors","events_json"."event" FROM events INNER JOIN events_json ON events.id = events_json.id WHERE "node_name" = \'{}\''
),
(
{
@@ -649,7 +649,7 @@ class TestMentatStorage(unittest.TestCase):
'detectors': ['__ANY__']
}
},
- b'SELECT "id","detecttime","category","description","source_ip","target_ip","source_ip_aggr_ip4","source_ip_aggr_ip6","target_ip_aggr_ip4","target_ip_aggr_ip6","source_port","target_port","source_type","target_type","protocol","node_name","node_type","cesnet_resolvedabuses","cesnet_storagetime","cesnet_eventclass","cesnet_eventseverity","cesnet_inspectionerrors","event" FROM events WHERE "node_name" != \'{}\''
+ b'SELECT "events"."id","events"."detecttime","events"."category","events"."description","events"."source_ip","events"."target_ip","events"."source_ip_aggr_ip4","events"."source_ip_aggr_ip6","events"."target_ip_aggr_ip4","events"."target_ip_aggr_ip6","events"."source_port","events"."target_port","events"."source_type","events"."target_type","events"."protocol","events"."node_name","events"."node_type","events"."cesnet_resolvedabuses","events"."cesnet_storagetime","events"."cesnet_eventclass","events"."cesnet_eventseverity","events"."cesnet_inspectionerrors","events_json"."event" FROM events INNER JOIN events_json ON events.id = events_json.id WHERE "node_name" != \'{}\''
),
(
{
@@ -657,7 +657,7 @@ class TestMentatStorage(unittest.TestCase):
'detector_types': ['Test', 'Tag']
}
},
- b'SELECT "id","detecttime","category","description","source_ip","target_ip","source_ip_aggr_ip4","source_ip_aggr_ip6","target_ip_aggr_ip4","target_ip_aggr_ip6","source_port","target_port","source_type","target_type","protocol","node_name","node_type","cesnet_resolvedabuses","cesnet_storagetime","cesnet_eventclass","cesnet_eventseverity","cesnet_inspectionerrors","event" FROM events WHERE "node_type" && ARRAY[\'Test\',\'Tag\']'
+ b'SELECT "events"."id","events"."detecttime","events"."category","events"."description","events"."source_ip","events"."target_ip","events"."source_ip_aggr_ip4","events"."source_ip_aggr_ip6","events"."target_ip_aggr_ip4","events"."target_ip_aggr_ip6","events"."source_port","events"."target_port","events"."source_type","events"."target_type","events"."protocol","events"."node_name","events"."node_type","events"."cesnet_resolvedabuses","events"."cesnet_storagetime","events"."cesnet_eventclass","events"."cesnet_eventseverity","events"."cesnet_inspectionerrors","events_json"."event" FROM events INNER JOIN events_json ON events.id = events_json.id WHERE "node_type" && ARRAY[\'Test\',\'Tag\']'
),
(
{
@@ -666,7 +666,7 @@ class TestMentatStorage(unittest.TestCase):
'not_detector_types': True
}
},
- b'SELECT "id","detecttime","category","description","source_ip","target_ip","source_ip_aggr_ip4","source_ip_aggr_ip6","target_ip_aggr_ip4","target_ip_aggr_ip6","source_port","target_port","source_type","target_type","protocol","node_name","node_type","cesnet_resolvedabuses","cesnet_storagetime","cesnet_eventclass","cesnet_eventseverity","cesnet_inspectionerrors","event" FROM events WHERE NOT ("node_type" && ARRAY[\'Test\',\'Tag\'])'
+ b'SELECT "events"."id","events"."detecttime","events"."category","events"."description","events"."source_ip","events"."target_ip","events"."source_ip_aggr_ip4","events"."source_ip_aggr_ip6","events"."target_ip_aggr_ip4","events"."target_ip_aggr_ip6","events"."source_port","events"."target_port","events"."source_type","events"."target_type","events"."protocol","events"."node_name","events"."node_type","events"."cesnet_resolvedabuses","events"."cesnet_storagetime","events"."cesnet_eventclass","events"."cesnet_eventseverity","events"."cesnet_inspectionerrors","events_json"."event" FROM events INNER JOIN events_json ON events.id = events_json.id WHERE NOT ("node_type" && ARRAY[\'Test\',\'Tag\'])'
),
(
{
@@ -674,7 +674,7 @@ class TestMentatStorage(unittest.TestCase):
'detector_types': ['__EMPTY__']
}
},
- b'SELECT "id","detecttime","category","description","source_ip","target_ip","source_ip_aggr_ip4","source_ip_aggr_ip6","target_ip_aggr_ip4","target_ip_aggr_ip6","source_port","target_port","source_type","target_type","protocol","node_name","node_type","cesnet_resolvedabuses","cesnet_storagetime","cesnet_eventclass","cesnet_eventseverity","cesnet_inspectionerrors","event" FROM events WHERE "node_type" = \'{}\''
+ b'SELECT "events"."id","events"."detecttime","events"."category","events"."description","events"."source_ip","events"."target_ip","events"."source_ip_aggr_ip4","events"."source_ip_aggr_ip6","events"."target_ip_aggr_ip4","events"."target_ip_aggr_ip6","events"."source_port","events"."target_port","events"."source_type","events"."target_type","events"."protocol","events"."node_name","events"."node_type","events"."cesnet_resolvedabuses","events"."cesnet_storagetime","events"."cesnet_eventclass","events"."cesnet_eventseverity","events"."cesnet_inspectionerrors","events_json"."event" FROM events INNER JOIN events_json ON events.id = events_json.id WHERE "node_type" = \'{}\''
),
(
{
@@ -682,7 +682,7 @@ class TestMentatStorage(unittest.TestCase):
'detector_types': ['__ANY__']
}
},
- b'SELECT "id","detecttime","category","description","source_ip","target_ip","source_ip_aggr_ip4","source_ip_aggr_ip6","target_ip_aggr_ip4","target_ip_aggr_ip6","source_port","target_port","source_type","target_type","protocol","node_name","node_type","cesnet_resolvedabuses","cesnet_storagetime","cesnet_eventclass","cesnet_eventseverity","cesnet_inspectionerrors","event" FROM events WHERE "node_type" != \'{}\''
+ b'SELECT "events"."id","events"."detecttime","events"."category","events"."description","events"."source_ip","events"."target_ip","events"."source_ip_aggr_ip4","events"."source_ip_aggr_ip6","events"."target_ip_aggr_ip4","events"."target_ip_aggr_ip6","events"."source_port","events"."target_port","events"."source_type","events"."target_type","events"."protocol","events"."node_name","events"."node_type","events"."cesnet_resolvedabuses","events"."cesnet_storagetime","events"."cesnet_eventclass","events"."cesnet_eventseverity","events"."cesnet_inspectionerrors","events_json"."event" FROM events INNER JOIN events_json ON events.id = events_json.id WHERE "node_type" != \'{}\''
),
(
{
@@ -690,7 +690,7 @@ class TestMentatStorage(unittest.TestCase):
'groups': ['abuse@cesnet.cz', 'abuse@nic.cz']
}
},
- b'SELECT "id","detecttime","category","description","source_ip","target_ip","source_ip_aggr_ip4","source_ip_aggr_ip6","target_ip_aggr_ip4","target_ip_aggr_ip6","source_port","target_port","source_type","target_type","protocol","node_name","node_type","cesnet_resolvedabuses","cesnet_storagetime","cesnet_eventclass","cesnet_eventseverity","cesnet_inspectionerrors","event" FROM events WHERE "cesnet_resolvedabuses" && ARRAY[\'abuse@cesnet.cz\',\'abuse@nic.cz\']'
+ b'SELECT "events"."id","events"."detecttime","events"."category","events"."description","events"."source_ip","events"."target_ip","events"."source_ip_aggr_ip4","events"."source_ip_aggr_ip6","events"."target_ip_aggr_ip4","events"."target_ip_aggr_ip6","events"."source_port","events"."target_port","events"."source_type","events"."target_type","events"."protocol","events"."node_name","events"."node_type","events"."cesnet_resolvedabuses","events"."cesnet_storagetime","events"."cesnet_eventclass","events"."cesnet_eventseverity","events"."cesnet_inspectionerrors","events_json"."event" FROM events INNER JOIN events_json ON events.id = events_json.id WHERE "cesnet_resolvedabuses" && ARRAY[\'abuse@cesnet.cz\',\'abuse@nic.cz\']'
),
(
{
@@ -699,7 +699,7 @@ class TestMentatStorage(unittest.TestCase):
'not_groups': True
}
},
- b'SELECT "id","detecttime","category","description","source_ip","target_ip","source_ip_aggr_ip4","source_ip_aggr_ip6","target_ip_aggr_ip4","target_ip_aggr_ip6","source_port","target_port","source_type","target_type","protocol","node_name","node_type","cesnet_resolvedabuses","cesnet_storagetime","cesnet_eventclass","cesnet_eventseverity","cesnet_inspectionerrors","event" FROM events WHERE NOT ("cesnet_resolvedabuses" && ARRAY[\'abuse@cesnet.cz\',\'abuse@nic.cz\'])'
+ b'SELECT "events"."id","events"."detecttime","events"."category","events"."description","events"."source_ip","events"."target_ip","events"."source_ip_aggr_ip4","events"."source_ip_aggr_ip6","events"."target_ip_aggr_ip4","events"."target_ip_aggr_ip6","events"."source_port","events"."target_port","events"."source_type","events"."target_type","events"."protocol","events"."node_name","events"."node_type","events"."cesnet_resolvedabuses","events"."cesnet_storagetime","events"."cesnet_eventclass","events"."cesnet_eventseverity","events"."cesnet_inspectionerrors","events_json"."event" FROM events INNER JOIN events_json ON events.id = events_json.id WHERE NOT ("cesnet_resolvedabuses" && ARRAY[\'abuse@cesnet.cz\',\'abuse@nic.cz\'])'
),
(
{
@@ -707,7 +707,7 @@ class TestMentatStorage(unittest.TestCase):
'groups': ['__EMPTY__']
}
},
- b'SELECT "id","detecttime","category","description","source_ip","target_ip","source_ip_aggr_ip4","source_ip_aggr_ip6","target_ip_aggr_ip4","target_ip_aggr_ip6","source_port","target_port","source_type","target_type","protocol","node_name","node_type","cesnet_resolvedabuses","cesnet_storagetime","cesnet_eventclass","cesnet_eventseverity","cesnet_inspectionerrors","event" FROM events WHERE "cesnet_resolvedabuses" = \'{}\''
+ b'SELECT "events"."id","events"."detecttime","events"."category","events"."description","events"."source_ip","events"."target_ip","events"."source_ip_aggr_ip4","events"."source_ip_aggr_ip6","events"."target_ip_aggr_ip4","events"."target_ip_aggr_ip6","events"."source_port","events"."target_port","events"."source_type","events"."target_type","events"."protocol","events"."node_name","events"."node_type","events"."cesnet_resolvedabuses","events"."cesnet_storagetime","events"."cesnet_eventclass","events"."cesnet_eventseverity","events"."cesnet_inspectionerrors","events_json"."event" FROM events INNER JOIN events_json ON events.id = events_json.id WHERE "cesnet_resolvedabuses" = \'{}\''
),
(
{
@@ -715,7 +715,7 @@ class TestMentatStorage(unittest.TestCase):
'groups': ['__ANY__']
}
},
- b'SELECT "id","detecttime","category","description","source_ip","target_ip","source_ip_aggr_ip4","source_ip_aggr_ip6","target_ip_aggr_ip4","target_ip_aggr_ip6","source_port","target_port","source_type","target_type","protocol","node_name","node_type","cesnet_resolvedabuses","cesnet_storagetime","cesnet_eventclass","cesnet_eventseverity","cesnet_inspectionerrors","event" FROM events WHERE "cesnet_resolvedabuses" != \'{}\''
+ b'SELECT "events"."id","events"."detecttime","events"."category","events"."description","events"."source_ip","events"."target_ip","events"."source_ip_aggr_ip4","events"."source_ip_aggr_ip6","events"."target_ip_aggr_ip4","events"."target_ip_aggr_ip6","events"."source_port","events"."target_port","events"."source_type","events"."target_type","events"."protocol","events"."node_name","events"."node_type","events"."cesnet_resolvedabuses","events"."cesnet_storagetime","events"."cesnet_eventclass","events"."cesnet_eventseverity","events"."cesnet_inspectionerrors","events_json"."event" FROM events INNER JOIN events_json ON events.id = events_json.id WHERE "cesnet_resolvedabuses" != \'{}\''
),
(
{
@@ -723,7 +723,7 @@ class TestMentatStorage(unittest.TestCase):
'description': 'Test description'
}
},
- b'SELECT "id","detecttime","category","description","source_ip","target_ip","source_ip_aggr_ip4","source_ip_aggr_ip6","target_ip_aggr_ip4","target_ip_aggr_ip6","source_port","target_port","source_type","target_type","protocol","node_name","node_type","cesnet_resolvedabuses","cesnet_storagetime","cesnet_eventclass","cesnet_eventseverity","cesnet_inspectionerrors","event" FROM events WHERE "description" = \'Test description\''
+ b'SELECT "events"."id","events"."detecttime","events"."category","events"."description","events"."source_ip","events"."target_ip","events"."source_ip_aggr_ip4","events"."source_ip_aggr_ip6","events"."target_ip_aggr_ip4","events"."target_ip_aggr_ip6","events"."source_port","events"."target_port","events"."source_type","events"."target_type","events"."protocol","events"."node_name","events"."node_type","events"."cesnet_resolvedabuses","events"."cesnet_storagetime","events"."cesnet_eventclass","events"."cesnet_eventseverity","events"."cesnet_inspectionerrors","events_json"."event" FROM events INNER JOIN events_json ON events.id = events_json.id WHERE "description" = \'Test description\''
),
(
{
@@ -731,7 +731,7 @@ class TestMentatStorage(unittest.TestCase):
'limit': 50
}
},
- b'SELECT "id","detecttime","category","description","source_ip","target_ip","source_ip_aggr_ip4","source_ip_aggr_ip6","target_ip_aggr_ip4","target_ip_aggr_ip6","source_port","target_port","source_type","target_type","protocol","node_name","node_type","cesnet_resolvedabuses","cesnet_storagetime","cesnet_eventclass","cesnet_eventseverity","cesnet_inspectionerrors","event" FROM events LIMIT 50'
+ b'SELECT "events"."id","events"."detecttime","events"."category","events"."description","events"."source_ip","events"."target_ip","events"."source_ip_aggr_ip4","events"."source_ip_aggr_ip6","events"."target_ip_aggr_ip4","events"."target_ip_aggr_ip6","events"."source_port","events"."target_port","events"."source_type","events"."target_type","events"."protocol","events"."node_name","events"."node_type","events"."cesnet_resolvedabuses","events"."cesnet_storagetime","events"."cesnet_eventclass","events"."cesnet_eventseverity","events"."cesnet_inspectionerrors","events_json"."event" FROM events INNER JOIN events_json ON events.id = events_json.id LIMIT 50'
),
(
{
@@ -740,7 +740,7 @@ class TestMentatStorage(unittest.TestCase):
'page': 11
}
},
- b'SELECT "id","detecttime","category","description","source_ip","target_ip","source_ip_aggr_ip4","source_ip_aggr_ip6","target_ip_aggr_ip4","target_ip_aggr_ip6","source_port","target_port","source_type","target_type","protocol","node_name","node_type","cesnet_resolvedabuses","cesnet_storagetime","cesnet_eventclass","cesnet_eventseverity","cesnet_inspectionerrors","event" FROM events LIMIT 50 OFFSET 500'
+ b'SELECT "events"."id","events"."detecttime","events"."category","events"."description","events"."source_ip","events"."target_ip","events"."source_ip_aggr_ip4","events"."source_ip_aggr_ip6","events"."target_ip_aggr_ip4","events"."target_ip_aggr_ip6","events"."source_port","events"."target_port","events"."source_type","events"."target_type","events"."protocol","events"."node_name","events"."node_type","events"."cesnet_resolvedabuses","events"."cesnet_storagetime","events"."cesnet_eventclass","events"."cesnet_eventseverity","events"."cesnet_inspectionerrors","events_json"."event" FROM events INNER JOIN events_json ON events.id = events_json.id LIMIT 50 OFFSET 500'
),
(
{
@@ -750,7 +750,7 @@ class TestMentatStorage(unittest.TestCase):
'page': 11
}
},
- b'SELECT "id","detecttime","category","description","source_ip","target_ip","source_ip_aggr_ip4","source_ip_aggr_ip6","target_ip_aggr_ip4","target_ip_aggr_ip6","source_port","target_port","source_type","target_type","protocol","node_name","node_type","cesnet_resolvedabuses","cesnet_storagetime","cesnet_eventclass","cesnet_eventseverity","cesnet_inspectionerrors","event" FROM events WHERE "cesnet_resolvedabuses" && ARRAY[\'abuse@cesnet.cz\',\'abuse@nic.cz\'] LIMIT 50 OFFSET 500'
+ b'SELECT "events"."id","events"."detecttime","events"."category","events"."description","events"."source_ip","events"."target_ip","events"."source_ip_aggr_ip4","events"."source_ip_aggr_ip6","events"."target_ip_aggr_ip4","events"."target_ip_aggr_ip6","events"."source_port","events"."target_port","events"."source_type","events"."target_type","events"."protocol","events"."node_name","events"."node_type","events"."cesnet_resolvedabuses","events"."cesnet_storagetime","events"."cesnet_eventclass","events"."cesnet_eventseverity","events"."cesnet_inspectionerrors","events_json"."event" FROM events INNER JOIN events_json ON events.id = events_json.id WHERE "cesnet_resolvedabuses" && ARRAY[\'abuse@cesnet.cz\',\'abuse@nic.cz\'] LIMIT 50 OFFSET 500'
),
(
{
@@ -758,7 +758,7 @@ class TestMentatStorage(unittest.TestCase):
'sortby': 'detecttime.desc'
}
},
- b'SELECT "id","detecttime","category","description","source_ip","target_ip","source_ip_aggr_ip4","source_ip_aggr_ip6","target_ip_aggr_ip4","target_ip_aggr_ip6","source_port","target_port","source_type","target_type","protocol","node_name","node_type","cesnet_resolvedabuses","cesnet_storagetime","cesnet_eventclass","cesnet_eventseverity","cesnet_inspectionerrors","event" FROM events ORDER BY "detecttime" DESC'
+ b'SELECT "events"."id","events"."detecttime","events"."category","events"."description","events"."source_ip","events"."target_ip","events"."source_ip_aggr_ip4","events"."source_ip_aggr_ip6","events"."target_ip_aggr_ip4","events"."target_ip_aggr_ip6","events"."source_port","events"."target_port","events"."source_type","events"."target_type","events"."protocol","events"."node_name","events"."node_type","events"."cesnet_resolvedabuses","events"."cesnet_storagetime","events"."cesnet_eventclass","events"."cesnet_eventseverity","events"."cesnet_inspectionerrors","events_json"."event" FROM events INNER JOIN events_json ON events.id = events_json.id ORDER BY "detecttime" DESC'
),
(
{
@@ -766,7 +766,7 @@ class TestMentatStorage(unittest.TestCase):
'sortby': 'detecttime.asc'
}
},
- b'SELECT "id","detecttime","category","description","source_ip","target_ip","source_ip_aggr_ip4","source_ip_aggr_ip6","target_ip_aggr_ip4","target_ip_aggr_ip6","source_port","target_port","source_type","target_type","protocol","node_name","node_type","cesnet_resolvedabuses","cesnet_storagetime","cesnet_eventclass","cesnet_eventseverity","cesnet_inspectionerrors","event" FROM events ORDER BY "detecttime" ASC'
+ b'SELECT "events"."id","events"."detecttime","events"."category","events"."description","events"."source_ip","events"."target_ip","events"."source_ip_aggr_ip4","events"."source_ip_aggr_ip6","events"."target_ip_aggr_ip4","events"."target_ip_aggr_ip6","events"."source_port","events"."target_port","events"."source_type","events"."target_type","events"."protocol","events"."node_name","events"."node_type","events"."cesnet_resolvedabuses","events"."cesnet_storagetime","events"."cesnet_eventclass","events"."cesnet_eventseverity","events"."cesnet_inspectionerrors","events_json"."event" FROM events INNER JOIN events_json ON events.id = events_json.id ORDER BY "detecttime" ASC'
),
(
{
@@ -774,7 +774,7 @@ class TestMentatStorage(unittest.TestCase):
'sortby': 'storagetime.desc'
}
},
- b'SELECT "id","detecttime","category","description","source_ip","target_ip","source_ip_aggr_ip4","source_ip_aggr_ip6","target_ip_aggr_ip4","target_ip_aggr_ip6","source_port","target_port","source_type","target_type","protocol","node_name","node_type","cesnet_resolvedabuses","cesnet_storagetime","cesnet_eventclass","cesnet_eventseverity","cesnet_inspectionerrors","event" FROM events ORDER BY "cesnet_storagetime" DESC'
+ b'SELECT "events"."id","events"."detecttime","events"."category","events"."description","events"."source_ip","events"."target_ip","events"."source_ip_aggr_ip4","events"."source_ip_aggr_ip6","events"."target_ip_aggr_ip4","events"."target_ip_aggr_ip6","events"."source_port","events"."target_port","events"."source_type","events"."target_type","events"."protocol","events"."node_name","events"."node_type","events"."cesnet_resolvedabuses","events"."cesnet_storagetime","events"."cesnet_eventclass","events"."cesnet_eventseverity","events"."cesnet_inspectionerrors","events_json"."event" FROM events INNER JOIN events_json ON events.id = events_json.id ORDER BY "cesnet_storagetime" DESC'
),
(
{
@@ -782,7 +782,7 @@ class TestMentatStorage(unittest.TestCase):
'sortby': 'storagetime.asc'
}
},
- b'SELECT "id","detecttime","category","description","source_ip","target_ip","source_ip_aggr_ip4","source_ip_aggr_ip6","target_ip_aggr_ip4","target_ip_aggr_ip6","source_port","target_port","source_type","target_type","protocol","node_name","node_type","cesnet_resolvedabuses","cesnet_storagetime","cesnet_eventclass","cesnet_eventseverity","cesnet_inspectionerrors","event" FROM events ORDER BY "cesnet_storagetime" ASC'
+ b'SELECT "events"."id","events"."detecttime","events"."category","events"."description","events"."source_ip","events"."target_ip","events"."source_ip_aggr_ip4","events"."source_ip_aggr_ip6","events"."target_ip_aggr_ip4","events"."target_ip_aggr_ip6","events"."source_port","events"."target_port","events"."source_type","events"."target_type","events"."protocol","events"."node_name","events"."node_type","events"."cesnet_resolvedabuses","events"."cesnet_storagetime","events"."cesnet_eventclass","events"."cesnet_eventseverity","events"."cesnet_inspectionerrors","events_json"."event" FROM events INNER JOIN events_json ON events.id = events_json.id ORDER BY "cesnet_storagetime" ASC'
),
]
@@ -1669,7 +1669,7 @@ class TestMentatStorage(unittest.TestCase):
{
'dt_from': datetime.datetime(2012, 11, 3, 10, 0, 7)
},
- event_factory = mentat.services.eventstorage.record_to_idea_ghost
+ qtype = mentat.services.eventstorage.QTYPE_SELECT_GHOST
)
if self.verbose:
pprint.pprint(ideas_from)
diff --git a/migrations-events/versions/0f31c168a714_split_event_column_from_events_to_.py b/migrations-events/versions/0f31c168a714_split_event_column_from_events_to_.py
new file mode 100644
index 000000000..5625a0f2f
--- /dev/null
+++ b/migrations-events/versions/0f31c168a714_split_event_column_from_events_to_.py
@@ -0,0 +1,46 @@
+"""Split event column from events to separate table
+
+Revision ID: 0f31c168a714
+Revises: 3fb6b209a5cd
+Create Date: 2019-11-06 10:24:06.856478
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '0f31c168a714' # pylint: disable=locally-disabled,invalid-name
+down_revision = '3fb6b209a5cd' # pylint: disable=locally-disabled,invalid-name
+branch_labels = None # pylint: disable=locally-disabled,invalid-name
+depends_on = None # pylint: disable=locally-disabled,invalid-name
+
+
+def upgrade(): # pylint: disable=locally-disabled,missing-docstring
+ # Create separate table for storing IDEA events serialized into JSON.
+ op.execute( # pylint: disable=locally-disabled,no-member
+ "CREATE TABLE IF NOT EXISTS events_json(id text PRIMARY KEY REFERENCES events(id) ON DELETE CASCADE, event bytea NOT NULL)"
+ )
+ # Copy data from 'events' table to 'events_json' table.
+ op.execute( # pylint: disable=locally-disabled,no-member
+ "INSERT INTO events_json(id, event) SELECT id, event FROM events"
+ )
+ # Drop now unnecessary column.
+ op.execute( # pylint: disable=locally-disabled,no-member
+ "ALTER TABLE events DROP COLUMN IF EXISTS event"
+ )
+
+
+def downgrade(): # pylint: disable=locally-disabled,missing-docstring
+ # Add back the 'event' column to 'events' table.
+ op.execute( # pylint: disable=locally-disabled,no-member
+ "ALTER TABLE events ADD COLUMN IF NOT EXISTS event bytea"
+ )
+ # Copy data from 'events_json' table to 'events' table.
+ op.execute( # pylint: disable=locally-disabled,no-member
+ "UPDATE events SET event = subquery.event FROM (SELECT id, event FROM events_json) AS subquery"
+ )
+ # Drop now unnecessary table.
+ op.execute( # pylint: disable=locally-disabled,no-member
+ "DROP TABLE IF EXISTS events_json"
+ )
--
GitLab