diff --git a/bin/mentat-dbmngr.py b/bin/mentat-dbmngr.py
index 41c0e02fdcc4cd5de3582d0567ac61760c5a2620..737d5562524fef0cc932730cb3fa77e9bffd65d7 100755
--- a/bin/mentat-dbmngr.py
+++ b/bin/mentat-dbmngr.py
@@ -22,16 +22,14 @@ from email.mime.text import MIMEText
#
import pyzenkit.jsonconf
import pyzenkit.zenscript
+import mentat.const
import mentat.storage
class MentatDbmngrScript(pyzenkit.zenscript.ZenScript):
"""
- Script providing Mentat system database management functions and features
+ Script providing Mentat system database management functions and features.
"""
- def __init__(self, **kwargs):
- super().__init__(**kwargs)
-
def get_default_command(self):
"""
Return the name of a default script operation.
@@ -49,8 +47,8 @@ class MentatDbmngrScript(pyzenkit.zenscript.ZenScript):
This command will determine the current status of all databases and its
collections.
"""
- db_config = self.c('_core_database')
- db_schema = db_config['schema']
+ db_config = self.c(mentat.const.CKEY_CORE_DATABASE)
+ db_schema = db_config[mentat.const.CKEY_CORE_DATABASE_SCHEMA]
s = mentat.storage.Storage()
for db_n in s.database_names():
@@ -58,13 +56,13 @@ class MentatDbmngrScript(pyzenkit.zenscript.ZenScript):
continue
self.logger.info("Inspecting database '{}'".format(db_n))
if not db_n in db_schema:
- self.logger.info("- database '{}' was not configured".format(db_n))
+ self.logger.warning("Database '{}' exists, but is not configured".format(db_n))
else:
db_h = s.database(db_n)
db_c = db_schema[db_n]
for col_n in db_h.collection_names():
if not col_n in db_c['collections']:
- self.logger.info("- collection '{}.{}' was not configured".format(db_n, col_n))
+ self.logger.warning("Collection '{}.{}' exists, but is not configured".format(db_n, col_n))
else:
col_h = db_h.collection(col_n)
col_c = db_schema[db_n]['collections'][col_n]
@@ -72,25 +70,13 @@ class MentatDbmngrScript(pyzenkit.zenscript.ZenScript):
index_list_config = [i['name'] for i in col_c['indexes']]
for idx_n in index_list_current:
if not idx_n in index_list_config:
- self.logger.info("- index '{}' in collection '{}.{}' was not configured".format(idx_n, db_n, col_n))
+ self.logger.info("Index '{}.{}':'{}' exists, but is not configured".format(db_n, col_n, idx_n))
for idx_n in index_list_config:
if not idx_n in index_list_current:
- self.logger.info("- index '{}' in collection '{}.{}' is missing".format(idx_n, db_n, col_n))
-
- #pprint.pprint(s.database_names())
- #pprint.pprint(s.database_walk())
- #pprint.pprint(s.storage_info())
- #d = s.database('mentat')
- #pprint.pprint(d.collection_names())
- #pprint.pprint(d.collection_stats('alerts'))
- #c1 = d.collection('alerts')
- #pprint.pprint(c1.count())
- #c2 = s.collection('mentat', 'alerts')
- #pprint.pprint(c2.count())
- #i2 = c2.list_indexes()
- #for idx in i2:
- # pprint.pprint(idx)
+ self.logger.info("Index '{}.{}':'{}' is missing".format(db_n, col_n, idx_n))
+
s.close()
+ return self.RESULT_SUCCESS
def cbk_command_init(self):
"""
@@ -98,23 +84,29 @@ class MentatDbmngrScript(pyzenkit.zenscript.ZenScript):
This command will perform all necessary database initializations.
"""
- dbconfig = self.c('_core_database')
- storage = mentat.storage.Storage()
- for db_n in sorted(dbconfig['schema'].keys()):
- db_c = dbconfig['schema'][db_n]
- db_h = storage.database(db_n)
+ db_config = self.c(mentat.const.CKEY_CORE_DATABASE)
+ db_schema = db_config[mentat.const.CKEY_CORE_DATABASE_SCHEMA]
+ s = mentat.storage.Storage()
+
+ for db_n in sorted(db_schema.keys()):
+ db_c = db_schema[db_n]
+ db_h = s.database(db_n)
+
self.logger.info("Initializing database '{}'".format(db_n))
for col_n in sorted(db_c['collections'].keys()):
col_c = db_c['collections'][col_n]
col_h = db_h.collection(col_n)
index_map = col_h.get_current_index_map()
+
self.logger.info("Initializing collection '{}.{}'".format(db_n, col_n))
for idx in col_c['indexes']:
if not idx['name'] in index_map:
- self.logger.info("Creating index '{}' for '{}.{}'".format(idx['name'], db_n, col_n))
+ self.logger.info("Creating index '{}.{}':'{}'".format(db_n, col_n, idx['name']))
+ col_h.index_create(**idx)
else:
- self.logger.info("Index '{}' for '{}.{}' already exists".format(idx['name'], db_n, col_n))
+ self.logger.info("Index '{}.{}':'{}' already exists".format(db_n, col_n, idx['name']))
+ s.close()
return self.RESULT_SUCCESS
def cbk_command_watchdog(self):
@@ -123,7 +115,7 @@ class MentatDbmngrScript(pyzenkit.zenscript.ZenScript):
This command will attempt to profile various collection data.
"""
- dbconfig = self.c('_core_database')
+ dbconfig = self.c(mentat.const.CKEY_CORE_DATABASE)
storage = mentat.storage.Storage()
database = storage.database('mentat')
collection = database.collection('alerts')
@@ -149,7 +141,7 @@ class MentatDbmngrScript(pyzenkit.zenscript.ZenScript):
This command will attempt to profile various collection data.
"""
- dbconfig = self.c('_core_database')
+ dbconfig = self.c(mentat.const.CKEY_CORE_DATABASE)
storage = mentat.storage.Storage()
database = storage.database('mentat')
collection = database.collection('alerts')
@@ -243,12 +235,21 @@ if __name__ == "__main__":
Execute the MentatDbmngrScript script.
"""
script = MentatDbmngrScript(
+
+ description = 'mentat-dbmngr.py - Mentat system database management script',
+
+ #
+ # Configure required daemon paths
+ #
path_bin = '/usr/local/bin',
path_cfg = '/etc/mentat',
path_log = '/var/mentat/log',
path_run = '/var/mentat/run',
path_tmp = '/tmp',
+ #
+ # Override default configurations
+ #
default_config_dir = '/etc/mentat/core',
)
script.run()
diff --git a/conf/core/database.conf b/conf/core/database.conf
deleted file mode 100644
index 84554cc7db9c373687bc8c545fd91ec67e58cf8b..0000000000000000000000000000000000000000
--- a/conf/core/database.conf
+++ /dev/null
@@ -1,34 +0,0 @@
-{
- "_core_database": {
- "config": {
- "db_main": "mentat",
- "col_main_alerts": "alerts",
- "db_stats": "mentat_stats"
- },
- "schema": {
- "mentat": {
- "collections": {
- "alerts": {
- "indexes": [
- { "name": "ts", "index": ["ts","ascending"] },
- { "name": "node_name", "index": ["Node.Name","ascending"] },
- { "name": "node_sw", "index": ["Node.SW","ascending"] },
- { "name": "category", "index": ["Category","ascending"] },
- { "name": "description", "index": ["Description","ascending"] },
- { "name": "detector", "index": [["Node.Name","ascending"], ["Node.SW","ascending"]] },
- { "name": "detecttime", "index": ["DetectTime","descending"] },
- { "name": "source_ip4", "index": ["Source.IP4.ip","ascending"] },
- { "name": "source_ip4_min", "index": ["Source.IP4.min","ascending"] },
- { "name": "source_ip4_max", "index": ["Source.IP4.max","ascending"] },
- { "name": "target_ip4", "index": ["Target.IP4.ip","ascending"] },
- { "name": "target_ip4_min", "index": ["Target.IP4.min","ascending"] },
- { "name": "target_ip4_max", "index": ["Target.IP4.max","ascending"] },
- { "name": "resolved_abuses", "index": ["_CESNET.ResolvedAbuses","ascending"] },
- { "name": "storage_time", "index": ["_CESNET.StorageTime","descending"] }
- ]
- }
- }
- }
- }
- }
-}
diff --git a/conf/core/database.json.conf b/conf/core/database.json.conf
new file mode 100644
index 0000000000000000000000000000000000000000..593b1ec17c337b43e791d6c076cb33e4df26908d
--- /dev/null
+++ b/conf/core/database.json.conf
@@ -0,0 +1,174 @@
+{
+ "__core__database": {
+
+ "config": {
+ "db": "mentat",
+ "db_test": "mentat_test",
+ "db_stats": "mentat_stats",
+
+ "col_alerts": "alerts",
+
+ "col_stats_alerts": "statistics"
+ },
+
+ "schema": {
+ "mentat": {
+ "collections": {
+ "alerts": {
+ "indexes": [
+ {
+ "name": "_id_",
+ "index": ["_id","ascending"],
+ "background": false
+ },
+ {
+ "name": "ts_1",
+ "index": ["ts","ascending"],
+ "background": true,
+ "sparse": false
+ },
+ {
+ "name": "DetectTime_-1_Node.Name_1",
+ "index": [["DetectTime","descending"],["Node.Name","ascending"]],
+ "background": true,
+ "sparse": false
+ },
+ {
+ "name": "DetectTime_-1_Category_1",
+ "index": [["DetectTime","descending"],["Category","ascending"]],
+ "background": true,
+ "sparse": false
+ },
+ {
+ "name": "Source.IP4.ip_1",
+ "index": ["Source.IP4.ip","ascending"],
+ "background": true,
+ "sparse": true
+ },
+ {
+ "name": "Target.IP4.ip_1",
+ "index": ["Target.IP4.ip","ascending"],
+ "background": true,
+ "sparse": true
+ },
+ {
+ "name": "Source.IP4.min_1_Source.IP4.max_1",
+ "index": [["Source.IP4.min","ascending"],["Source.IP4.max","ascending"]],
+ "background": true,
+ "sparse": true
+ },
+ {
+ "name": "Target.IP4.min_1_Target.IP4.max_1",
+ "index": [["Target.IP4.min","ascending"],["Target.IP4.max","ascending"]],
+ "background": true,
+ "sparse": true
+ },
+ {
+ "name": "DetectTime_-1_Target.IP4.min_1_Target.IP4.max_1",
+ "index": [["DetectTime","descending"],["Target.IP4.min","ascending"],["Target.IP4.max","ascending"]],
+ "background": true,
+ "sparse": false
+ },
+ {
+ "name": "_CESNET.EventClass_1",
+ "index": ["_CESNET.EventClass","ascending"],
+ "background": true,
+ "sparse": false
+ },
+ {
+ "name": "ts_-1__CESNET.ResolvedAbuses_1",
+ "index": [["ts","descending"],["_CESNET.ResolvedAbuses","ascending"]],
+ "background": true,
+ "sparse": false,
+ "partialFilterExpression" : {
+ "_CESNET.ResolvedAbuses" : {
+ "$exists" : true
+ }
+ }
+ },
+ {
+ "name": "_CESNET.StorageTime_-1",
+ "index": ["_CESNET.StorageTime","descending"],
+ "background": true,
+ "sparse": false
+ }
+ ]
+ }
+ }
+ },
+
+ "mentat_stats": {
+ "collections": {
+ "statistics": {
+ "indexes": [
+ {
+ "name": "_id_",
+ "index": ["_id","ascending"],
+ "background": false
+ },
+ {
+ "name": "ts",
+ "index": ["ts","descending"],
+ "background": true,
+ "sparse": false
+ },
+ {
+ "name": "ts_from",
+ "index": ["ts_from","descending"],
+ "background": true,
+ "sparse": false
+ },
+ {
+ "name": "ts_to",
+ "index": ["ts_to","descending"],
+ "background": true,
+ "sparse": false
+ }
+ ]
+ }
+ }
+ }
+ }
+ }
+}
+
+#
+# Following indices were defined previously, but they are not used anymore.
+# Perhaps they will be handy again.
+#
+#{
+# "name": "node_name",
+# "index": ["Node.Name","ascending"],
+# "background": true,
+# "sparse": false
+#}
+#{
+# "name": "node_sw",
+# "index": ["Node.SW","ascending"],
+# "background": true,
+# "sparse": false
+#}
+#{
+# "name": "category",
+# "index": ["Category","ascending"],
+# "background": true,
+# "sparse": false
+#}
+#{
+# "name": "description",
+# "index": ["Description","ascending"],
+# "background": true,
+# "sparse": false
+#}
+#{
+# "name": "detector",
+# "index": [["Node.Name","ascending"], ["Node.SW","ascending"]],
+# "background": true,
+# "sparse": false
+#}
+#{
+# "name": "detecttime",
+# "index": ["DetectTime","descending"],
+# "background": true,
+# "sparse": false
+#}
diff --git a/deploy/ctrl/conffiles b/deploy/ctrl/conffiles
index c7a29327d1e52fbcbee8c14907a49fba147559df..951fd42bb2a7a87976dcb9584f5880798acfd1d3 100755
--- a/deploy/ctrl/conffiles
+++ b/deploy/ctrl/conffiles
@@ -9,4 +9,4 @@
/etc/mentat/mentat-ideagen.py.conf
/etc/mentat/mentat-inspector.py.conf
/etc/mentat/mentat-sampler.py.conf
-/etc/mentat/core/database.conf
+/etc/mentat/core/database.json.conf
diff --git a/lib/mentat/const.py b/lib/mentat/const.py
new file mode 100644
index 0000000000000000000000000000000000000000..1763e76a55e95ed0c9845fca919c1e9be54dec51
--- /dev/null
+++ b/lib/mentat/const.py
@@ -0,0 +1,10 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+#-------------------------------------------------------------------------------
+# Copyright (C) since 2011 CESNET, z.s.p.o
+# Use of this source is governed by the MIT license, see LICENSE file.
+#-------------------------------------------------------------------------------
+
+CKEY_CORE_DATABASE = '__core__database'
+CKEY_CORE_DATABASE_SCHEMA = 'schema'
+CKEY_CORE_DATABASE_CONFIG = 'config'