Skip to content
Snippets Groups Projects
Commit 4707b488 authored by Jan Mach's avatar Jan Mach
Browse files

Implemented basic index building mechanism into mentat-dbmngr.py script.

parent da82ff8c
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
bin/mentat-dbmngr.py
+ 34
− 33
View file @ 4707b488
......@@ -22,16 +22,14 @@ from email.mime.text import MIMEText
#
import pyzenkit.jsonconf
import pyzenkit.zenscript
import mentat.const
import mentat.storage
class MentatDbmngrScript(pyzenkit.zenscript.ZenScript):
"""
Script providing Mentat system database management functions and features
Script providing Mentat system database management functions and features.
"""
def __init__(self, **kwargs):
super().__init__(**kwargs)
def get_default_command(self):
"""
Return the name of a default script operation.
......@@ -49,8 +47,8 @@ class MentatDbmngrScript(pyzenkit.zenscript.ZenScript):
This command will determine the current status of all databases and its
collections.
"""
db_config = self.c('_core_database')
db_schema = db_config['schema']
db_config = self.c(mentat.const.CKEY_CORE_DATABASE)
db_schema = db_config[mentat.const.CKEY_CORE_DATABASE_SCHEMA]
s = mentat.storage.Storage()
for db_n in s.database_names():
......@@ -58,13 +56,13 @@ class MentatDbmngrScript(pyzenkit.zenscript.ZenScript):
continue
self.logger.info("Inspecting database '{}'".format(db_n))
if not db_n in db_schema:
self.logger.info("- database '{}' was not configured".format(db_n))
self.logger.warning("Database '{}' exists, but is not configured".format(db_n))
else:
db_h = s.database(db_n)
db_c = db_schema[db_n]
for col_n in db_h.collection_names():
if not col_n in db_c['collections']:
self.logger.info("- collection '{}.{}' was not configured".format(db_n, col_n))
self.logger.warning("Collection '{}.{}' exists, but is not configured".format(db_n, col_n))
else:
col_h = db_h.collection(col_n)
col_c = db_schema[db_n]['collections'][col_n]
......@@ -72,25 +70,13 @@ class MentatDbmngrScript(pyzenkit.zenscript.ZenScript):
index_list_config = [i['name'] for i in col_c['indexes']]
for idx_n in index_list_current:
if not idx_n in index_list_config:
self.logger.info("- index '{}' in collection '{}.{}' was not configured".format(idx_n, db_n, col_n))
self.logger.info("Index '{}.{}':'{}' exists, but is not configured".format(db_n, col_n, idx_n))
for idx_n in index_list_config:
if not idx_n in index_list_current:
self.logger.info("- index '{}' in collection '{}.{}' is missing".format(idx_n, db_n, col_n))
#pprint.pprint(s.database_names())
#pprint.pprint(s.database_walk())
#pprint.pprint(s.storage_info())
#d = s.database('mentat')
#pprint.pprint(d.collection_names())
#pprint.pprint(d.collection_stats('alerts'))
#c1 = d.collection('alerts')
#pprint.pprint(c1.count())
#c2 = s.collection('mentat', 'alerts')
#pprint.pprint(c2.count())
#i2 = c2.list_indexes()
#for idx in i2:
# pprint.pprint(idx)
self.logger.info("Index '{}.{}':'{}' is missing".format(db_n, col_n, idx_n))
s.close()
return self.RESULT_SUCCESS
def cbk_command_init(self):
"""
......@@ -98,23 +84,29 @@ class MentatDbmngrScript(pyzenkit.zenscript.ZenScript):
This command will perform all necessary database initializations.
"""
dbconfig = self.c('_core_database')
storage = mentat.storage.Storage()
for db_n in sorted(dbconfig['schema'].keys()):
db_c = dbconfig['schema'][db_n]
db_h = storage.database(db_n)
db_config = self.c(mentat.const.CKEY_CORE_DATABASE)
db_schema = db_config[mentat.const.CKEY_CORE_DATABASE_SCHEMA]
s = mentat.storage.Storage()
for db_n in sorted(db_schema.keys()):
db_c = db_schema[db_n]
db_h = s.database(db_n)
self.logger.info("Initializing database '{}'".format(db_n))
for col_n in sorted(db_c['collections'].keys()):
col_c = db_c['collections'][col_n]
col_h = db_h.collection(col_n)
index_map = col_h.get_current_index_map()
self.logger.info("Initializing collection '{}.{}'".format(db_n, col_n))
for idx in col_c['indexes']:
if not idx['name'] in index_map:
self.logger.info("Creating index '{}' for '{}.{}'".format(idx['name'], db_n, col_n))
self.logger.info("Creating index '{}.{}':'{}'".format(db_n, col_n, idx['name']))
col_h.index_create(**idx)
else:
self.logger.info("Index '{}' for '{}.{}' already exists".format(idx['name'], db_n, col_n))
self.logger.info("Index '{}.{}':'{}' already exists".format(db_n, col_n, idx['name']))
s.close()
return self.RESULT_SUCCESS
def cbk_command_watchdog(self):
......@@ -123,7 +115,7 @@ class MentatDbmngrScript(pyzenkit.zenscript.ZenScript):
This command will attempt to profile various collection data.
"""
dbconfig = self.c('_core_database')
dbconfig = self.c(mentat.const.CKEY_CORE_DATABASE)
storage = mentat.storage.Storage()
database = storage.database('mentat')
collection = database.collection('alerts')
......@@ -149,7 +141,7 @@ class MentatDbmngrScript(pyzenkit.zenscript.ZenScript):
This command will attempt to profile various collection data.
"""
dbconfig = self.c('_core_database')
dbconfig = self.c(mentat.const.CKEY_CORE_DATABASE)
storage = mentat.storage.Storage()
database = storage.database('mentat')
collection = database.collection('alerts')
......@@ -243,12 +235,21 @@ if __name__ == "__main__":
Execute the MentatDbmngrScript script.
"""
script = MentatDbmngrScript(
description = 'mentat-dbmngr.py - Mentat system database management script',
#
# Configure required daemon paths
#
path_bin = '/usr/local/bin',
path_cfg = '/etc/mentat',
path_log = '/var/mentat/log',
path_run = '/var/mentat/run',
path_tmp = '/tmp',
#
# Override default configurations
#
default_config_dir = '/etc/mentat/core',
)
script.run()
conf/core/database.conf deleted 100644 → 0
+ 0
− 34
View file @ da82ff8c
{
"_core_database": {
"config": {
"db_main": "mentat",
"col_main_alerts": "alerts",
"db_stats": "mentat_stats"
},
"schema": {
"mentat": {
"collections": {
"alerts": {
"indexes": [
{ "name": "ts", "index": ["ts","ascending"] },
{ "name": "node_name", "index": ["Node.Name","ascending"] },
{ "name": "node_sw", "index": ["Node.SW","ascending"] },
{ "name": "category", "index": ["Category","ascending"] },
{ "name": "description", "index": ["Description","ascending"] },
{ "name": "detector", "index": [["Node.Name","ascending"], ["Node.SW","ascending"]] },
{ "name": "detecttime", "index": ["DetectTime","descending"] },
{ "name": "source_ip4", "index": ["Source.IP4.ip","ascending"] },
{ "name": "source_ip4_min", "index": ["Source.IP4.min","ascending"] },
{ "name": "source_ip4_max", "index": ["Source.IP4.max","ascending"] },
{ "name": "target_ip4", "index": ["Target.IP4.ip","ascending"] },
{ "name": "target_ip4_min", "index": ["Target.IP4.min","ascending"] },
{ "name": "target_ip4_max", "index": ["Target.IP4.max","ascending"] },
{ "name": "resolved_abuses", "index": ["_CESNET.ResolvedAbuses","ascending"] },
{ "name": "storage_time", "index": ["_CESNET.StorageTime","descending"] }
]
}
}
}
}
}
}
conf/core/database.json.conf 0 → 100644
+ 174
− 0
View file @ 4707b488
{
"__core__database": {
"config": {
"db": "mentat",
"db_test": "mentat_test",
"db_stats": "mentat_stats",
"col_alerts": "alerts",
"col_stats_alerts": "statistics"
},
"schema": {
"mentat": {
"collections": {
"alerts": {
"indexes": [
{
"name": "_id_",
"index": ["_id","ascending"],
"background": false
},
{
"name": "ts_1",
"index": ["ts","ascending"],
"background": true,
"sparse": false
},
{
"name": "DetectTime_-1_Node.Name_1",
"index": [["DetectTime","descending"],["Node.Name","ascending"]],
"background": true,
"sparse": false
},
{
"name": "DetectTime_-1_Category_1",
"index": [["DetectTime","descending"],["Category","ascending"]],
"background": true,
"sparse": false
},
{
"name": "Source.IP4.ip_1",
"index": ["Source.IP4.ip","ascending"],
"background": true,
"sparse": true
},
{
"name": "Target.IP4.ip_1",
"index": ["Target.IP4.ip","ascending"],
"background": true,
"sparse": true
},
{
"name": "Source.IP4.min_1_Source.IP4.max_1",
"index": [["Source.IP4.min","ascending"],["Source.IP4.max","ascending"]],
"background": true,
"sparse": true
},
{
"name": "Target.IP4.min_1_Target.IP4.max_1",
"index": [["Target.IP4.min","ascending"],["Target.IP4.max","ascending"]],
"background": true,
"sparse": true
},
{
"name": "DetectTime_-1_Target.IP4.min_1_Target.IP4.max_1",
"index": [["DetectTime","descending"],["Target.IP4.min","ascending"],["Target.IP4.max","ascending"]],
"background": true,
"sparse": false
},
{
"name": "_CESNET.EventClass_1",
"index": ["_CESNET.EventClass","ascending"],
"background": true,
"sparse": false
},
{
"name": "ts_-1__CESNET.ResolvedAbuses_1",
"index": [["ts","descending"],["_CESNET.ResolvedAbuses","ascending"]],
"background": true,
"sparse": false,
"partialFilterExpression" : {
"_CESNET.ResolvedAbuses" : {
"$exists" : true
}
}
},
{
"name": "_CESNET.StorageTime_-1",
"index": ["_CESNET.StorageTime","descending"],
"background": true,
"sparse": false
}
]
}
}
},
"mentat_stats": {
"collections": {
"statistics": {
"indexes": [
{
"name": "_id_",
"index": ["_id","ascending"],
"background": false
},
{
"name": "ts",
"index": ["ts","descending"],
"background": true,
"sparse": false
},
{
"name": "ts_from",
"index": ["ts_from","descending"],
"background": true,
"sparse": false
},
{
"name": "ts_to",
"index": ["ts_to","descending"],
"background": true,
"sparse": false
}
]
}
}
}
}
}
}
#
# Following indices were defined previously, but they are not used anymore.
# Perhaps they will be handy again.
#
#{
# "name": "node_name",
# "index": ["Node.Name","ascending"],
# "background": true,
# "sparse": false
#}
#{
# "name": "node_sw",
# "index": ["Node.SW","ascending"],
# "background": true,
# "sparse": false
#}
#{
# "name": "category",
# "index": ["Category","ascending"],
# "background": true,
# "sparse": false
#}
#{
# "name": "description",
# "index": ["Description","ascending"],
# "background": true,
# "sparse": false
#}
#{
# "name": "detector",
# "index": [["Node.Name","ascending"], ["Node.SW","ascending"]],
# "background": true,
# "sparse": false
#}
#{
# "name": "detecttime",
# "index": ["DetectTime","descending"],
# "background": true,
# "sparse": false
#}
deploy/ctrl/conffiles
+ 1
− 1
View file @ 4707b488
......@@ -9,4 +9,4 @@
/etc/mentat/mentat-ideagen.py.conf
/etc/mentat/mentat-inspector.py.conf
/etc/mentat/mentat-sampler.py.conf
/etc/mentat/core/database.conf
/etc/mentat/core/database.json.conf
lib/mentat/const.py 0 → 100644
+ 10
− 0
View file @ 4707b488
#!/usr/bin/python3
# -*- coding: utf-8 -*-
#-------------------------------------------------------------------------------
# Copyright (C) since 2011 CESNET, z.s.p.o
# Use of this source is governed by the MIT license, see LICENSE file.
#-------------------------------------------------------------------------------
CKEY_CORE_DATABASE = '__core__database'
CKEY_CORE_DATABASE_SCHEMA = 'schema'
CKEY_CORE_DATABASE_CONFIG = 'config'
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment