Skip to content
Snippets Groups Projects
Commit 754ca77f authored by Jan Mach's avatar Jan Mach
Browse files

Improved the mentat-cleanup.py module to cleanup module runlog directories and remove old runlogs. 

(Redmine issue: #3381)
parent 637fc5a9
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
conf/mentat-cleanup.py.conf
+ 14
− 3
View file @ 754ca77f
......@@ -41,19 +41,30 @@
#
"caches": [
{
"cache": "/var/mentat/cache",
"cachedir": "/var/mentat/cache",
"threshold_type": "w"
},
{
"cache": "/var/mentat/charts",
"cachedir": "/var/mentat/charts",
"threshold_type": "w"
},
{
"cache": "/var/mentat/reports/reporter-ng",
"cachedir": "/var/mentat/reports/reporter-ng",
"threshold_type": "y"
}
],
# List of maintained runlog folders.
# default: empty list
# type: list of dicts
#
"runlogs": [
{
"runlogdir": "/var/mentat/run",
"threshold_type": "4w"
}
],
#---------------------------------------------------------------------------
# Common script configurations
#---------------------------------------------------------------------------
......
lib/mentat/module/cleanup.py
+ 129
− 33
View file @ 754ca77f
......@@ -91,6 +91,7 @@ __credits__ = "Pavel Kácha <pavel.kacha@cesnet.cz>, Andrea Kropáčová <andrea
import os
import glob
import datetime
import json
......@@ -157,6 +158,7 @@ class MentatCleanupScript(mentat.script.fetcher.FetcherScript):
CONFIG_SIMULATE = 'simulate'
CONFIG_EVENTS = 'events'
CONFIG_CACHES = 'caches'
CONFIG_RUNLOGS = 'runlogs'
def __init__(self):
......@@ -233,6 +235,7 @@ class MentatCleanupScript(mentat.script.fetcher.FetcherScript):
(self.CONFIG_SIMULATE, False),
(self.CONFIG_EVENTS, []),
(self.CONFIG_CACHES, []),
(self.CONFIG_RUNLOGS, []),
) + cfgs
return super()._init_config(cfgs, **kwargs)
......@@ -258,13 +261,16 @@ class MentatCleanupScript(mentat.script.fetcher.FetcherScript):
"""
command = runlog.get(self.RLANKEY_COMMAND, None)
if command == 'cleanup':
analysis[command] = {'caches': {}, 'events': {}}
for cache in runlog[command]['caches']:
for counter in ('files_cnt','files_bytes','kept_cnt','kept_bytes','removed_cnt','removed_bytes', 'error_cnt'):
analysis[command]['caches'][counter] = analysis[command]['caches'].get(counter, 0) + cache.get(counter, 0)
for rule in runlog[command]['events']:
analysis[command] = {'caches': {}, 'events': {}, 'runlogs': {}}
for rule in runlog[command].get('events', []):
for counter in ('removed_cnt',):
analysis[command]['events'][counter] = analysis[command]['events'].get(counter, 0) + rule.get(counter, 0)
for cachedir in runlog[command].get('caches', []):
for counter in ('files_cnt','files_bytes','kept_cnt','kept_bytes','removed_cnt','removed_bytes', 'error_cnt'):
analysis[command]['caches'][counter] = analysis[command]['caches'].get(counter, 0) + cachedir.get(counter, 0)
for runlogdir in runlog[command].get('runlogs', []):
for counter in ('files_cnt','files_bytes','kept_cnt','kept_bytes','removed_cnt','removed_bytes', 'error_cnt'):
analysis[command]['runlogs'][counter] = analysis[command]['runlogs'].get(counter, 0) + runlogdir.get(counter, 0)
return analysis
def _sub_runlog_format_analysis(self, analysis):
......@@ -273,6 +279,27 @@ class MentatCleanupScript(mentat.script.fetcher.FetcherScript):
"""
if analysis.get(self.RLANKEY_COMMAND) == 'cleanup':
tablew = pydgets.widgets.TableWidget()
table_columns = [
{ 'label': 'Filter' },
{ 'label': 'Removed [#]', 'data_formating': '{:,d}', 'align': '>' },
{ 'label': 'Count [#]', 'data_formating': '{:,d}', 'align': '>' },
{ 'label': 'DB size [MB]', 'data_type': 'sizemb', 'align': '>' },
{ 'label': 'Storage size [MB]', 'data_type': 'sizemb', 'align': '>' },
]
table_data = []
for rule in analysis[self.RLANKEY_RUNLOG]['cleanup']['events']:
table_data.append(
[
rule['filter'],
int(rule['removed_cnt']),
int(rule['stats_post']['row_estimate']),
rule['stats_post']['table_bytes'],
rule['stats_post']['total_bytes'],
]
)
print("Event cleanup statistics:")
tablew.display(table_data, columns = table_columns)
table_columns = [
{ 'label': 'Cache' },
{ 'label': 'Removed [#]', 'data_formating': '{:,d}', 'align': '>' },
......@@ -282,49 +309,53 @@ class MentatCleanupScript(mentat.script.fetcher.FetcherScript):
{ 'label': 'Errors [#]', 'data_formating': '{:,d}', 'align': '>' },
]
table_data = []
for cache in analysis[self.RLANKEY_RUNLOG]['cleanup']['caches']:
for cachedir in analysis[self.RLANKEY_RUNLOG]['cleanup']['caches']:
table_data.append(
[
cache['cache'],
cache['removed_cnt'],
cache['removed_bytes'],
cache['kept_cnt'],
cache['kept_bytes'],
cache['error_cnt'],
cachedir['cache'],
cachedir['removed_cnt'],
cachedir['removed_bytes'],
cachedir['kept_cnt'],
cachedir['kept_bytes'],
cachedir['error_cnt'],
]
)
print("Cache cleanup statistics:")
tablew.display(table_data, columns = table_columns)
table_columns = [
{ 'label': 'Filter' },
{ 'label': 'Removed [#]', 'data_formating': '{:,d}', 'align': '>' },
{ 'label': 'Count [#]', 'data_formating': '{:,d}', 'align': '>' },
{ 'label': 'DB size [MB]', 'data_type': 'sizemb', 'align': '>' },
{ 'label': 'Storage size [MB]', 'data_type': 'sizemb', 'align': '>' },
{ 'label': 'Runlogdir' },
{ 'label': 'Removed [#]', 'data_formating': '{:,d}', 'align': '>' },
{ 'label': 'Removed [MB]', 'data_type': 'sizemb', 'align': '>' },
{ 'label': 'Files [#]', 'data_formating': '{:,d}', 'align': '>' },
{ 'label': 'Size [MB]', 'data_type': 'sizemb', 'align': '>' },
{ 'label': 'Errors [#]', 'data_formating': '{:,d}', 'align': '>' },
]
table_data = []
for rule in analysis[self.RLANKEY_RUNLOG]['cleanup']['events']:
for runlogdir in analysis[self.RLANKEY_RUNLOG]['cleanup']['runlogs']:
table_data.append(
[
rule['filter'],
int(rule['removed_cnt']),
int(rule['stats_post']['row_estimate']),
rule['stats_post']['table_bytes'],
rule['stats_post']['total_bytes'],
runlogdir['runlogdir'],
runlogdir['removed_cnt'],
runlogdir['removed_bytes'],
runlogdir['kept_cnt'],
runlogdir['kept_bytes'],
runlogdir['error_cnt'],
]
)
print("Event cleanup statistics:")
print("Runlog cleanup statistics:")
tablew.display(table_data, columns = table_columns)
def _sub_runlogs_format_evaluation(self, evaluation):
"""
"""
table_columns = [
{ 'label': 'Date' },
{ 'label': 'FS removed [#]', 'data_formating': '{:,d}', 'align': '>' },
{ 'label': 'DB removed [#]', 'data_formating': '{:,d}', 'align': '>' },
{ 'label': 'FS removed [#]', 'data_formating': '{:,d}', 'align': '>' },
{ 'label': 'RL removed [#]', 'data_formating': '{:,d}', 'align': '>' },
]
table_data = []
for anl in evaluation[self.RLEVKEY_ANALYSES]:
......@@ -333,8 +364,9 @@ class MentatCleanupScript(mentat.script.fetcher.FetcherScript):
table_data.append(
[
anl['label'],
anl['cleanup']['caches']['removed_cnt'],
anl['cleanup']['events']['removed_cnt'],
anl['cleanup']['events'].get('removed_cnt', 0),
anl['cleanup']['caches'].get('removed_cnt', 0),
anl['cleanup']['runlogs'].get('removed_cnt', 0),
]
)
else:
......@@ -343,6 +375,7 @@ class MentatCleanupScript(mentat.script.fetcher.FetcherScript):
anl['label'],
0,
0,
0,
]
)
print("Result overview for 'cleanup' command:")
......@@ -388,10 +421,16 @@ class MentatCleanupScript(mentat.script.fetcher.FetcherScript):
# Perform cleanup of selected folder caches.
result['caches'] = []
for cache in self.c(self.CONFIG_CACHES):
res = self._cleanup_cache(dt_current, **cache)
for cachedir in self.c(self.CONFIG_CACHES):
res = self._cleanup_cachedir(dt_current, **cachedir)
result['caches'].append(res)
# Perform cleanup of selected folder caches.
result['runlogs'] = []
for runlogdir in self.c(self.CONFIG_RUNLOGS):
res = self._cleanup_runlogdir(dt_current, **runlogdir)
result['runlogs'].append(res)
# Measure disk usage after cleanup.
result['fsstats_post'] = self._fsstats(self.c(self.CONFIG_DB_PATH))
......@@ -461,7 +500,7 @@ class MentatCleanupScript(mentat.script.fetcher.FetcherScript):
return result
def _cleanup_cache(self, dt_current, cache, threshold_type):
def _cleanup_cachedir(self, dt_current, cachedir, threshold_type):
"""
Cleanup given filesystem cache.
"""
......@@ -470,7 +509,7 @@ class MentatCleanupScript(mentat.script.fetcher.FetcherScript):
dt_threshold = dt_current - threshold['d']
result = {
'cache': cache,
'cache': cachedir,
'threshold': threshold_type,
'threshold_int': dt_threshold.timestamp(),
'threshold_str': dt_threshold.isoformat(),
......@@ -491,8 +530,8 @@ class MentatCleanupScript(mentat.script.fetcher.FetcherScript):
result['threshold'],
result['threshold_str']
)
for fln in os.listdir(cache):
flp = os.path.join(cache, fln)
for fln in os.listdir(cachedir):
flp = os.path.join(cachedir, fln)
if os.path.isfile(flp):
fst = os.stat(flp)
result['files_cnt'] += 1
......@@ -518,3 +557,60 @@ class MentatCleanupScript(mentat.script.fetcher.FetcherScript):
result['error_cnt']
))
return result
def _cleanup_runlogdir(self, dt_current, runlogdir, threshold_type):
"""
Cleanup given filesystem cache.
"""
# Determine the time threshold for cleanup operation.
threshold = THRESHOLDS[threshold_type]
dt_threshold = dt_current - threshold['d']
result = {
'runlogdir': runlogdir,
'threshold': threshold_type,
'threshold_int': dt_threshold.timestamp(),
'threshold_str': dt_threshold.isoformat(),
'threshold_lbl': threshold['l'],
'files_cnt': 0,
'files_bytes': 0,
'kept_cnt': 0,
'kept_bytes': 0,
'removed_cnt': 0,
'removed_bytes': 0,
'error_cnt': 0,
'errors': [],
}
self.logger.info(
"Runlog '%s' cleanup started with threshold '%s': '%s'",
result['runlogdir'],
result['threshold'],
result['threshold_str']
)
for flp in glob.iglob('{}/**/*.runlog'.format(runlogdir)):
if os.path.isfile(flp):
fst = os.stat(flp)
result['files_cnt'] += 1
result['files_bytes'] += fst.st_size
if fst.st_mtime < result['threshold_int']:
try:
if not self.c('simulate'):
os.remove(flp)
result['removed_cnt'] += 1
result['removed_bytes'] += fst.st_size
except:
self.error("Unable to remove runlog file '{}'".format(flp))
result['error_cnt'] += 1
result['errors'].append(flp)
else:
result['kept_cnt'] += 1
result['kept_bytes'] += fst.st_size
self.logger.info("Runlog '{}' cleanup done, removed: {:,d} | kept: {:,d} | errors: {:,d}".format(
result['runlogdir'],
result['removed_cnt'],
result['kept_cnt'],
result['error_cnt']
))
return result
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment