diff --git a/lib/hawat/blueprints/events/__init__.py b/lib/hawat/blueprints/events/__init__.py
index 7c60d29ab9d5384e2d00befb08929e0ece92b6f7..ceb57f672c5cc5a5f681b4385aac042614d88f35 100644
--- a/lib/hawat/blueprints/events/__init__.py
+++ b/lib/hawat/blueprints/events/__init__.py
@@ -18,6 +18,7 @@ __author__ = "Jan Mach <jan.mach@cesnet.cz>"
__credits__ = "Pavel Kácha <pavel.kacha@cesnet.cz>, Andrea Kropáčová <andrea.kropacova@cesnet.cz>"
import datetime
+import pytz
import flask
from flask_babel import lazy_gettext
@@ -349,7 +350,8 @@ class AbstractDashboardView(SQLAlchemyMixin, BaseSearchView): # pylint: disable
dt_from=dt_from,
dt_to=dt_to,
max_count=flask.current_app.config['HAWAT_CHART_TIMELINE_MAXSTEPS'],
- min_step=300
+ min_step=300,
+ timezone=pytz.timezone(flask.session.get('timezone', 'UTC'))
)
)
diff --git a/lib/hawat/blueprints/hosts/__init__.py b/lib/hawat/blueprints/hosts/__init__.py
index a25b9346f7338be9ace02c3b699f0454882612a4..0f1d65e913fdcf78db0deade9ff86cdb3effdd24 100644
--- a/lib/hawat/blueprints/hosts/__init__.py
+++ b/lib/hawat/blueprints/hosts/__init__.py
@@ -96,7 +96,8 @@ class AbstractSearchView(PsycopgMixin, BaseSearchView): # pylint: disable=local
items,
dt_from=dt_from,
dt_to=dt_to,
- max_count=flask.current_app.config['HAWAT_CHART_TIMELINE_MAXSTEPS']
+ max_count=flask.current_app.config['HAWAT_CHART_TIMELINE_MAXSTEPS'],
+ timezone=pytz.timezone(flask.session.get('timezone', 'UTC'))
)
)
self.response_context.pop('items', None)
diff --git a/lib/hawat/blueprints/timeline/__init__.py b/lib/hawat/blueprints/timeline/__init__.py
index 6bf2701d98216a47118b7507700f40b4fcca1872..3688f8d2194ca7b2deb4d2c3bb26799c97f1665a 100644
--- a/lib/hawat/blueprints/timeline/__init__.py
+++ b/lib/hawat/blueprints/timeline/__init__.py
@@ -147,13 +147,15 @@ class AbstractSearchView(PsycopgMixin, CustomSearchView): # pylint: disable=loc
timeline_cfg = mentat.stats.idea.calculate_timeline_config(
form_data['dt_from'],
form_data['dt_to'],
- flask.current_app.config['HAWAT_CHART_TIMELINE_MAXSTEPS']
+ flask.current_app.config['HAWAT_CHART_TIMELINE_MAXSTEPS'],
+ timezone=pytz.timezone(flask.session.get('timezone', 'UTC'))
)
bucket_list = self.get_db().query_direct(
- 'SELECT generate_series(%s, %s, %s) AS bucket ORDER BY bucket',
+ '(SELECT %s AS bucket) UNION (SELECT generate_series(%s, %s, %s) AS bucket) ORDER BY bucket',
None,
[
timeline_cfg['dt_from'],
+ timeline_cfg['first_step'],
timeline_cfg['dt_to'],
timeline_cfg['step']
]
@@ -168,6 +170,7 @@ class AbstractSearchView(PsycopgMixin, CustomSearchView): # pylint: disable=loc
# Put calculated parameters together with other search form parameters.
form_data['dt_from'] = timeline_cfg['dt_from']
+ form_data['first_step'] = timeline_cfg['first_step']
form_data['dt_to'] = timeline_cfg['dt_to']
form_data['step'] = timeline_cfg['step']
@@ -423,7 +426,8 @@ class AbstractLegacySearchView(PsycopgMixin, BaseSearchView): # pylint: disable
items,
dt_from=dt_from,
dt_to=dt_to,
- max_count=flask.current_app.config['HAWAT_CHART_TIMELINE_MAXSTEPS']
+ max_count=flask.current_app.config['HAWAT_CHART_TIMELINE_MAXSTEPS'],
+ timezone=pytz.timezone(flask.session.get('timezone', 'UTC'))
)
)
self.response_context.pop('items', None)
diff --git a/lib/mentat/services/eventstorage.py b/lib/mentat/services/eventstorage.py
index 5a626aef4cd51e8e2b4aad173ec3cb9103656771..6027c356e45d446db5f5908f36c5536bc6d02f5f 100644
--- a/lib/mentat/services/eventstorage.py
+++ b/lib/mentat/services/eventstorage.py
@@ -275,10 +275,11 @@ def _bq_qbase_aggregate(parameters = None, qname = None, query = None, params =
def _bq_qbase_timeline(parameters = None, qname = None): # pylint: disable=locally-disabled,unused-argument
params = []
- query = psycopg2.sql.SQL('SELECT %s + %s * (width_bucket(detecttime, (SELECT array_agg(buckets) FROM generate_series(%s, %s, %s) AS buckets)) - 1) AS bucket,')
+ query = psycopg2.sql.SQL('SELECT GREATEST(%s, %s + %s * (width_bucket(detecttime, (SELECT array_agg(buckets) FROM generate_series(%s, %s, %s) AS buckets)) - 1)) AS bucket,')
params.append(parameters['dt_from'])
+ params.append(parameters['first_step'])
params.append(parameters['step'])
- params.append(parameters['dt_from'])
+ params.append(parameters['first_step'])
params.append(parameters['dt_to'])
params.append(parameters['step'])
return _bq_qbase_aggregate(parameters, qname, query, params)
diff --git a/lib/mentat/services/test_eventstorage.py b/lib/mentat/services/test_eventstorage.py
index f16e9d9aba68ea9a6e48fff981d08e314d7c901b..b4f6db73d3840e9c11a1c381705753e79fc16800 100644
--- a/lib/mentat/services/test_eventstorage.py
+++ b/lib/mentat/services/test_eventstorage.py
@@ -912,36 +912,39 @@ class TestMentatStorage(unittest.TestCase):
{
'parameters': {
'dt_from': datetime.datetime(2012, 11, 3, 10, 0, 7),
+ 'first_step': datetime.datetime(2012, 11, 3),
'dt_to': datetime.datetime(2012, 12, 3, 10, 0, 7),
'step': datetime.timedelta(days = 1),
},
'qtype': 'timeline'
},
- b'SELECT \'2012-11-03T10:00:07\'::timestamp + \'1 days 0.000000 seconds\'::interval * (width_bucket(detecttime,(SELECT array_agg(buckets) FROM generate_series(\'2012-11-03T10:00:07\'::timestamp,\'2012-12-03T10:00:07\'::timestamp,\'1 days 0.000000 seconds\'::interval) AS buckets)) - 1) AS bucket,COUNT(*) FROM events WHERE "detecttime" >= \'2012-11-03T10:00:07\'::timestamp AND "detecttime" <= \'2012-12-03T10:00:07\'::timestamp GROUP BY bucket ORDER BY bucket ASC'
+ b'SELECT GREATEST(\'2012-11-03T10:00:07\'::timestamp, \'2012-11-03T00:00:00\'::timestamp + \'1 days 0.000000 seconds\'::interval * (width_bucket(detecttime, (SELECT array_agg(buckets) FROM generate_series(\'2012-11-03T00:00:00\'::timestamp, \'2012-12-03T10:00:07\'::timestamp, \'1 days 0.000000 seconds\'::interval) AS buckets)) - 1)) AS bucket, COUNT(*) FROM events WHERE "detecttime" >= \'2012-11-03T10:00:07\'::timestamp AND "detecttime" <= \'2012-12-03T10:00:07\'::timestamp GROUP BY bucket ORDER BY bucket ASC'
),
(
{
'parameters': {
'dt_from': datetime.datetime(2012, 11, 3, 10, 0, 7),
+ 'first_step': datetime.datetime(2012, 11, 3),
'dt_to': datetime.datetime(2012, 12, 3, 10, 0, 7),
'step': datetime.timedelta(days = 1),
'aggr_set': 'eventclass',
},
'qtype': 'timeline'
},
- b'SELECT \'2012-11-03T10:00:07\'::timestamp + \'1 days 0.000000 seconds\'::interval * (width_bucket(detecttime,(SELECT array_agg(buckets) FROM generate_series(\'2012-11-03T10:00:07\'::timestamp,\'2012-12-03T10:00:07\'::timestamp,\'1 days 0.000000 seconds\'::interval) AS buckets)) - 1) AS bucket,"eventclass" AS set,COUNT(*) FROM events WHERE "detecttime" >= \'2012-11-03T10:00:07\'::timestamp AND "detecttime" <= \'2012-12-03T10:00:07\'::timestamp GROUP BY bucket, set ORDER BY bucket ASC'
+ b'SELECT GREATEST(\'2012-11-03T10:00:07\'::timestamp, \'2012-11-03T00:00:00\'::timestamp + \'1 days 0.000000 seconds\'::interval * (width_bucket(detecttime, (SELECT array_agg(buckets) FROM generate_series(\'2012-11-03T00:00:00\'::timestamp, \'2012-12-03T10:00:07\'::timestamp, \'1 days 0.000000 seconds\'::interval) AS buckets)) - 1)) AS bucket, "eventclass" AS set,COUNT(*) FROM events WHERE "detecttime" >= \'2012-11-03T10:00:07\'::timestamp AND "detecttime" <= \'2012-12-03T10:00:07\'::timestamp GROUP BY bucket, set ORDER BY bucket ASC'
),
(
{
'parameters': {
'dt_from': datetime.datetime(2012, 11, 3, 10, 0, 7),
+ 'first_step': datetime.datetime(2012, 11, 3),
'dt_to': datetime.datetime(2012, 12, 3, 10, 0, 7),
'step': datetime.timedelta(days = 1),
'aggr_set': 'category',
},
'qtype': 'timeline'
},
- b'SELECT \'2012-11-03T10:00:07\'::timestamp + \'1 days 0.000000 seconds\'::interval * (width_bucket(detecttime,(SELECT array_agg(buckets) FROM generate_series(\'2012-11-03T10:00:07\'::timestamp,\'2012-12-03T10:00:07\'::timestamp,\'1 days 0.000000 seconds\'::interval) AS buckets)) - 1) AS bucket,unnest("category") AS set,COUNT(*) FROM events WHERE "detecttime" >= \'2012-11-03T10:00:07\'::timestamp AND "detecttime" <= \'2012-12-03T10:00:07\'::timestamp GROUP BY bucket, set ORDER BY bucket ASC'
+ b'SELECT GREATEST(\'2012-11-03T10:00:07\'::timestamp, \'2012-11-03T00:00:00\'::timestamp + \'1 days 0.000000 seconds\'::interval * (width_bucket(detecttime, (SELECT array_agg(buckets) FROM generate_series(\'2012-11-03T00:00:00\'::timestamp, \'2012-12-03T10:00:07\'::timestamp, \'1 days 0.000000 seconds\'::interval) AS buckets)) - 1)) AS bucket, unnest("category") AS set, COUNT(*) FROM events WHERE "detecttime" >= \'2012-11-03T10:00:07\'::timestamp AND "detecttime" <= \'2012-12-03T10:00:07\'::timestamp GROUP BY bucket, set ORDER BY bucket ASC'
),
]
diff --git a/lib/mentat/stats/idea.py b/lib/mentat/stats/idea.py
index de25327b06a0acd6e4d12f39850a81d90bba7b7b..0571de004de04221b7142f2ae5c124ad4d13ece0 100644
--- a/lib/mentat/stats/idea.py
+++ b/lib/mentat/stats/idea.py
@@ -151,12 +151,15 @@ LIST_CALCSTAT_KEYS = tuple(
"""List of subkey names of all calculated statistics."""
-LIST_OPTIMAL_STEPS = (
- 1, 2, 3, 4, 5, 6, 10, 12, 15, 20, 30, # seconds
- 1*60, 2*60, 3*60, 4*60, 5*60, 6*60, 10*60, 12*60, 15*60, 20*60, 30*60, # minutes
- 1*3600, 2*3600, 3*3600, 4*3600, 6*3600, 8*3600, 12*3600, # hours
- 1*24*3600, 2*24*3600, 3*24*3600, 4*24*3600, 5*24*3600, 6*24*3600, 7*24*3600, 10*24*3600, 14*24*3600 # days
-)
+LIST_OPTIMAL_STEPS = [
+ datetime.timedelta(seconds=x) for x in
+ (
+ 1, 2, 3, 4, 5, 6, 10, 12, 15, 20, 30, # seconds
+ 1*60, 2*60, 3*60, 4*60, 5*60, 6*60, 10*60, 12*60, 15*60, 20*60, 30*60, # minutes
+ 1*3600, 2*3600, 3*3600, 4*3600, 6*3600, 8*3600, 12*3600, # hours
+ 1*24*3600, 2*24*3600, 3*24*3600, 4*24*3600, 5*24*3600, 6*24*3600, 7*24*3600, 10*24*3600, 14*24*3600 # days
+ )
+]
"""List of optimal timeline steps. This list is populated with values, that round nicelly in time calculations."""
@@ -275,7 +278,7 @@ def evaluate_events(events, stats = None):
return stats
-def evaluate_timeline_events(events, dt_from, dt_to, max_count, stats = None):
+def evaluate_timeline_events(events, dt_from, dt_to, max_count, timezone = None, stats = None):
"""
Evaluate statistics for given list of IDEA events and produce statistical
record for timeline visualisations.
@@ -301,7 +304,7 @@ def evaluate_timeline_events(events, dt_from, dt_to, max_count, stats = None):
# Prepare structure for storing IDEA event timeline statistics.
if ST_SKEY_TIMELINE not in stats:
- stats[ST_SKEY_TIMELINE], stats[ST_SKEY_TLCFG] = _init_timeline(dt_from, dt_to, max_count)
+ stats[ST_SKEY_TIMELINE], stats[ST_SKEY_TLCFG] = _init_timeline(dt_from, dt_to, max_count, timezone=timezone)
# Prepare event thresholding cache for detection of recurring events.
tcache = SimpleMemoryThresholdingCache()
@@ -336,7 +339,7 @@ def evaluate_timeline_events(events, dt_from, dt_to, max_count, stats = None):
return stats
-def evaluate_singlehost_events(host, events, dt_from, dt_to, max_count, stats = None):
+def evaluate_singlehost_events(host, events, dt_from, dt_to, max_count, timezone = None, stats = None):
"""
Evaluate statistics for given list of IDEA events and produce statistical
record for single host visualisations.
@@ -363,7 +366,7 @@ def evaluate_singlehost_events(host, events, dt_from, dt_to, max_count, stats =
# Prepare structure for storing IDEA event timeline statistics.
if ST_SKEY_TIMELINE not in stats:
- stats[ST_SKEY_TIMELINE], stats[ST_SKEY_TLCFG] = _init_timeline(dt_from, dt_to, max_count)
+ stats[ST_SKEY_TIMELINE], stats[ST_SKEY_TLCFG] = _init_timeline(dt_from, dt_to, max_count, timezone=timezone)
# Prepare event thresholding cache for detection of recurring events.
tcache = SingleSourceThresholdingCache(host)
@@ -534,7 +537,7 @@ def evaluate_dbstats_events(stats):
def group_events(events):
"""
- Group events according to the presence of the ``_Mentat.ResolvedAbuses`` (or
+ Group events according to the presence of the ``_Mentat.ResolvedAbuses`` (or
``_CESNET.ResolvedAbuses``) key.
Each event will be added to group ``overall`` and then to either ``internal``,
or ``external`` based on the presence of the key mentioned above.
@@ -620,7 +623,7 @@ def aggregate_stat_groups(stats_list, result = None):
return result
-def aggregate_timeline_groups(stats_list, dt_from, dt_to, max_count, min_step = None, result = None):
+def aggregate_timeline_groups(stats_list, dt_from, dt_to, max_count, min_step = None, timezone = None, result = None):
"""
Aggregate multiple full statistical records produced by the
:py:func:`mentat.stats.idea.evaluate_event_groups` function and later retrieved
@@ -663,7 +666,8 @@ def aggregate_timeline_groups(stats_list, dt_from, dt_to, max_count, min_step =
dt_from,
dt_to,
max_count,
- min_step
+ min_step,
+ timezone
)
# Precalculate list of timeline keys for subsequent bisection search.
@@ -703,16 +707,16 @@ def aggregate_timeline_groups(stats_list, dt_from, dt_to, max_count, min_step =
return result
-def calculate_timeline_config(dt_from, dt_to, max_count, min_step = None):
+def calculate_timeline_config(dt_from, dt_to, max_count, min_step = None, timezone = None):
"""
Calculate optimal configurations for timeline chart dataset.
"""
- dt_from, dt_to, delta = _calculate_timeline_boundaries(dt_from, dt_to) # pylint: disable=locally-disabled,unused-variable
- step, step_count = _calculate_timeline_steps(dt_from, dt_to, max_count, min_step)
+ first_step, step, step_count = _calculate_timeline_steps(dt_from, dt_to, max_count, min_step, timezone)
return {
ST_SKEY_DT_FROM: dt_from,
ST_SKEY_DT_TO: dt_to,
'step': step,
+ 'first_step': first_step,
'count': step_count
}
@@ -1039,88 +1043,103 @@ def _mask_toplist(stats, mask, dict_key, top_threshold, force = False):
return stats
-def _calculate_timeline_boundaries(dt_from, dt_to):
- """
- Calculate optimal and rounded values for lower and upper timeline boundaries
- from given timestamps.
- """
- delta_minute = datetime.timedelta(minutes = 1)
- delta_hour = datetime.timedelta(hours = 1)
- delta_day = datetime.timedelta(days = 1)
+def _round_datetime(datetime_, round_to, timezone = None, direction = None):
+ if timezone is None or not hasattr(timezone, 'localize'):
+ epoch = datetime.datetime(1970, 1, 1, tzinfo=timezone or datetime.timezone.utc)
+ else:
+ epoch = timezone.localize(datetime.datetime(1970, 1, 1))
- delta = dt_to - dt_from
+ mod = (datetime_.astimezone(datetime.timezone.utc) - epoch) % round_to
- # For delta of timeline boundaries below one hour round to the whole 5 minutes.
- if delta <= delta_hour:
- return (
- dt_from.replace(
- minute = dt_from.minute - (dt_from.minute % 5),
- second = 0,
- microsecond = 0
- ),
- dt_to.replace(
- second = 0,
- microsecond = 0
- ) + (delta_minute * (5 - (dt_to.minute % 5))),
- delta_minute * 5
- )
+ if not mod:
+ return datetime_
- # For delta of timeline boundaries below one day round to the whole hours.
- if delta <= delta_day:
- return (
- dt_from.replace(
- minute = 0,
- second = 0,
- microsecond = 0
- ),
- dt_to.replace(
- minute = 0,
- second = 0,
- microsecond = 0
- ) + delta_hour,
- delta_hour
- )
+ direction = direction or (mod < round_to / 2 and 'down') or 'up'
- # Everything else round to the whole days.
- return (
- dt_from.replace(
- hour = 0,
- minute = 0,
- second = 0,
- microsecond = 0
- ),
- dt_to.replace(
- hour = 0,
- minute = 0,
- second = 0,
- microsecond = 0
- ) + delta_day,
- delta_day
+ if direction == 'up':
+ return datetime_ + (round_to - mod)
+ return datetime_ - mod
+
+
+def _round_timedelta(delta, round_to, direction = None):
+ directions = {
+ 'up': math.ceil,
+ 'down': math.floor,
+ }
+ round_to_seconds = round_to.total_seconds()
+ delta_seconds = delta.total_seconds()
+ return datetime.timedelta(
+ seconds=round_to_seconds * directions.get(direction, round)(delta_seconds/ round_to_seconds)
)
-def _calculate_timeline_steps(dt_from, dt_to, max_count, min_step = None):
+def _optimize_step(step):
+ if step < LIST_OPTIMAL_STEPS[0]:
+ # Set the step size to lowest larger than step 1, 2, 5 or 10 times
+ # the largest smaller or equal than step negative power of 10
+ lower_bound = datetime.timedelta(
+ seconds=10 ** math.floor(math.log10(step.total_seconds()))
+ )
+ return lower_bound * next(
+ filter(lambda x: x * lower_bound >= step, (1, 2, 5, 10)),
+ 10 # This value should not be reachable
+ )
+
+ if step <= LIST_OPTIMAL_STEPS[-1]:
+ # Set the step size to the nearest larger or equal size from LIST_OPTIMAL_STEPS
+ idx = bisect.bisect_left(LIST_OPTIMAL_STEPS, step)
+ return LIST_OPTIMAL_STEPS[idx]
+
+ # Otherwise round the step to whole days
+ delta_day = datetime.timedelta(days = 1)
+ return _round_timedelta(step, delta_day, direction='up')
+
+
+def _calculate_timeline_steps(dt_from, dt_to, max_count, min_step = None, timezone = None):
"""
- Calculate optimal timeline step/window from given optimal lower and upper
- boundary and maximal number of steps.
+ Calculates the first step, the step size and actual step count
+
+ Example for calculated step size of 5s where:
+ dt_from = YYYY-MM-DDThh:mm:02,
+ dt_to = YYYY-MM-DDThh:mm:52,
+ max_count = 12
+
+ |---|-----|-----|-----|-----|-----|-----|-----|-----|-----|--|
+ │ ╰───────╮ ╰──┬──╯ │
+ dt_from first_step (rounded up to nearest 5s) step = 5s dt_to
+
+ step_count = 11
"""
delta = dt_to - dt_from
- step_size = int(math.ceil(delta.total_seconds()/max_count))
- if min_step and step_size < min_step:
- step_size = min_step
+ if not min_step:
+ min_step_delta = datetime.timedelta(microseconds=1)
+ else:
+ min_step_delta = datetime.timedelta(seconds=min_step)
+
+ if delta <= datetime.timedelta(0):
+ return dt_from, min_step_delta, 0
- # Attempt to optimalize the step size
- idx = bisect.bisect_left(LIST_OPTIMAL_STEPS, step_size)
- if idx != len(LIST_OPTIMAL_STEPS):
- step_size = LIST_OPTIMAL_STEPS[idx]
+ step = max(delta / max_count, min_step_delta)
+ step = _optimize_step(step)
- step = datetime.timedelta(seconds = step_size)
+ first_step = _round_datetime(dt_from, step, timezone=timezone, direction='up')
# Calculate actual step count, that will cover the requested timeline.
- step_count = int(math.ceil(delta/step))
+ step_count = math.ceil((dt_to - first_step) / step)
+
+ if step_count == max_count and first_step != dt_from:
+ # In case the step count would be higher than the max_count
+ # due to the shift of the first step, recalculate
+ step = max(delta / (max_count - 1), min_step_delta)
+ step = _optimize_step(step)
+ first_step = _round_datetime(dt_from, step, timezone=timezone, direction='up')
+ step_count = math.ceil((dt_to - first_step) / step)
+
+ if first_step != dt_from:
+ step_count += 1
- return (step, step_count)
+ return first_step, step, step_count
def _init_time_scatter():
@@ -1130,17 +1149,30 @@ def _init_time_scatter():
return [[{} for y in range(24)] for x in range(7)]
-def _init_timeline(dt_from, dt_to, max_count, min_step = None):
+def _steps_iter(dt_from, first_step, step, count):
+ yield dt_from
+ count -= 1
+ if first_step != dt_from:
+ yield first_step
+ count -= 1
+ for _ in range(count):
+ first_step += step
+ yield first_step
+
+
+def _init_timeline(dt_from, dt_to, max_count, min_step = None, timezone = None):
"""
Init structure for timeline chart dataset.
"""
- timeline_cfg = calculate_timeline_config(dt_from, dt_to, max_count, min_step)
+ timeline_cfg = calculate_timeline_config(dt_from, dt_to, max_count, min_step, timezone)
+
+ timeline = [[s, {}] for s in _steps_iter(
+ timeline_cfg[ST_SKEY_DT_FROM],
+ timeline_cfg['first_step'],
+ timeline_cfg['step'],
+ timeline_cfg['count']
+ )]
- dt_from = timeline_cfg[ST_SKEY_DT_FROM]
- timeline = list()
- for i in range(timeline_cfg['count']): # pylint: disable=locally-disabled,unused-variable
- timeline.append([dt_from, {}])
- dt_from = dt_from + timeline_cfg['step']
return timeline, timeline_cfg
diff --git a/lib/mentat/stats/test_idea.py b/lib/mentat/stats/test_idea.py
index acd99a02ab42c5f72fdb97dbcf2bfd8fc7389e47..b0892faeeab374677cfdc03201eab8dfca4850d5 100644
--- a/lib/mentat/stats/test_idea.py
+++ b/lib/mentat/stats/test_idea.py
@@ -10,7 +10,7 @@
import unittest
from pprint import pprint
-
+import pytz
import datetime
import mentat.stats.idea
@@ -301,58 +301,193 @@ class TestMentatStatsIdea(unittest.TestCase):
}
})
- def test_03_timeline_boundaries(self):
+ def test_03_datetime_rounding(self):
+ """Test datetime rounding"""
+ self.maxDiff = None
+
+ self.assertEqual(
+ mentat.stats.idea._round_datetime( # pylint: disable=locally-disabled,protected-access
+ datetime.datetime(2022, 10, 11, 11, 32, 12),
+ datetime.timedelta(hours=1),
+ direction='up'
+ ),
+ datetime.datetime(2022, 10, 11, 12, 0, 0)
+ )
+ self.assertEqual(
+ mentat.stats.idea._round_datetime( # pylint: disable=locally-disabled,protected-access
+ datetime.datetime(2022, 10, 11, 11, 32, 12),
+ datetime.timedelta(hours=1),
+ direction='down'
+ ),
+ datetime.datetime(2022, 10, 11, 11, 0, 0)
+ )
+ self.assertEqual(
+ mentat.stats.idea._round_datetime( # pylint: disable=locally-disabled,protected-access
+ datetime.datetime(2022, 10, 11, 11, 32, 12),
+ datetime.timedelta(hours=1)
+ ),
+ datetime.datetime(2022, 10, 11, 12, 0, 0)
+ )
+ self.assertEqual(
+ mentat.stats.idea._round_datetime( # pylint: disable=locally-disabled,protected-access
+ datetime.datetime(2022, 10, 11, 11, 32, 12),
+ datetime.timedelta(minutes=5),
+ direction='up'
+ ),
+ datetime.datetime(2022, 10, 11, 11, 35, 0)
+ )
+ self.assertEqual(
+ mentat.stats.idea._round_datetime( # pylint: disable=locally-disabled,protected-access
+ datetime.datetime(2022, 10, 11, 11, 32, 12),
+ datetime.timedelta(minutes=5),
+ direction='down'
+ ),
+ datetime.datetime(2022, 10, 11, 11, 30, 0)
+ )
+ self.assertEqual(
+ mentat.stats.idea._round_datetime( # pylint: disable=locally-disabled,protected-access
+ datetime.datetime(2022, 10, 11, 11, 32, 12),
+ datetime.timedelta(minutes=5)
+ ),
+ datetime.datetime(2022, 10, 11, 11, 30, 0)
+ )
+ self.assertEqual(
+ mentat.stats.idea._round_datetime( # pylint: disable=locally-disabled,protected-access
+ datetime.datetime(2022, 10, 11, 11, 32, 12),
+ datetime.timedelta(hours=1),
+ timezone=pytz.timezone('Europe/Prague'),
+ direction='up'
+ ),
+ datetime.datetime(2022, 10, 11, 12, 0, 0)
+ )
+ self.assertEqual(
+ mentat.stats.idea._round_datetime( # pylint: disable=locally-disabled,protected-access
+ datetime.datetime(2022, 10, 11, 11, 32, 12),
+ datetime.timedelta(hours=1),
+ timezone=pytz.timezone('Europe/Prague'),
+ direction='down'
+ ),
+ datetime.datetime(2022, 10, 11, 11, 0, 0)
+ )
+ self.assertEqual(
+ mentat.stats.idea._round_datetime( # pylint: disable=locally-disabled,protected-access
+ datetime.datetime(2022, 10, 11, 11, 32, 12),
+ datetime.timedelta(hours=1),
+ timezone=pytz.timezone('Europe/Prague')
+ ),
+ datetime.datetime(2022, 10, 11, 12, 0, 0)
+ )
+ self.assertEqual(
+ mentat.stats.idea._round_datetime( # pylint: disable=locally-disabled,protected-access
+ datetime.datetime(2022, 10, 11, 11, 32, 12),
+ datetime.timedelta(days=1),
+ timezone=pytz.timezone('Europe/Prague'),
+ direction='up'
+ ),
+ datetime.datetime(2022, 10, 11, 23, 0, 0)
+ )
+ self.assertEqual(
+ mentat.stats.idea._round_datetime( # pylint: disable=locally-disabled,protected-access
+ datetime.datetime(2022, 10, 11, 11, 32, 12),
+ datetime.timedelta(days=1),
+ timezone=pytz.timezone('Europe/Prague'),
+ direction='down'
+ ),
+ datetime.datetime(2022, 10, 10, 23, 0, 0)
+ )
+ self.assertEqual(
+ mentat.stats.idea._round_datetime( # pylint: disable=locally-disabled,protected-access
+ datetime.datetime(2022, 10, 11, 11, 32, 12),
+ datetime.timedelta(days=1),
+ timezone=pytz.timezone('Europe/Prague')
+ ),
+ datetime.datetime(2022, 10, 11, 23, 0, 0)
+ )
+ self.assertEqual(
+ mentat.stats.idea._round_datetime( # pylint: disable=locally-disabled,protected-access
+ datetime.datetime(2022, 10, 11, 11, 32, 12),
+ datetime.timedelta(days=1)
+ ),
+ datetime.datetime(2022, 10, 11, 0, 0, 0)
+ )
+
+ def test_04_timedelta_rounding(self):
"""
- Test timeline boundary calculations.
+ Test rounding of timedelta
"""
self.maxDiff = None
self.assertEqual(
- mentat.stats.idea._calculate_timeline_boundaries( # pylint: disable=locally-disabled,protected-access
- datetime.datetime(2018, 1, 1, 1, 11, 1),
- datetime.datetime(2018, 1, 1, 1, 31, 31)
+ mentat.stats.idea._round_timedelta( # pylint: disable=locally-disabled,protected-access
+ datetime.timedelta(seconds=42),
+ datetime.timedelta(seconds=10),
+ direction='up'
),
- (
- datetime.datetime(2018, 1, 1, 1, 10, 0),
- datetime.datetime(2018, 1, 1, 1, 35, 0),
- datetime.timedelta(seconds=300)
- )
+ datetime.timedelta(seconds=50)
)
self.assertEqual(
- mentat.stats.idea._calculate_timeline_boundaries( # pylint: disable=locally-disabled,protected-access
- datetime.datetime(2018, 1, 1, 1, 1, 1),
- datetime.datetime(2018, 1, 1, 1, 59, 31)
+ mentat.stats.idea._round_timedelta( # pylint: disable=locally-disabled,protected-access
+ datetime.timedelta(seconds=42),
+ datetime.timedelta(seconds=10),
+ direction='down'
),
- (
- datetime.datetime(2018, 1, 1, 1, 0, 0),
- datetime.datetime(2018, 1, 1, 2, 0, 0),
- datetime.timedelta(seconds=300)
- )
+ datetime.timedelta(seconds=40)
)
self.assertEqual(
- mentat.stats.idea._calculate_timeline_boundaries( # pylint: disable=locally-disabled,protected-access
- datetime.datetime(2018, 1, 1, 1, 11, 1),
- datetime.datetime(2018, 1, 1, 23, 59, 31)
+ mentat.stats.idea._round_timedelta( # pylint: disable=locally-disabled,protected-access
+ datetime.timedelta(seconds=42),
+ datetime.timedelta(seconds=10)
),
- (
- datetime.datetime(2018, 1, 1, 1, 0, 0),
- datetime.datetime(2018, 1, 2, 0, 0, 0),
- datetime.timedelta(seconds=3600)
- )
+ datetime.timedelta(seconds=40)
)
self.assertEqual(
- mentat.stats.idea._calculate_timeline_boundaries( # pylint: disable=locally-disabled,protected-access
- datetime.datetime(2018, 1, 1, 1, 11, 1),
- datetime.datetime(2018, 1, 11, 23, 59, 31)
+ mentat.stats.idea._round_timedelta( # pylint: disable=locally-disabled,protected-access
+ datetime.timedelta(microseconds=687231),
+ datetime.timedelta(microseconds=500),
+ direction='up'
),
- (
- datetime.datetime(2018, 1, 1, 0, 0, 0),
- datetime.datetime(2018, 1, 12, 0, 0, 0),
- datetime.timedelta(seconds=86400)
- )
+ datetime.timedelta(microseconds=687500)
+ )
+ self.assertEqual(
+ mentat.stats.idea._round_timedelta( # pylint: disable=locally-disabled,protected-access
+ datetime.timedelta(microseconds=687231),
+ datetime.timedelta(microseconds=500),
+ direction='down'
+ ),
+ datetime.timedelta(microseconds=687000)
+ )
+ self.assertEqual(
+ mentat.stats.idea._round_timedelta( # pylint: disable=locally-disabled,protected-access
+ datetime.timedelta(microseconds=687231),
+ datetime.timedelta(microseconds=500)
+ ),
+ datetime.timedelta(microseconds=687000)
+ )
+ self.assertEqual(
+ mentat.stats.idea._round_timedelta( # pylint: disable=locally-disabled,protected-access
+ datetime.timedelta(microseconds=687231),
+ datetime.timedelta(seconds=2),
+ direction='up'
+ ),
+ datetime.timedelta(seconds=2)
+ )
+ self.assertEqual(
+ mentat.stats.idea._round_timedelta( # pylint: disable=locally-disabled,protected-access
+ datetime.timedelta(microseconds=687231),
+ datetime.timedelta(seconds=2),
+ direction='down'
+ ),
+ datetime.timedelta(seconds=0)
+ )
+ self.assertEqual(
+ mentat.stats.idea._round_timedelta( # pylint: disable=locally-disabled,protected-access
+ datetime.timedelta(microseconds=687231),
+ datetime.timedelta(seconds=2)
+ ),
+ datetime.timedelta(seconds=0)
)
- def test_04_timeline_steps(self):
+ def test_05_timeline_steps(self):
"""
Test timeline step calculations.
"""
@@ -365,17 +500,130 @@ class TestMentatStatsIdea(unittest.TestCase):
100
),
(
+ datetime.datetime(2018, 1, 1, 3, 0, 0),
datetime.timedelta(seconds=10800),
88
)
)
+ self.assertEqual(
+ mentat.stats.idea._calculate_timeline_steps( # pylint: disable=locally-disabled,protected-access
+ datetime.datetime(2022, 10, 24, 15, 17, 5),
+ datetime.datetime(2022, 10, 24, 15, 22, 32),
+ 42
+ ),
+ (
+ datetime.datetime(2022, 10, 24, 15, 17, 10),
+ datetime.timedelta(seconds=10),
+ 34
+ )
+ )
+ self.assertEqual(
+ mentat.stats.idea._calculate_timeline_steps( # pylint: disable=locally-disabled,protected-access
+ datetime.datetime(2022, 10, 24, 15, 17, 5),
+ datetime.datetime(2022, 10, 24, 15, 18, 23),
+ 400
+ ),
+ (
+ datetime.datetime(2022, 10, 24, 15, 17, 5),
+ datetime.timedelta(microseconds=200000),
+ 390
+ )
+ )
+ self.assertEqual(
+ mentat.stats.idea._calculate_timeline_steps( # pylint: disable=locally-disabled,protected-access
+ datetime.datetime(2022, 10, 24, 15, 17, 5),
+ datetime.datetime(2022, 10, 24, 15, 18, 23),
+ 400,
+ 1
+ ),
+ (
+ datetime.datetime(2022, 10, 24, 15, 17, 5),
+ datetime.timedelta(seconds=1),
+ 78
+ )
+ )
+ self.assertEqual(
+ mentat.stats.idea._calculate_timeline_steps( # pylint: disable=locally-disabled,protected-access
+ datetime.datetime(2022, 10, 24, 15, 17, 5),
+ datetime.datetime(2022, 10, 24, 15, 18, 23),
+ 400,
+ 0.314159
+ ),
+ (
+ datetime.datetime(2022, 10, 24, 15, 17, 5),
+ datetime.timedelta(microseconds=500000),
+ 156
+ )
+ )
+ self.assertEqual(
+ mentat.stats.idea._calculate_timeline_steps( # pylint: disable=locally-disabled,protected-access
+ datetime.datetime(2022, 10, 24, 15, 17, 5),
+ datetime.datetime(2022, 10, 24, 15, 17, 5),
+ 200
+ ),
+ (
+ datetime.datetime(2022, 10, 24, 15, 17, 5),
+ datetime.timedelta(microseconds=1),
+ 0
+ )
+ )
+ self.assertEqual(
+ mentat.stats.idea._calculate_timeline_steps( # pylint: disable=locally-disabled,protected-access
+ datetime.datetime(2022, 10, 24, 15, 17, 5),
+ datetime.datetime(2022, 10, 24, 15, 17, 5),
+ 42,
+ 1
+ ),
+ (
+ datetime.datetime(2022, 10, 24, 15, 17, 5),
+ datetime.timedelta(seconds=1),
+ 0
+ )
+ )
+ self.assertEqual(
+ mentat.stats.idea._calculate_timeline_steps( # pylint: disable=locally-disabled,protected-access
+ datetime.datetime(2022, 10, 24, 15, 17, 0),
+ datetime.datetime(2022, 10, 24, 15, 23, 40),
+ 200
+ ),
+ (
+ datetime.datetime(2022, 10, 24, 15, 17, 0),
+ datetime.timedelta(seconds=2),
+ 200
+ )
+ )
+ self.assertEqual(
+ mentat.stats.idea._calculate_timeline_steps( # pylint: disable=locally-disabled,protected-access
+ datetime.datetime(2022, 10, 24, 15, 17, 1),
+ datetime.datetime(2022, 10, 24, 15, 23, 41),
+ 200
+ ),
+ (
+ datetime.datetime(2022, 10, 24, 15, 17, 3),
+ datetime.timedelta(seconds=3),
+ 134
+ )
+ )
+ self.assertEqual(
+ mentat.stats.idea._calculate_timeline_steps(
+ datetime.datetime(2022, 6, 2, 2, 17, 1),
+ datetime.datetime(2022, 12, 9, 12, 28, 34),
+ 200,
+ timezone=pytz.timezone('Canada/Newfoundland')
+ ),
+ (
+ datetime.datetime(2022, 6, 2, 3, 30, 0),
+ datetime.timedelta(days=1),
+ 192
+ )
+ )
- def test_05_calc_timeline_cfg(self):
+ def test_06_calc_timeline_cfg(self):
"""
Test timeline config calculations.
"""
- def test_06_evaluate_events(self):
+ def test_07_evaluate_events(self):
"""
Perform the message evaluation tests.
"""
@@ -429,7 +677,7 @@ class TestMentatStatsIdea(unittest.TestCase):
'severities': {'__unknown__': 6}
})
- def test_07_truncate_stats(self):
+ def test_08_truncate_stats(self):
"""
Perform the basic operativity tests.
"""
@@ -488,7 +736,7 @@ class TestMentatStatsIdea(unittest.TestCase):
}
)
- def test_08_group_events(self):
+ def test_09_group_events(self):
"""
Perform the basic operativity tests.
"""
@@ -676,7 +924,7 @@ class TestMentatStatsIdea(unittest.TestCase):
}
)
- def test_09_evaluate_event_groups(self):
+ def test_10_evaluate_event_groups(self):
"""
Perform the basic operativity tests.
"""
@@ -692,7 +940,7 @@ class TestMentatStatsIdea(unittest.TestCase):
pprint(result)
self.assertTrue(result)
- def test_10_merge_stats(self):
+ def test_11_merge_stats(self):
"""
Perform the statistics aggregation tests.
"""
@@ -746,7 +994,7 @@ class TestMentatStatsIdea(unittest.TestCase):
}
)
- def test_11_aggregate_stat_groups(self):
+ def test_12_aggregate_stat_groups(self):
"""
Perform the statistic group aggregation tests.
"""