Aggregation aggregations are calculated from timeline aggregations.

(Redmine issue: #6308)

lib/hawat/blueprints/timeline/__init__.py

@@ -29,6 +29,7 @@ from flask_babel import lazy_gettext
 import mentat.stats.idea
 import mentat.services.eventstorage
 from mentat.const import tr_
+from mentat.services.eventstorage import QTYPE_TIMELINE
 
 import hawat.events
 import vial.const
@@ -103,7 +104,7 @@ class AbstractSearchView(PsycopgMixin, CustomSearchView):  # pylint: disable=loc
 
     authorization = [vial.acl.PERMISSION_ANY]
 
-    url_params_unsupported = ('page', 'limit', 'sortby')
+    url_params_unsupported = ('page', 'sortby')
 
     @classmethod
     def get_view_icon(cls):
@@ -128,6 +129,7 @@ class AbstractSearchView(PsycopgMixin, CustomSearchView):  # pylint: disable=loc
 
         form_data['groups'] = [item.name for item in form_data['groups']]
 
+        # Timestamp timezone mangling magic. Convert everything to UTC naive.
         dt_from = form_data.get('dt_from', None)
         if dt_from:
             dt_from = dt_from.astimezone(pytz.utc)
@@ -141,6 +143,7 @@ class AbstractSearchView(PsycopgMixin, CustomSearchView):  # pylint: disable=loc
         form_data['dt_from'] = dt_from
         form_data['dt_to'] = dt_to
 
+        # Determine configurations for timelines.
         timeline_cfg = mentat.stats.idea.calculate_timeline_config(
             form_data['dt_from'],
             form_data['dt_to'],
@@ -156,7 +159,6 @@ class AbstractSearchView(PsycopgMixin, CustomSearchView):  # pylint: disable=loc
             ]
         )
         timeline_cfg['buckets'] = [x[0] for x in bucket_list]
-
         self.response_context['sqlqueries'].append(
             self.get_db().cursor.lastquery.decode('utf-8')
         )
@@ -164,6 +166,7 @@ class AbstractSearchView(PsycopgMixin, CustomSearchView):  # pylint: disable=loc
             timeline_cfg = timeline_cfg
         )
 
+        # Put calculated parameters together with other search form parameters.
         form_data['dt_from'] = timeline_cfg['dt_from']
         form_data['dt_to']   = timeline_cfg['dt_to']
         form_data['step']    = timeline_cfg['step']
@@ -202,40 +205,44 @@ class AbstractSearchView(PsycopgMixin, CustomSearchView):  # pylint: disable=loc
         self.response_context['search_result']["{}:{}".format(qtype, aggr_name)] = search_result
 
     def custom_search(self, form_args):
-        """
-        Perform actual search with given query.
-        """
         self.response_context.update(
-            search_result = {}
+            search_result = {},  # Raw database query results (rows).
+            aggregations = []    # Note all performed aggregations for further processing.
         )
 
-        # Total calculations should always run.
-        for qtype in ('aggregate', 'timeline'):
-            fargs = copy.deepcopy(form_args)
-            fargs.update({"aggr_set": None})
-            self._search_events_aggr(fargs, qtype, mentat.stats.idea.ST_SKEY_CNT_EVENTS, False)
-
-        for qtype in ('aggregate', 'timeline'):
-            for aggr_name, _, faupdates in AGGREGATIONS:
-                if 'aggregations' in form_args and form_args['aggregations'] and aggr_name not in form_args['aggregations']:
-                    self.logger.debug(
-                        "Skipping aggregation search {}:{}.".format(qtype, aggr_name)
-                    )
-                else:
-                    fargs = copy.deepcopy(form_args)
-                    fargs.update(faupdates)
-                    self._search_events_aggr(fargs, qtype, aggr_name)
+        # Always perform total event count timeline calculations.
+        fargs = copy.deepcopy(form_args)
+        fargs.update({"aggr_set": None})
+        self._search_events_aggr(
+            fargs,
+            QTYPE_TIMELINE,
+            mentat.stats.idea.ST_SKEY_CNT_EVENTS,
+            False
+        )
+
+        # Perform only timeline aggregations for the rest of selected aggragtions, aggregation
+        # aggregations will be computed from timelines later.
+        qtype = QTYPE_TIMELINE
+        for aggr_name, _, faupdates in AGGREGATIONS:
+            if 'aggregations' in form_args and form_args['aggregations'] and aggr_name not in form_args['aggregations']:
+                self.logger.debug(
+                    "Skipping aggregation search {}:{}.".format(qtype, aggr_name)
+                )
+            else:
+                fargs = copy.deepcopy(form_args)
+                fargs.update(faupdates)
+                self._search_events_aggr(fargs, qtype, aggr_name)
+                self.response_context['aggregations'].append(aggr_name)
+
 
     def do_after_search(self):
-        self.logger.debug(
-            "Calculating IDEA event timeline from search result."
-        )
         self.response_context.update(
             statistics = {
                 'timeline_cfg': self.response_context['timeline_cfg']
             }
         )
 
+        # Convert raw database rows into dataset structures.
         self.mark_time(
             'result_convert',
             'begin',
@@ -243,29 +250,26 @@ class AbstractSearchView(PsycopgMixin, CustomSearchView):  # pylint: disable=loc
             label = 'Converting result from database rows to statistical dataset',
             log = True
         )
-        # Total calculations should always run.
-        for aggr_type in ('aggregate', 'timeline'):
-            key = "{}:{}".format(aggr_type,  mentat.stats.idea.ST_SKEY_CNT_EVENTS)
-            mentat.stats.idea.aggregate_dbstats_events(
-                aggr_type,
-                mentat.stats.idea.ST_SKEY_CNT_EVENTS,
-                self.response_context['search_result'][key],
-                0,
-                self.response_context['timeline_cfg'],
-                self.response_context['statistics']
-            )
-        for aggr_type in ('aggregate', 'timeline'):
-            for aggr_name, default_val, _ in AGGREGATIONS:
-                key = "{}:{}".format(aggr_type, aggr_name)
-                if key in self.response_context['search_result']:
-                    mentat.stats.idea.aggregate_dbstats_events(
-                        aggr_type,
-                        aggr_name,
-                        self.response_context['search_result'][key],
-                        default_val,
-                        self.response_context['timeline_cfg'],
-                        self.response_context['statistics']
-                    )
+        key = "{}:{}".format(QTYPE_TIMELINE, mentat.stats.idea.ST_SKEY_CNT_EVENTS)
+        mentat.stats.idea.aggregate_dbstats_events(
+            QTYPE_TIMELINE,
+            mentat.stats.idea.ST_SKEY_CNT_EVENTS,
+            self.response_context['search_result'][key],
+            0,
+            self.response_context['timeline_cfg'],
+            self.response_context['statistics']
+        )
+        for aggr_name, default_val, _ in AGGREGATIONS:
+            key = "{}:{}".format(QTYPE_TIMELINE, aggr_name)
+            if key in self.response_context['search_result']:
+                mentat.stats.idea.aggregate_dbstats_events(
+                    QTYPE_TIMELINE,
+                    aggr_name,
+                    self.response_context['search_result'][key],
+                    default_val,
+                    self.response_context['timeline_cfg'],
+                    self.response_context['statistics']
+                )
         self.mark_time(
             'result_convert',
             'end',
@@ -274,32 +278,39 @@ class AbstractSearchView(PsycopgMixin, CustomSearchView):  # pylint: disable=loc
             log = True
         )
 
+        # Calculate aggregation aggregations from timeline aggregations.
+        aggr_name = mentat.stats.idea.ST_SKEY_CNT_EVENTS
+        self.mark_time(
+            'result_calculate',
+            'begin',
+            tag = 'calculate',
+            label = 'Calculating aggregation aggregations from timeline aggregations',
+            log = True
+        )
+        mentat.stats.idea.postcalculate_dbstats_events(
+            aggr_name,
+            self.response_context['statistics']
+        )
         for aggr_name, default_val, _ in AGGREGATIONS:
-            if aggr_name in self.response_context['statistics']:
-                self.mark_time(
-                    'result_recalculate_{}'.format(aggr_name),
-                    'begin',
-                    tag = 'calculate',
-                    label = 'Recalculating totals of {} in program for comparison'.format(aggr_name),
-                    log = True
-                )
+            if aggr_name in self.response_context['aggregations']:
                 mentat.stats.idea.postcalculate_dbstats_events(
                     aggr_name,
                     self.response_context['statistics']
                 )
-                self.mark_time(
-                    'result_recalculate_{}'.format(aggr_name),
-                    'end',
-                    tag = 'calculate',
-                    label = 'Done recalculating totals of {} in program for comparison'.format(aggr_name),
-                    log = True
-                )
+        self.mark_time(
+            'result_calculate',
+            'end',
+            tag = 'calculate',
+            label = 'Done calculating aggregation aggregations from timeline aggregations',
+            log = True
+        )
 
+        # Truncate all datasets
         self.mark_time(
             'result_truncate',
             'begin',
             tag = 'calculate',
-            label = 'Truncating dataset',
+            label = 'Truncating datasets',
             log = True
         )
         self.response_context['statistics'] = mentat.stats.idea.evaluate_dbstats_events(
@@ -309,7 +320,7 @@ class AbstractSearchView(PsycopgMixin, CustomSearchView):  # pylint: disable=loc
             'result_truncate',
             'end',
             tag = 'calculate',
-            label = 'Done truncating dataset',
+            label = 'Done truncating datasets',
             log = True
         )
 

lib/mentat/stats/idea.py

@@ -505,14 +505,13 @@ def postcalculate_dbstats_events(aggr_name, result):
     """
 
     """
-    result.setdefault('postcalc', {})
-    result['postcalc'][ST_SKEY_CNT_EVENTS] = result[ST_SKEY_CNT_EVENTS]
-    result['postcalc'].setdefault(aggr_name, {})
     for tlbucket in result[ST_SKEY_TIMELINE]:
         if aggr_name in tlbucket[1]:
-            for key, value in tlbucket[1][aggr_name].items():
-                _counter_inc(result['postcalc'], aggr_name, key, value)
-    result['postcalc'] = truncate_stats(result['postcalc'])
+            if isinstance(tlbucket[1][aggr_name], dict):
+                for key, value in tlbucket[1][aggr_name].items():
+                    _counter_inc(result, aggr_name, key, value)
+            else:
+                _counter_inc_one(result, aggr_name, tlbucket[1][aggr_name])
 
 def evaluate_dbstats_events(stats):
     """
@@ -783,6 +782,26 @@ def _counter_inc_all(stats, stat, key_list, increment = 1):
             _counter_inc(stats, stat, key, increment)
     return stats
 
+def _counter_inc_one(stats, stat, increment = 1):
+    """
+    Helper for incrementing given statistical parameter within given statistical
+    bundle.
+
+    :param dict stats: Structure containing all statistics.
+    :param str stat: Name of the statistical category.
+    :param str key: Name of the statistical key.
+    :param int increment: Counter increment.
+    :return: Updated structure containing statistics.
+    :rtype: dict
+    """
+
+    # I have considered using setdefault() method, but the performance is worse
+    # in comparison with using if (measured with cProfile module).
+    if not stat in stats:
+        stats[stat] = 0
+    stats[stat] += increment
+    return stats
+
 
 def _include_event_to_stats(stats, event, recurring = False, skip = None):
     """