From 922fb9746d569cc14e3567941debc004eac806be Mon Sep 17 00:00:00 2001
From: Jan Zerdik <zerdik@cesnet.cz>
Date: Fri, 15 Mar 2019 16:48:00 +0100
Subject: [PATCH] Better work with datetime in json Datetime in json
structured_data is now store in utc timezone, change to report timezone is
done in templates. (Redmine issue: #4499)
---
conf/requirements-latest.pip | 1 +
conf/requirements.pip | 1 +
conf/templates/reporter/_macros.common.txt.j2 | 2 +-
lib/hawat/blueprints/reports/__init__.py | 21 +++++++++++++++----
.../reports/templates/reports/show.html | 4 ++--
lib/mentat/reports/base.py | 17 ++++++++++++++-
lib/mentat/reports/event.py | 4 ++--
7 files changed, 40 insertions(+), 10 deletions(-)
diff --git a/conf/requirements-latest.pip b/conf/requirements-latest.pip
index c4c2a48b6..e2dd8d284 100644
--- a/conf/requirements-latest.pip
+++ b/conf/requirements-latest.pip
@@ -27,3 +27,4 @@ pynspect
ipranges
typedcols
idea-format
+python-dateutil
diff --git a/conf/requirements.pip b/conf/requirements.pip
index eb863de79..de67d8d3a 100644
--- a/conf/requirements.pip
+++ b/conf/requirements.pip
@@ -27,3 +27,4 @@ pynspect==0.16
ipranges==0.1.10
typedcols==0.1.13
idea-format==0.1.11
+python-dateutil==2.8.0
diff --git a/conf/templates/reporter/_macros.common.txt.j2 b/conf/templates/reporter/_macros.common.txt.j2
index d6f22d957..83e3a5900 100644
--- a/conf/templates/reporter/_macros.common.txt.j2
+++ b/conf/templates/reporter/_macros.common.txt.j2
@@ -36,7 +36,7 @@
{{ '{:30s}'.format(_('Source')) }} {{ '{:25s}'.format(_('First event time')) }} {{ '{:25s}'.format(_('Last event time')) }} {{ '{:>7s}'.format(_('Count')) }} {{ _('Protocol') }}
{{ '{}'.format('─' * 110) }}
{%- for ip in section_data | dictsort %}
- {{ '{:30s}'.format(ip[0]) }} {{ '{:<25s}'.format(ip[1]['first_time']) }} {{ '{:<25s}'.format(ip[1]['last_time']) }} {{ '{:>7s}'.format(format_decimal(ip[1]['count'])) }} {% if ip[1]['ports'] %}{{ ', '.join(ip[1]['ports']) }}{% else %}---{% endif %}
+ {{ '{:30s}'.format(ip[0]) }} {{ '{:<25s}'.format(format_rfctzdatetime(ip[1]['first_time'])) }} {{ '{:<25s}'.format(format_rfctzdatetime(ip[1]['last_time'])) }} {{ '{:>7s}'.format(format_decimal(ip[1]['count'])) }} {% if ip[1]['ports'] %}{{ ', '.join(ip[1]['ports']) }}{% else %}---{% endif %}
{%- endfor %}
{{ '{}'.format('─' * 110) }}
{{ report_references.render_section_reference(section_name, logger) }}
diff --git a/lib/hawat/blueprints/reports/__init__.py b/lib/hawat/blueprints/reports/__init__.py
index 0dce075af..6ba2832e7 100644
--- a/lib/hawat/blueprints/reports/__init__.py
+++ b/lib/hawat/blueprints/reports/__init__.py
@@ -24,6 +24,7 @@ __credits__ = "Pavel Kácha <pavel.kacha@cesnet.cz>, Andrea Kropáčová <andrea
import datetime
import pytz
import json
+import dateutil.parser
#
# Flask related modules.
@@ -32,7 +33,7 @@ import flask
import flask_login
import flask_principal
import flask_mail
-from flask_babel import gettext, lazy_gettext, force_locale
+from flask_babel import gettext, lazy_gettext, force_locale, format_datetime
from jinja2.loaders import ChoiceLoader, FileSystemLoader
from flask.helpers import locked_cached_property
import os.path
@@ -55,6 +56,8 @@ from hawat.blueprints.reports.forms import EventReportSearchForm, ReportingDashb
BLUEPRINT_NAME = 'reports'
"""Name of the blueprint as module global constant."""
+BABEL_RFC3339_FORMAT = "yyyy-MM-ddTHH:mm:ssZZZ"
+
def build_related_search_params(item):
"""
@@ -298,13 +301,23 @@ class ShowView(HTMLMixin, SQLAlchemyMixin, ItemShowView):
)
return action_menu
+ @staticmethod
+ def format_datetime(val, tz):
+ """
+ Static method that take string with isoformat datetime in utc and return
+ string with BABEL_RFC3339_FORMAT formated datetime in tz timezone
+ """
+ return format_datetime(dateutil.parser.parse(val).replace(tzinfo=pytz.utc).astimezone(tz), BABEL_RFC3339_FORMAT, rebase=False)
+
def do_before_response(self, **kwargs):
"""*Implementation* of :py:func:`hawat.base.RenderableView.do_before_response`."""
if 'item' in self.response_context and self.response_context['item']:
self.response_context.update(
- statistics = self.response_context['item'].statistics,
- template_vars = flask.current_app.mconfig[mentat.const.CKEY_CORE_REPORTER][mentat.const.CKEY_CORE_REPORTER_TEMPLATEVARS],
- form = FeedbackForm()
+ statistics = self.response_context['item'].statistics,
+ template_vars = flask.current_app.mconfig[mentat.const.CKEY_CORE_REPORTER][mentat.const.CKEY_CORE_REPORTER_TEMPLATEVARS],
+ form = FeedbackForm(),
+ format_datetime = ShowView.format_datetime,
+ tz = pytz.timezone(self.response_context['item'].structured_data["timezone"])
)
diff --git a/lib/hawat/blueprints/reports/templates/reports/show.html b/lib/hawat/blueprints/reports/templates/reports/show.html
index bdec1eccf..036bd5719 100644
--- a/lib/hawat/blueprints/reports/templates/reports/show.html
+++ b/lib/hawat/blueprints/reports/templates/reports/show.html
@@ -135,8 +135,8 @@
{%- set classes = section_name -%}
{%- endif -%}
<a href="{{ url_for('events.search', source_addrs=ip[0], classes=classes, st_from=item.dt_from, st_to=item.dt_to, severity=item.severity, submit='Search') }}">{% endif %}{{ ip[0] }}</td>{% if current_user.is_authenticated %}</a>{% endif %}
- <td>{{ ip[1]['first_time'] }}</td>
- <td>{{ ip[1]['last_time'] }}</td>
+ <td>{{ format_datetime(ip[1]['first_time'], tz) }}</td>
+ <td>{{ format_datetime(ip[1]['last_time'], tz) }}</td>
<td style="text-align:right">{{ ip[1]['count'] }}</td>
<td>{% if ip[1]['ports'] %}{{ ', '.join(ip[1]['ports']) }}{% else %}---{% endif %}</td>
{%- if current_user.is_authenticated %}
diff --git a/lib/mentat/reports/base.py b/lib/mentat/reports/base.py
index f091fc6ce..58d0ee58b 100644
--- a/lib/mentat/reports/base.py
+++ b/lib/mentat/reports/base.py
@@ -26,6 +26,7 @@ import time
import datetime
import gettext
import jinja2
+import dateutil.parser
from babel.numbers import format_decimal
from babel.dates import format_datetime, format_timedelta, get_timezone, UTC
@@ -163,11 +164,23 @@ class BaseReporter:
"""
return format_decimal(val, locale = self.locale)
+ def get_datetime(self, val):
+ """
+ Method tries parse datetime if provided with string, otherwise return val directly.
+ """
+ if isinstance(val, str):
+ try:
+ return dateutil.parser.parse(val)
+ except:
+ pass
+ return val
+
def format_datetime(self, val):
"""
Simple wrapper around :py:func:babel.dates.format_datetime` function
that takes care of figuring out the appropriate locale.
"""
+ val = self.get_datetime(val)
return format_datetime(val, locale = self.locale)
def format_localdatetime(self, val):
@@ -176,6 +189,7 @@ class BaseReporter:
that takes care of figuring out the appropriate locale and prints the
datetime in local timezone (local to the server).
"""
+ val = self.get_datetime(val)
epoch = time.mktime(val.timetuple())
offset = datetime.datetime.fromtimestamp(epoch) - datetime.datetime.utcfromtimestamp(epoch)
return format_datetime(val + offset, locale = self.locale)
@@ -186,6 +200,7 @@ class BaseReporter:
that takes care of figuring out the appropriate locale and prints the
datetime in configured timezone.
"""
+ val = self.get_datetime(val)
if self.timezone != 'UTC':
return format_datetime(val, tzinfo = self.tzinfo, locale = self.locale)
return format_datetime(val, locale = self.locale)
@@ -196,7 +211,7 @@ class BaseReporter:
that prints the datetime in configured timezone and enforced RFC 3339
format.
"""
-
+ val = self.get_datetime(val)
if self.timezone != 'UTC':
return format_datetime(val, BABEL_RFC3339_FORMAT, tzinfo = self.tzinfo, locale = self.locale)
return format_datetime(val, BABEL_RFC3339_FORMAT, locale = self.locale)
diff --git a/lib/mentat/reports/event.py b/lib/mentat/reports/event.py
index d7f2e1227..564dc495d 100644
--- a/lib/mentat/reports/event.py
+++ b/lib/mentat/reports/event.py
@@ -803,8 +803,8 @@ class EventReporter(BaseReporter):
ip_result["count"] += 1
for abuse_value in result.values():
for ip_value in abuse_value.values():
- ip_value["first_time"] = self.format_rfctzdatetime(ip_value["first_time"])
- ip_value["last_time"] = self.format_rfctzdatetime(ip_value["last_time"])
+ ip_value["first_time"] = ip_value["first_time"].isoformat()
+ ip_value["last_time"] = ip_value["last_time"].isoformat()
ip_value["ports"] = sorted(set(ip_value["ports"]))
return result
--
GitLab