Extended structured data.

More information in structured_data column in report table that can be used by macros, update of default and example macros. (Redmine issue: #5751)

conf/event_classes/default/email.j2

@@ -3,5 +3,5 @@
 {%- endmacro %}
 
 {% macro row(key, data) -%}
-{{ '{:30s}'.format(key) }}  {{ '{:<25s}'.format(format_rfctzdatetime(data['first_time'])) }}  {{ '{:<25s}'.format(format_rfctzdatetime(data['last_time'])) }}  {{ '{:>7s}'.format(format_decimal(data['count'])) }}  {% if data['ports'] %}{{ ', '.join(data['ports']) }}{% else %}---{% endif %}
+{{ '{:30s}'.format(key) }}  {{ '{:<25s}'.format(format_rfctzdatetime(data['first_time'])) }}  {{ '{:<25s}'.format(format_rfctzdatetime(data['last_time'])) }}  {{ '{:>7s}'.format(format_decimal(data['count'])) }}  {% if data['source']['proto'] or data['target']['proto'] %}{{ ', '.join((data['source']['proto'] + data['target']['proto']) | unique(case_sensitive=True) | sort) }}{% else %}---{% endif %}
 {%- endmacro %}
\ No newline at end of file

conf/event_classes/default/hawat.j2

-{% macro header() %}
+{% macro header(data) %}
                                             <th>{{ _('Source') }}</th>
                                             <th>{{ _('First event time') }}</th>
                                             <th>{{ _('Last event time') }}</th>
@@ -15,5 +15,5 @@
                                                 <td>{{ format_datetime(data['first_time'], tz) }}</td>
                                                 <td>{{ format_datetime(data['last_time'], tz) }}</td>
                                                 <td style="text-align:right">{{ data['count'] }}</td>
-                                                <td>{% if data['ports'] %}{{ ', '.join(data['ports']) }}{% else %}---{% endif %}</td>
+                                                <td>{% if 'source' in data and 'target' in data and (data['source']['proto'] or data['target']['proto']) %}{{ ', '.join((data['source']['proto'] + data['target']['proto']) | unique(case_sensitive=True) | sort) }}{% elif data['ports'] %}{{ ', '.join(data['ports']) }}{% else %}---{% endif %}</td>
 {% endmacro %}

conf/event_classes/event-class-01/email.j2

@@ -5,5 +5,5 @@
 {%- endmacro %}
 
 {% macro row(key, data) -%}
-{% if data['url'] %}{{ '{:40s}'.format(data['url']) }}{% else %}{{ '{:40s}'.format('---') }}{% endif %}  {{ '{:<15.15s}'.format(_('Too long string that should be truncated in email.')) }}  {{ '{:>7s}'.format(format_decimal(data['count'])) }}  {% if data['count'] % 2 == 0 %}{{ '{:>18s}'.format(_('Yes')) }}{% else %}{{ '{:>18s}'.format(_('No')) }}{% endif %}
+{% if data['source']['url'] %}{{ '{:40.40s}'.format(', '.join(data['source']['url'])) }}{% else %}{{ '{:40s}'.format('---') }}{% endif %}  {{ '{:<15.15s}'.format(_('Too long string that should be truncated in email.')) }}  {{ '{:>7s}'.format(format_decimal(data['count'])) }}  {% if data['count'] % 2 == 0 %}{{ '{:>18s}'.format(_('Yes')) }}{% else %}{{ '{:>18s}'.format(_('No')) }}{% endif %}
 {%- endmacro %}
\ No newline at end of file

conf/event_classes/event-class-01/hawat.j2

 {# This is testing file with macros for example class generatad by mentat-ideagen script. #}
 
-{% macro header() %}
-                                            <th>{{ _('URL') }}</th>
+{% macro header(data) %}
+                                            {%- for v in data.values() %}{% if 'source' in v and loop.first %}<th>{{ _('URL') }}</th>{% endif %}{% endfor %}
                                             <th>{{ _('Some other data') }}</th>
                                             <th style="text-align:right">{{ _('Count') }}</th>
                                             <th style="text-align:right">{{ _('Is count even?') }}</th>
 {% endmacro %}
 
 {% macro row(key, data, is_authenticated) %}
-                                                {%- if data['url'] %}
-                                                <td>{{ data['url'] }}</td>
+                                                {%- if 'source' in data %}
+                                                {%- if data['source']['url'] %}
+                                                <td>{{ data['source']['url'] }}</td>
                                                 {%- else %}
                                                 <td>---</td>
                                                 {%- endif %}
+                                                {%- endif %}
                                                 <td>{{ _('Too long string that should be truncated in email.') }}</td>
                                                 <td style="text-align:right">{{ data['count'] }}</td>
                                                 <td style="text-align:right">{% if data['count'] % 2 == 0 %}{{ _('Yes') }}{% else %}{{ _('No') }}{% endif %}</td>

lib/hawat/blueprints/reports/templates/reports/show.html

@@ -137,7 +137,7 @@
                                             {% if header is undefined %}
                                                 {% set header = default_header %}
                                             {% endif %}
-                                            {{ header() }}
+                                            {{ header(section_data) }}
                                             {%- if current_user.is_authenticated %}
                                             <th>{{ _('Feedback') }}</th>
                                             {%- endif %}

lib/mentat/reports/event.py

@@ -709,18 +709,47 @@ class EventReporter(BaseReporter):
                 ip_result = result.setdefault(event_class, {}).setdefault(str(ip), {
                     "first_time": datetime.datetime.max,
                     "last_time": datetime.datetime.min,
-                    "ports": [],
                     "count": 0,
+                    "conn_count": 0,
+                    "flow_count": 0,
+                    "packet_count": 0,
+                    "byte_count": 0,
+                    "source": {
+                        "hostname": {},
+                        "mac": {},
+                        "port": {},
+                        "proto": {},
+                        "url": {},
+                        "email": {},
+                    },
+                    "target": {
+                        "hostname": {},
+                        "mac": {},
+                        "port": {},
+                        "proto": {},
+                        "url": {},
+                        "email": {},
+                    },
                 })
-                ip_result["first_time"] = min(event["DetectTime"], ip_result["first_time"])
-                ip_result["last_time"] = max(event["DetectTime"], ip_result["last_time"])
-                ip_result["ports"] += jpath_values(event, "Source.Proto") + jpath_values(event, "Target.Proto")
+                ip_result["first_time"] = min(event.get("EventTime") or event["DetectTime"], ip_result["first_time"])
+                ip_result["last_time"] = max(event.get("CeaseTime") or event.get("EventTime") or event["DetectTime"], ip_result["last_time"])
                 ip_result["count"] += 1
+
+                for data_key, idea_key in (("conn_count", "ConnCount"), ("flow_count", "FlowCount"), ("packet_count", "PacketCount"), ("byte_count", "ByteCount")):
+                    ip_result[data_key] += event.get(idea_key, 0)
+
+                for st in ("Source", "Target"):
+                    for k in ("Hostname", "MAC", "Port", "Proto", "URL", "Email"):
+                        for v in jpath_values(event, st + "." + k):
+                            ip_result[st.lower()][k.lower()][v] = 1
+
         for abuse_value in result.values():
             for ip_value in abuse_value.values():
                 ip_value["first_time"] = ip_value["first_time"].isoformat()
                 ip_value["last_time"] = ip_value["last_time"].isoformat()
-                ip_value["ports"] = sorted(set(ip_value["ports"]))
+                for st in ("source", "target"):
+                    for k in ("hostname", "mac", "port", "proto", "url", "email"):
+                        ip_value[st][k] = sorted(ip_value[st][k].keys())
         return result
 
     def choose_attachments(self, ident, settings):