diff --git a/lib/hawat/blueprints/detectors/__init__.py b/lib/hawat/blueprints/detectors/__init__.py
index e521ed78d97b20f018c1b3bbdd87f7c4e077ebee..f346a7c0f5fa4be2795853417100234d3ed532ad 100644
--- a/lib/hawat/blueprints/detectors/__init__.py
+++ b/lib/hawat/blueprints/detectors/__init__.py
@@ -130,6 +130,8 @@ class ListView(HTMLMixin, SQLAlchemyMixin, ItemListView):
'createtime.asc': lambda x, y: x.order_by(y.createtime.asc()),
'name.desc': lambda x, y: x.order_by(y.name.desc()),
'name.asc': lambda x, y: x.order_by(y.name.asc()),
+ 'hits.desc': lambda x, y: x.order_by(y.hits.desc()),
+ 'hits.asc': lambda x, y: x.order_by(y.hits.asc()),
'credibility.desc': lambda x, y: x.order_by(y.credibility.desc()),
'credibility.asc': lambda x, y: x.order_by(y.credibility.asc())
}
diff --git a/lib/hawat/blueprints/detectors/forms.py b/lib/hawat/blueprints/detectors/forms.py
index a25833599d4b17fe6767b890148a35e523a400a0..57fcbaad8d563f55bc3dd954f8cfab9fcbea50e4 100644
--- a/lib/hawat/blueprints/detectors/forms.py
+++ b/lib/hawat/blueprints/detectors/forms.py
@@ -175,6 +175,8 @@ class DetectorSearchForm(hawat.forms.BaseSearchForm):
('createtime.asc', lazy_gettext('by creation time ascending')),
('name.desc', lazy_gettext('by name descending')),
('name.asc', lazy_gettext('by name ascending')),
+ ('hits.desc', lazy_gettext('by number of hits descending')),
+ ('hits.asc', lazy_gettext('by number of hits ascending')),
('credibility.asc', lazy_gettext('by credibility ascending')),
('credibility.asc', lazy_gettext('by credibility descending'))
],
diff --git a/lib/hawat/blueprints/detectors/templates/detectors/list.html b/lib/hawat/blueprints/detectors/templates/detectors/list.html
index cc75aa51504ce1987c2bf52c29bb2d9d4cc99f65..1fcb87cba6495a25381aee13083a1470a35c3f67 100644
--- a/lib/hawat/blueprints/detectors/templates/detectors/list.html
+++ b/lib/hawat/blueprints/detectors/templates/detectors/list.html
@@ -38,6 +38,9 @@
<th>
{{ _('Description') }}
</th>
+ <th>
+ {{ _('Hits') }} {{ macros_site.render_sorter(request.endpoint, query_params, 'hits') }}
+ </th>
<th data-toggle="tooltip" title="{{ _('Contextual item actions') }}">
{{ get_icon('actions') }} {{ _('Actions') }}
</th>
@@ -58,6 +61,9 @@
<td>
{{ item.description | default(_('<< unknown >>'), True) | truncate(50) }}
</td>
+ <td>
+ {{ item.hits }}
+ </td>
<td class="column-actions">
{{ macros_page.render_menu_context_actions(item) }}
</td>
diff --git a/lib/hawat/blueprints/detectors/templates/detectors/show.html b/lib/hawat/blueprints/detectors/templates/detectors/show.html
index f54dab447a69cbff0e576829761763186c833dcd..e46377c67a9235cf455888606605904c3efa5b9c 100644
--- a/lib/hawat/blueprints/detectors/templates/detectors/show.html
+++ b/lib/hawat/blueprints/detectors/templates/detectors/show.html
@@ -67,6 +67,22 @@
{{ item.description | default(_('<< unknown >>'), True) }}
</td>
</tr>
+ <tr>
+ <th>
+ {{ _('Registered') }}:
+ </th>
+ <td>
+ {{ babel_format_datetime(item.registered) | default(_('<< unknown >>'), True) }}
+ </td>
+ </tr>
+ <tr>
+ <th>
+ {{ _('Hits') }}:
+ </th>
+ <td>
+ <span class="badge">{{ item.hits }}</span>
+ </td>
+ </tr>
<tr>
<th>
{{ _('Credibility') }}:
diff --git a/lib/hawat/blueprints/reports/templates/reports/search.html b/lib/hawat/blueprints/reports/templates/reports/search.html
index 2a652122012de813457cdc0161e4518698bd4fa6..c768d772ca4e7b9e248755efb8c169e4e3b366fe 100644
--- a/lib/hawat/blueprints/reports/templates/reports/search.html
+++ b/lib/hawat/blueprints/reports/templates/reports/search.html
@@ -67,7 +67,7 @@
|
<strong>{{ _('Report window:') }}</strong> {{ babel_format_datetime(item.dt_from) }} - {{ babel_format_datetime(item.dt_to) }} ({{ babel_format_timedelta(item.delta) }})
|
- <strong>{{ _('Event counts:') }}</strong> {{ item.evcount_rep }} {{ _('reported') }}{% if item.type == 'summary' %} ({{ item.evcount_all }} {{ _('matched') }}, {{ item.evcount_new }} {{ _('new events') }}, {{ item.evcount_flt_blk }} {{ _('filtered out') }}, {{ item.evcount_thr_blk }} {{ _('thresholded') }}, {{ item.evcount_rlp }} {{ _('relapsed') }}){% else %}, {{ item.evcount_all }} {{ _('total in parent summary report') }}{% endif %}
+ <strong>{{ _('Event counts:') }}</strong> {{ item.evcount_rep }} {{ _('reported') }}{% if item.type == 'summary' %} ({{ item.evcount_all }} {{ _('matched') }}, {{ item.evcount_new }} {{ _('new events') }}, {{ item.evcount_flt_blk }} {{ _('filtered out') }}, {{ item.evcount_det_blk if item.evcount_det_blk else 0 }} {{ _('uncredible') }}, {{ item.evcount_thr_blk }} {{ _('thresholded') }}, {{ item.evcount_rlp }} {{ _('relapsed') }}){% else %}, {{ item.evcount_all }} {{ _('total in parent summary report') }}{% endif %}
</p>
<hr>
<div class="progress" data-toggle="tooltip" title="{{ _('Number of events in this report in comparison with other reports in the result.') }}">
diff --git a/lib/hawat/blueprints/reports/templates/reports/show.html b/lib/hawat/blueprints/reports/templates/reports/show.html
index 8d9b143828494dcb3190dad9991b319c8d962f17..8cc612b1aca8bb16666cff5d6c6afd142d260009 100644
--- a/lib/hawat/blueprints/reports/templates/reports/show.html
+++ b/lib/hawat/blueprints/reports/templates/reports/show.html
@@ -302,7 +302,7 @@
</div>
</div>
{%- endif %}
- <strong>{{ item.evcount_rep }} ({{ babel_format_percent(item.evcount_rep / item.evcount_all) }})</strong> {{ _('reported') }}{% if item.type == 'summary' %}, {{ item.evcount_all }} ({{ babel_format_percent(item.evcount_all / item.evcount_all) }}) {{ _('matched') }}, {{ item.evcount_new }} ({{ babel_format_percent(item.evcount_new / item.evcount_all) }}) {{ _('new events') }}, {{ item.evcount_flt_blk }} ({{ babel_format_percent(item.evcount_flt_blk / item.evcount_all) }}) {{ _('filtered out') }}, {{ item.evcount_thr_blk }} ({{ babel_format_percent(item.evcount_thr_blk / item.evcount_all) }}) {{ _('thresholded') }}, {{ item.evcount_rlp }} ({{ babel_format_percent(item.evcount_rlp / item.evcount_all) }}) {{ _('relapsed') }}{% else %}, {{ item.evcount_all }} {{ _('total in parent summary report') }}{%- endif %}
+ <strong>{{ item.evcount_rep }} ({{ babel_format_percent(item.evcount_rep / item.evcount_all) }})</strong> {{ _('reported') }}{% if item.type == 'summary' %}, {{ item.evcount_all }} ({{ babel_format_percent(item.evcount_all / item.evcount_all) }}) {{ _('matched') }}, {{ item.evcount_new }} ({{ babel_format_percent(item.evcount_new / item.evcount_all) }}) {{ _('new events') }}, {{ item.evcount_flt_blk }} ({{ babel_format_percent(item.evcount_flt_blk / item.evcount_all) }}) {{ _('filtered out') }}, {{ item.evcount_det_blk if item.evcount_det_blk else 0 }} ({{ babel_format_percent((item.evcount_det_blk if item.evcount_det_blk else 0) / item.evcount_all) }}) {{ _('uncredible') }}, {{ item.evcount_thr_blk }} ({{ babel_format_percent(item.evcount_thr_blk / item.evcount_all) }}) {{ _('thresholded') }}, {{ item.evcount_rlp }} ({{ babel_format_percent(item.evcount_rlp / item.evcount_all) }}) {{ _('relapsed') }}{% else %}, {{ item.evcount_all }} {{ _('total in parent summary report') }}{%- endif %}
</td>
</tr>
<tr>
diff --git a/lib/hawat/migrations/versions/4e6cef4ff5ce_add_registered_column.py b/lib/hawat/migrations/versions/4e6cef4ff5ce_add_registered_column.py
new file mode 100644
index 0000000000000000000000000000000000000000..e808fa650b371d9d62e2d1c0057fcd74b56bf464
--- /dev/null
+++ b/lib/hawat/migrations/versions/4e6cef4ff5ce_add_registered_column.py
@@ -0,0 +1,25 @@
+"""Add registered and hits columns
+
+Revision ID: 4e6cef4ff5ce
+Revises: 1a58ce62406a
+Create Date: 2023-01-02 12:13:41.583488
+
+"""
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision = '4e6cef4ff5ce'
+down_revision = '1a58ce62406a'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+ op.execute("ALTER TABLE detectors ADD COLUMN registered TIMESTAMP without time zone")
+ op.execute("ALTER TABLE detectors ADD COLUMN hits INTEGER NOT NULL DEFAULT 0")
+
+
+def downgrade():
+ op.execute("ALTER TABLE detectors DROP COLUMN registered")
+ op.execute("ALTER TABLE detectors DROP COLUMN hits")
diff --git a/lib/hawat/migrations/versions/d76d19d30c2c_add_count_of_uncredible_events_to_.py b/lib/hawat/migrations/versions/d76d19d30c2c_add_count_of_uncredible_events_to_.py
new file mode 100644
index 0000000000000000000000000000000000000000..41ffd008e6a89c00a5529910ab444c9d333db499
--- /dev/null
+++ b/lib/hawat/migrations/versions/d76d19d30c2c_add_count_of_uncredible_events_to_.py
@@ -0,0 +1,25 @@
+"""Add count of uncredible events to reports
+
+Revision ID: d76d19d30c2c
+Revises: 4e6cef4ff5ce
+Create Date: 2023-01-05 11:50:03.315521
+
+"""
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision = 'd76d19d30c2c'
+down_revision = '4e6cef4ff5ce'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+ op.execute("ALTER TABLE reports_events ADD COLUMN evcount_det INTEGER")
+ op.execute("ALTER TABLE reports_events ADD COLUMN evcount_det_blk INTEGER")
+
+
+def downgrade():
+ op.execute("ALTER TABLE reports_events DROP COLUMN evcount_det")
+ op.execute("ALTER TABLE reports_events DROP COLUMN evcount_det_blk")
diff --git a/lib/mentat/datatype/internal.py b/lib/mentat/datatype/internal.py
index 46bd8382a4cb7868ce1f12d62131bfff3e88c230..8b3ad251b1b87e8732baabfc78abe31e539ac843 100644
--- a/lib/mentat/datatype/internal.py
+++ b/lib/mentat/datatype/internal.py
@@ -421,7 +421,8 @@ def t_detector_record(val, source):
record['credibility'] = val['credibility']
else:
record['credibility'] = 1.0
- #record['credibility']
+ if 'registered' in val:
+ record['registered'] = val['registered']
except:
raise ValueError('Unknown detector record {}'.format(pprint.pformat(val)))
return Detector(record)
@@ -1006,6 +1007,13 @@ def typedef_detector(flavour, list_flavour, addon = None):
'credibility': {
'type': flavour['Float'],
'required': True
+ },
+ 'registered': {
+ 'type': flavour['Datetime'],
+ 'default': time.time
+ },
+ 'hits': {
+ 'type': flavour['Integer']
}
}
if addon is not None:
@@ -1119,6 +1127,14 @@ def typedef_report(flavour, list_flavour, addon = None):
'type': flavour['Integer'],
'required': True
},
+ 'cnt_det': {
+ 'type': flavour['Integer'],
+ 'required': True
+ },
+ 'cnt_det_blk': {
+ 'type': flavour['Integer'],
+ 'required': True
+ },
'cnt_thr': {
'type': flavour['Integer'],
'required': True
diff --git a/lib/mentat/datatype/sqldb.py b/lib/mentat/datatype/sqldb.py
index 386badd0703e166d9bd68e96de3dbafbe402c65b..182724bd041e424eaa4297f70b1b6c777dbc3b05 100644
--- a/lib/mentat/datatype/sqldb.py
+++ b/lib/mentat/datatype/sqldb.py
@@ -834,9 +834,13 @@ class EventReportModel(MODEL):
evcount_flt = sqlalchemy.Column(sqlalchemy.Integer)
# Number of events blocked by filters (evcount_new - evcount_flt).
evcount_flt_blk = sqlalchemy.Column(sqlalchemy.Integer)
+ # Number of events remaining after filtering by detectors credibility.
+ evcount_det = sqlalchemy.Column(sqlalchemy.Integer)
+ # Number of events coming from uncredible detectors (evcount_flt - evcount_dlt).
+ evcount_det_blk = sqlalchemy.Column(sqlalchemy.Integer)
# Number of events remaining after thresholding.
evcount_thr = sqlalchemy.Column(sqlalchemy.Integer)
- # Number of events blocked by thresholds (evcount_flt - evcount_thr).
+ # Number of events blocked by thresholds (evcount_dlt - evcount_thr).
evcount_thr_blk = sqlalchemy.Column(sqlalchemy.Integer)
# Number of relapsed events.
evcount_rlp = sqlalchemy.Column(sqlalchemy.Integer)
@@ -878,6 +882,8 @@ class EventReportModel(MODEL):
'evcount_new': int(self.evcount_new) if self.evcount_new else 0,
'evcount_flt': int(self.evcount_flt) if self.evcount_flt else 0,
'evcount_flt_blk': int(self.evcount_flt_blk) if self.evcount_flt_blk else 0,
+ 'evcount_det': int(self.evcount_det) if self.evcount_det else 0,
+ 'evcount_det_blk': int(self.evcount_det_blk) if self.evcount_det_blk else 0,
'evcount_thr': int(self.evcount_thr) if self.evcount_thr else 0,
'evcount_thr_blk': int(self.evcount_thr_blk) if self.evcount_thr_blk else 0,
'evcount_rlp': int(self.evcount_rlp) if self.evcount_rlp else 0,
@@ -955,6 +961,8 @@ class DetectorModel(MODEL): # pylint: disable=locally-disabled,too-few-public-m
description = sqlalchemy.Column(sqlalchemy.String)
source = sqlalchemy.Column(sqlalchemy.String(50), nullable = False)
credibility = sqlalchemy.Column(sqlalchemy.Float, nullable = False)
+ registered = sqlalchemy.Column(sqlalchemy.DateTime)
+ hits = sqlalchemy.Column(sqlalchemy.Integer, default = 0, nullable = False)
def __repr__(self):
return "<Detector(id='%d',name='%s')>" % (int(self.id), str(self.name))
@@ -969,7 +977,9 @@ class DetectorModel(MODEL): # pylint: disable=locally-disabled,too-few-public-m
'name': str(self.name),
'source': str(self.source),
'description': str(self.description),
- 'credibility': float(self.credibility)
+ 'credibility': float(self.credibility),
+ 'registered': str(self.registered),
+ 'hits': int(self.hits)
}
def detectormodel_from_typeddict(structure, defaults = None):
@@ -985,6 +995,8 @@ def detectormodel_from_typeddict(structure, defaults = None):
sqlobj.source = structure.get('source')
sqlobj.credibility = structure.get('credibility', defaults.get('credibility', 1.0))
sqlobj.description = structure.get('description', defaults.get('description', None))
+ sqlobj.registered = structure.get('registered', defaults.get('registered', None))
+ sqlobj.hits = structure.get('hits', 0)
return sqlobj
diff --git a/lib/mentat/datatype/test_sqldb.py b/lib/mentat/datatype/test_sqldb.py
index d912abd1297c50d1e3e2e2943796693c465ae1c9..fe5c55f19c9df5973947647747c46f85d7711837 100644
--- a/lib/mentat/datatype/test_sqldb.py
+++ b/lib/mentat/datatype/test_sqldb.py
@@ -531,6 +531,8 @@ class TestMentatDatatypeSqldb(unittest.TestCase):
evcount_new = 3,
evcount_flt = 4,
evcount_flt_blk = 5,
+ evcount_det = 6,
+ evcount_det_blk = 7,
evcount_thr = 6,
evcount_thr_blk = 7,
evcount_rlp = 8,
diff --git a/lib/mentat/module/detmngr.py b/lib/mentat/module/detmngr.py
index 6afdf2a00ad9f2f5a30a4a0ad9a0dd4cc2a88851..5cd00780ecb96135cefbab8973f99d71ee372d4b 100644
--- a/lib/mentat/module/detmngr.py
+++ b/lib/mentat/module/detmngr.py
@@ -346,6 +346,7 @@ class MentatDetmngrScript(mentat.script.fetcher.FetcherScript):
continue
self.logger.warning("Updating existing detector '%s'.", detkey)
detector_db.credibility = detector_file['credibility']
+ detector_db.registered = detector_file['registered']
self.sqlservice.session.commit()
@@ -360,4 +361,4 @@ class MentatDetmngrScript(mentat.script.fetcher.FetcherScript):
:return: True or False
:rtype: bool
"""
- return det_db.credibility != det_file['credibility']
+ return det_db.credibility != det_file['credibility'] or det_db.registered != det_file['registered']
diff --git a/lib/mentat/module/reporter.py b/lib/mentat/module/reporter.py
index 25669f149e747a0bdc9aa44b3ef2a7ddc11bd7c0..d2808ed22b40903f184b340ae9501d768c6b5cfc 100644
--- a/lib/mentat/module/reporter.py
+++ b/lib/mentat/module/reporter.py
@@ -67,6 +67,7 @@ Reporting algorithm follows these steps:
#. Fetch events with given severity, that appeared in database in given
time window and belonging to that particular group.
#. Filter events with configured reporting filters.
+ #. Remove events from detectors with low credibility.
#. Threshold already reported events.
#. Fetch relapsed events.
#. Generate *summary* and/or *extra* reports and store them to database.
@@ -578,6 +579,7 @@ class MentatReporterScript(mentat.script.fetcher.FetcherScript):
['Relapsed [#]', '{:,d}'.format(analysis['report']['evcount_rlp'])],
['Filtered [#]', '{:,d}'.format(analysis['report']['evcount_flt_blk'])],
['Thresholded [#]', '{:,d}'.format(analysis['report']['evcount_thr_blk'])],
+ ['Uncredible [#]', '{:,d}'.format(analysis['report']['evcount_det_blk'])],
['Reports [#]', '{:,d}'.format(analysis['report']['report_count'])],
]
@@ -623,6 +625,7 @@ class MentatReporterScript(mentat.script.fetcher.FetcherScript):
{ 'label': 'New [#]', 'data_formating': '{:,d}', 'align': '>' },
{ 'label': 'Relapsed [#]', 'data_formating': '{:,d}', 'align': '>' },
{ 'label': 'Filtered [#]', 'data_formating': '{:,d}', 'align': '>' },
+ { 'label': 'Uncredible [#]', 'data_formating': '{:,d}', 'align': '>' },
{ 'label': 'Thresholded [#]', 'data_formating': '{:,d}', 'align': '>' },
{ 'label': 'Reports [#]', 'data_formating': '{:,d}', 'align': '>' },
]
@@ -638,6 +641,7 @@ class MentatReporterScript(mentat.script.fetcher.FetcherScript):
anl['report'].get('evcount_new', 0),
anl['report'].get('evcount_rlp', 0),
anl['report'].get('evcount_flt_blk', 0),
+ anl['report'].get('evcount_det_blk', 0),
anl['report'].get('evcount_thr_blk', 0),
anl['report'].get('report_count', 0),
]
diff --git a/lib/mentat/reports/event.py b/lib/mentat/reports/event.py
index 3e2f02bb10920e6d26d728ca792f022a62fbd013..f8986080cb9a9a043f3f47ec6ae681170f04e28c 100644
--- a/lib/mentat/reports/event.py
+++ b/lib/mentat/reports/event.py
@@ -41,7 +41,7 @@ import mentat.stats.idea
import mentat.services.whois
from mentat.const import tr_
from mentat.reports.utils import StorageThresholdingCache, NoThresholdingCache
-from mentat.datatype.sqldb import EventReportModel
+from mentat.datatype.sqldb import EventReportModel, DetectorModel
from mentat.emails.event import ReportEmail
from mentat.reports.base import BaseReporter
from mentat.services.eventstorage import record_to_idea
@@ -94,6 +94,7 @@ class EventReporter(BaseReporter):
self.groups_dict = groups_dict
self.settings_dict = settings_dict
+ self.detectors_dict = {det.name : det for det in self.sqlservice.session.query(DetectorModel).all()}
self.filter_parser.build()
@@ -210,17 +211,30 @@ class EventReporter(BaseReporter):
)
break
+ # Create new dictionary to store events coming from credible detectors.
+ aggregated_credible_events = {}
for groups, events_aggr in aggregated_events.items():
group_chain = groups[0]
- # C: Perform event thresholding.
+ # C: Discard events from detectors with low credibility.
+ _events_aggr, blocked_cnt = self.filter_events_by_credibility(events_aggr)
+ # If all events were discarded, _events_aggr is None.
+ if _events_aggr:
+ aggregated_credible_events[groups] = _events_aggr
+ # Save information about how many events passed and how many were discarded.
+ result[str(group_chain)]['evcount_det'] = result['evcount_flt'] - blocked_cnt
+ result[str(group_chain)]['evcount_det_blk'] = blocked_cnt
+
+ for groups, events_aggr in aggregated_credible_events.items():
+ group_chain = groups[0]
+ # D: Perform event thresholding.
events_thr, events_aggr = self.threshold_events(events_aggr, abuse_group, group_chain, severity, time_h)
result[str(group_chain)]['evcount_thr'] = len(events_thr)
- result[str(group_chain)]['evcount_thr_blk'] = len(events_passed_filters[groups]) - len(events_thr)
+ result[str(group_chain)]['evcount_thr_blk'] = result[str(group_chain)]['evcount_det'] - len(events_thr)
if not events_thr:
continue
- # D: Save aggregated events for further processing.
+ # E: Save aggregated events for further processing.
events[groups] = {}
events[groups]['regular'] = events_thr
events[groups]['regular_aggr'] = events_aggr
@@ -307,6 +321,8 @@ class EventReporter(BaseReporter):
evcount_new = result[str(group_chain)].get('evcount_new', 0) + evcount_flt_blk,
evcount_flt = result[str(group_chain)].get('evcount_new', 0),
evcount_flt_blk = evcount_flt_blk,
+ evcount_det = result[str(group_chain)].get('evcount_det', 0),
+ evcount_det_blk = result[str(group_chain)].get('evcount_det_blk', 0),
evcount_thr = result[str(group_chain)].get('evcount_thr', 0),
evcount_thr_blk = result[str(group_chain)].get('evcount_thr_blk', 0),
evcount_rlp = result[str(group_chain)].get('evcount_rlp', 0),
@@ -350,7 +366,7 @@ class EventReporter(BaseReporter):
"""
Generate extra reports from given events for given abuse group, severity and period.
- :param mentat.datatype.internal.EventReportModel parent_rep: Parent summary report.
+ :param mentat.datatype.sqldb.EventReportModel parent_rep: Parent summary report.
:param dict result: Reporting result structure with various usefull metadata.
:param dict events: Dictionary structure with IDEA events to be reported.
:param list group_chain: List of resolved abuse groups.
@@ -537,6 +553,42 @@ class EventReporter(BaseReporter):
fallback_groups, fltlog = self._filter_groups(fallback_groups, event, fltlog)
return filtered_groups, fallback_groups, fltlog
+ def filter_events_by_credibility(self, events_aggr):
+ """
+ Filter given dictionary of IDEA events aggregated by the source IP address by detector credibility.
+ If the resulting credibility is less than 0.5, the event is discarded from the report.
+
+ :param dict events_aggt: Dictionary of IDEA events as :py:class:`mentat.idea.internal.Idea` objects.
+ :return: Tuple with filtered dictionary, number of events passed, number of events discarded.
+ :rtype: tuple
+ """
+ blocked = set()
+ _events_aggr = {}
+ for ip in events_aggr:
+ for event in events_aggr[ip]:
+ _pass = 1.0
+ for detector in event.get_detectors():
+ if detector not in self.detectors_dict:
+ self.logger.warning("Event with ID '%s' contains unknown detector '%s'. Assuming full credibility.", event.get_id(), detector)
+ continue
+ _pass *= self.detectors_dict[detector].credibility
+ if _pass < 0.5:
+ if event.get_id() in blocked:
+ continue
+ self.logger.debug("Discarding event with ID '%s'.", event.get_id())
+ blocked.add(event.get_id())
+ # Increase number of hits.
+ sql_detector = self.detectors_dict[event.get_detectors()[-1]]
+ sql_detector.hits += 1
+ # Inefficient but rare so should be alright.
+ self.sqlservice.session.add(sql_detector)
+ self.sqlservice.session.commit()
+ else:
+ if ip not in _events_aggr:
+ _events_aggr[ip] = []
+ _events_aggr[ip].append(event)
+ return _events_aggr, len(blocked)
+
def filter_events(self, main_group, events):
"""
Filter given list of IDEA events according to given abuse group settings.
diff --git a/lib/mentat/reports/test_event.py b/lib/mentat/reports/test_event.py
index 2e71b35b5c0482dab9892315a247aeee56879acc..09d8fc025cdaebc01eae7fd3bd67132c3fd079c6 100644
--- a/lib/mentat/reports/test_event.py
+++ b/lib/mentat/reports/test_event.py
@@ -31,7 +31,7 @@ import mentat.idea.internal
import mentat.reports.utils
import mentat.reports.event
from mentat.datatype.sqldb import GroupModel, FilterModel, NetworkModel, \
- SettingsReportingModel, EventReportModel
+ SettingsReportingModel, EventReportModel, DetectorModel
from pynspect.jpath import jpath_values
#-------------------------------------------------------------------------------
@@ -165,7 +165,12 @@ class TestMentatReportsEvent(unittest.TestCase):
NetworkModel(group = group, netname = 'UNET1', source = 'manual', network = '10.0.0.0/8')
SettingsReportingModel(group = group)
+ det1 = DetectorModel(name='org.example.kippo_honey', source='manual', credibility=0.72, hits=12)
+ det2 = DetectorModel(name='org.example.dionaea', source='manual', credibility=0.36, hits=121)
+
self.sqlstorage.session.add(group)
+ self.sqlstorage.session.add(det1)
+ self.sqlstorage.session.add(det2)
self.sqlstorage.session.commit()
self.reporting_settings = mentat.reports.utils.ReportingSettings(group)
@@ -483,6 +488,40 @@ class TestMentatReportsEvent(unittest.TestCase):
self.assertTrue(report_txt)
self.assertEqual(report_txt.split('\n')[0], 'Váženà kolegové.')
+ def test_08_filter_events_by_credibility(self):
+ """
+ Test :py:class:`mentat.reports.event.EventReporter.filter_events_by_credibility` function.
+ """
+ self.maxDiff = None
+
+ ev1 = Mock(mentat.idea.internal.Idea)
+ ev1.get_detectors = Mock(return_value=['org.example.kippo_honey'])
+ ev1.get_id = Mock(return_value='idea_event1')
+ ev2 = Mock(mentat.idea.internal.Idea)
+ ev2.get_detectors = Mock(return_value=['org.example.dionaea'])
+ ev2.get_id = Mock(return_value='idea_event2')
+ ev3 = Mock(mentat.idea.internal.Idea)
+ ev3.get_detectors = Mock(return_value=['org.example.new_detector'])
+ ev3.get_id = Mock(return_value='idea_event3')
+
+ events = {'10.3.12.13' : [ev1, ev2], '133.13.42.13' : [ev2], '64.24.35.24' : [ev3]}
+
+ _events_aggr, blocked_cnt = self.reporter.filter_events_by_credibility(events)
+
+ self.assertEqual(blocked_cnt, 1)
+ self.assertEqual(_events_aggr, {'10.3.12.13' : [ev1], '64.24.35.24' : [ev3]})
+ self.reporter.logger.assert_has_calls([
+ call.debug("Discarding event with ID '%s'.", 'idea_event2'),
+ call.warning("Event with ID '%s' contains unknown detector '%s'. Assuming full credibility.", 'idea_event3', 'org.example.new_detector')
+ ])
+
+ _events_aggr, _ = self.reporter.filter_events_by_credibility({'133.13.42.13' : [ev2]})
+ self.assertFalse(_events_aggr)
+
+ detectors = {det.name : det for det in self.sqlstorage.session.query(DetectorModel).all()}
+ self.assertEqual(detectors['org.example.kippo_honey'].hits, 12)
+ self.assertEqual(detectors['org.example.dionaea'].hits, 123)
+
#---------------------------------------------------------------------------