From 8b7b6d9c81be6740a8ffbad972d7e0fc089478aa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rajmund=20Hru=C5=A1ka?= <rajmund.hruska@cesnet.cz>
Date: Wed, 6 Apr 2022 15:48:10 +0200
Subject: [PATCH] Feature: Add the API for retrieving the overall reports
dashboard.
There is an endpoint /api/reports/dashboard that provides JSON document containing overall reports statistics. The data is the same as with the /reports/dashboard view. (Redmine issue: #7578)
---
doc/sphinx/_doclib/hawat.rst | 4 +
doc/sphinx/_doclib/hawat_plugin_reports.rst | 166 +++++++++++++++++++-
lib/hawat/blueprints/reports/__init__.py | 46 ++++--
3 files changed, 199 insertions(+), 17 deletions(-)
diff --git a/doc/sphinx/_doclib/hawat.rst b/doc/sphinx/_doclib/hawat.rst
index ff19f8f5f..6f1ebc52e 100644
--- a/doc/sphinx/_doclib/hawat.rst
+++ b/doc/sphinx/_doclib/hawat.rst
@@ -135,3 +135,7 @@ After that you have two options for using that key:
* :ref:`section-hawat-plugin-events-webapi-show`
* :ref:`section-hawat-plugin-events-webapi-dashboard`
* :ref:`section-hawat-plugin-events-webapi-metadata`
+
+* :ref:`section-hawat-plugin-reports`
+
+ * :ref:`section-hawat-plugin-reports-webapi-dashboard`
diff --git a/doc/sphinx/_doclib/hawat_plugin_reports.rst b/doc/sphinx/_doclib/hawat_plugin_reports.rst
index 1db997ba4..14df1dacd 100644
--- a/doc/sphinx/_doclib/hawat_plugin_reports.rst
+++ b/doc/sphinx/_doclib/hawat_plugin_reports.rst
@@ -3,5 +3,167 @@
reports
================================================================================
-.. automodule:: hawat.blueprints.reports
- :noindex:
+This pluggable module provides access to periodical event reports.
+
+.. _section-hawat-plugin-reports-endpoints:
+
+Provided endpoints
+--------------------------------------------------------------------------------
+
+``/api/events/dashboard``
+ * *Reference:* :ref:`section-hawat-plugin-reports-webapi-dashboard`
+
+
+.. _section-hawat-plugin-reports-webapi:
+
+Web API
+--------------------------------------------------------------------------------
+
+For general information about web API please refer to section :ref:`section-hawat-webapi`.
+
+Following is a list of all currently available API endpoints. These endpoints
+provide results as JSON document instead of full HTML page.
+
+
+.. _section-hawat-plugin-reports-webapi-dashboard:
+
+API endpoint: **dashboard**
+````````````````````````````````````````````````````````````````````````````````
+
+**Relevant endpoint:**
+
+``/api/reports/dashboard``
+ * *Authentication:* login required
+ * *Authorization:* any role
+ * *Methods:* ``GET``, ``POST``
+
+The URL for web API interface is available as normal endpoint to the user of the web
+interface. This fact can be used to debug the queries interactively and then simply
+copy them to another application. One might for example start with filling in the
+search form in the ``/reports/dashboard`` endpoint. Once you are satisfied with the
+result, you can simply switch the base URL to the ``/api/reports/dashboard`` endpoint
+and you are all set.
+
+**Available query parameters**:
+
+Following parameters may be specified as standard HTTP query parameters:
+
+``dt_from``
+ * *Description:* Lower event detection time boundary
+ * *Datatype:* Datetime in the format ``YYYY-MM-DD HH:MM:SS``, for example ``2022-01-01 00:00:00``
+
+``dt_to``
+ * *Description:* Upper event detection time boundary
+ * *Datatype:* Datetime in the format ``YYYY-MM-DD HH:MM:SS``, for example ``2022-01-01 00:00:00``
+
+``groups``
+ * *Description:* List of required abuse groups owning the reports
+ * *Datatype:* ``list of strings``
+ * *Logical operation:* All given values are *ORed*
+
+
+**Response format**
+
+JSON document, that will be received as a response for the search, can contain
+following keys:
+
+``form_data``
+ * *Description:* This subkey is present in case search operation was triggered.
+ It contains a dictionary with all query parameters described above and their
+ appropriate processed values.
+ * *Datatype:* ``dictionary``
+
+``form_errors``
+ * *Description:* This subkey is present in case there were any errors in the
+ submitted search form and the search operation could not be triggered. So
+ in another words the presence of this subkey is an indication of search failure.
+ This subkey contains list of all form errors as pairs of strings: name of
+ the form field and error description. The error description is localized
+ according to the user`s preferences.
+ * *Datatype:* ``list of tuples of strings``
+ * *Example:* ``[["dt_from", "Not a valid datetime value"]]``
+
+``statistics``
+ * *Description:* This subkey is present in case search operation was triggered.
+ It contains the actual result of the search. Following subkeys can be found
+ in this dictionary:
+
+ * ``cnt_emails`` - Total number of report emails generated
+ * ``cnt_events`` - Total number of matched events
+ * ``cnt_events_all`` - Total number of reported events
+ * ``cnt_events_filtered`` - Total number of matched events that were filtered and thus not reported
+ * ``cnt_events_new`` - Total number of new events
+ * ``cnt_events_relapsed`` - Total number of relapsed events
+ * ``cnt_events_thresholded`` - Total number of thresholded events
+ * ``cnt_recurring`` - Total number of relapsed events
+ * ``cnt_reports`` - Total number of reports created
+ * ``cnt_reports_summary`` - Total number of summary reports created
+ * ``cnt_unique`` - Total number of unique events
+ * ``dt_from`` - Lower time boundary of the result dataset
+ * ``dt_to`` - Upper time boundary of the result dataset
+ * ``abuses`` - Dictionary of the abuse groups and the number of events
+ * ``analyzers`` - Dictionary of the analyzers and the number of events
+ * ``asns`` - Dictionary of the ASNs and the number of reported events
+ * ``categories`` - Dictionary of the categories and the number of reported events
+ * ``category_sets`` - Dictionary of the category sets and the number of reported events
+ * ``classes`` - Dictionary of the classes and the number of reported events
+ * ``countries`` - Dictionary of the countries and the number of reported events
+ * ``detectors`` - Dictionary of the detectors and the number of reported events
+ * ``detectorsws`` - Dictionary of the detectors and analyzers and the number of reported events
+ * ``emails`` - Dictionary of the emails and the number of reports
+ * ``ips`` - Dictionary of the IPs and the number of reported events
+ * ``severities`` - Dictionary of the severities and the number of reported events
+ * ``timeline_cfg`` - Pre-calculated optimized timeline configurations
+
+ * *Datatype:* ``list of dictionaries``
+
+``items_count``
+ * *Description:* This subkey is present in case search operation was triggered.
+ It contains the number of original datasets that have been processed to produce
+ final dataset ``statistics``.
+ * *Datatype:* ``integer``
+
+``query_params``
+ * *Description:* This subkey is always present in the response. It contains
+ processed search query parameters that the user actually explicitly specified.
+ * *Datatype:* ``dictionary``
+ * *Example:* ``{"dt_from": "", "submit": "Search"}``
+
+``time_marks``
+ * *Description:* This subkey is present in case search operation was triggered.
+ It contains list of time marks that can be used to calculate the duration of
+ various processing steps like queriing database, processing and rendering the
+ result.
+ * *Datatype:* ``list of lists``
+
+``searched``
+ * *Description:* This subkey is present in case search operation was triggered.
+ It is a simple indication of the successful search operation.
+ * *Datatype:* ``boolean`` always set to ``True``
+
+``search_widget_item_limit``
+ * *Description:* This subkey is always present in the response. It is intended
+ for internal purposes.
+ * *Datatype:* ``integer``
+
+``view_icon``
+ * *Description:* This subkey is always present in the response. It is intended
+ for internal purposes.
+ * *Datatype:* ``string``
+
+``view_title``
+ * *Description:* This subkey is always present in the response. It is intended
+ for internal purposes.
+ * *Datatype:* ``string``
+
+**Example usage with curl:**
+
+.. code-block:: shell
+
+ $ curl -X POST -d "api_key=your%AP1_k3y" "https://.../api/reports/dashboard?submit=Search"
+
+.. note::
+
+ Please be aware, that this endpoint provides raw statistical dataset. The :ref:`section-hawat-plugin-reports-features-dashboard`
+ endpoint performs some additional client-side processing with JavaScript to transform
+ datasets provided by this endpoint into format appropriate for chart and table rendering.
diff --git a/lib/hawat/blueprints/reports/__init__.py b/lib/hawat/blueprints/reports/__init__.py
index c9543a331..aff577013 100644
--- a/lib/hawat/blueprints/reports/__init__.py
+++ b/lib/hawat/blueprints/reports/__init__.py
@@ -368,16 +368,12 @@ class DataView(FileIdView):
return 'security-report-{}.{}'.format(fileid, fileext)
-class DashboardView(HTMLMixin, SQLAlchemyMixin, BaseSearchView): # pylint: disable=locally-disabled,too-many-ancestors
+class AbstractDashboardView(SQLAlchemyMixin, BaseSearchView): # pylint: disable=locally-disabled,too-many-ancestors
"""
- View responsible for presenting reporting dashboard.
+ Base class responsible for presenting reporting dashboard.
"""
authentication = True
- @classmethod
- def get_view_name(cls):
- return 'dashboard'
-
@classmethod
def get_view_icon(cls):
return 'module-{}'.format(cls.module_name)
@@ -456,6 +452,26 @@ class DashboardView(HTMLMixin, SQLAlchemyMixin, BaseSearchView): # pylint: disa
)
+class DashboardView(HTMLMixin, AbstractDashboardView): # pylint: disable=locally-disabled,too-many-ancestors
+ """
+ View responsible for presenting reporting dashboard in the form of HTML page.
+ """
+
+ @classmethod
+ def get_view_name(cls):
+ return 'dashboard'
+
+
+class APIDashboardView(AJAXMixin, AbstractDashboardView): # pylint: disable=locally-disabled,too-many-ancestors
+ """
+ View responsible for presenting reporting dashboard in the form of JSON document.
+ """
+
+ @classmethod
+ def get_view_name(cls):
+ return 'apidashboard'
+
+
class DeleteView(HTMLMixin, SQLAlchemyMixin, ItemDeleteView): # pylint: disable=locally-disabled,too-many-ancestors
"""
View for deleting existing user accounts.
@@ -628,16 +644,16 @@ def get_blueprint():
hbp = ReportsBlueprint(
BLUEPRINT_NAME,
__name__,
- template_folder = 'templates',
- url_prefix = '/{}'.format(BLUEPRINT_NAME)
+ template_folder = 'templates'
)
- hbp.register_view_class(SearchView, '/search')
- hbp.register_view_class(ShowView, '/<int:item_id>/show')
- hbp.register_view_class(UnauthShowView, '/<item_id>/unauth')
- hbp.register_view_class(DataView, '/data/<fileid>/<filetype>')
- hbp.register_view_class(DashboardView, '/dashboard')
- hbp.register_view_class(DeleteView, '/<int:item_id>/delete')
- hbp.register_view_class(FeedbackView, '/<item_id>/feedback')
+ hbp.register_view_class(SearchView, '/{}/search'.format(BLUEPRINT_NAME))
+ hbp.register_view_class(ShowView, '/{}/<int:item_id>/show'.format(BLUEPRINT_NAME))
+ hbp.register_view_class(UnauthShowView, '/{}/<item_id>/unauth'.format(BLUEPRINT_NAME))
+ hbp.register_view_class(DataView, '/{}/data/<fileid>/<filetype>'.format(BLUEPRINT_NAME))
+ hbp.register_view_class(DashboardView, '/{}/dashboard'.format(BLUEPRINT_NAME))
+ hbp.register_view_class(DeleteView, '/{}/<int:item_id>/delete'.format(BLUEPRINT_NAME))
+ hbp.register_view_class(FeedbackView, '/{}/<item_id>/feedback'.format(BLUEPRINT_NAME))
+ hbp.register_view_class(APIDashboardView, '/api/{}/dashboard'.format(BLUEPRINT_NAME))
return hbp
--
GitLab