Implemented basic filer playground for testing filtering rules.

This commit brings in basic filtering rule playground view, that enables users to simply paste IDEA event as raw JSON string and then write and immediatelly check filtering rule againsta that event. Possible improvements are better integration with filters.show, filters.create, filters.update and events.show views. (Redmine issue: #4745)

lib/hawat/blueprints/design/templates/_macros_form.html

@@ -4,10 +4,10 @@
 
 ----------------------------------------------------------------------------- #}
 
-{%- macro render_form_item_default(form_item, with_errors = True) %}
+{%- macro render_form_item_default(form_item, with_errors = True, placeholder = None) %}
                                 <div class="form-group{% if form_item.errors %}{{ ' has-error' }}{% endif %}">
                                     {{ form_item.label }}
-                                    {{ form_item(class_='form-control') }}
+                                    {{ form_item(class_='form-control', placeholder=placeholder) }}
     {%- if with_errors %}
         {%- for err in form_item.errors %}
                                     <span class="help-block form-error">{{ get_icon('form-error') }} {{ err }}</span>{%- if not loop.last %}<br>{%- endif %}

lib/hawat/blueprints/filters/__init__.py

@@ -27,9 +27,8 @@ __author__ = "Jan Mach <jan.mach@cesnet.cz>"
 __credits__ = "Pavel Kácha <pavel.kacha@cesnet.cz>, Andrea Kropáčová <andrea.kropacova@cesnet.cz>"
 
 
-import ipranges
-import pynspect.gparser
-import pynspect.traversers
+import sys
+import traceback
 
 #
 # Flask related modules.
@@ -42,20 +41,32 @@ from flask_babel import gettext, lazy_gettext
 #
 # Custom modules.
 #
+import ipranges
+import pynspect.gparser
+import pynspect.traversers
+import pynspect.filters
+from pynspect.gparser import PynspectFilterParser
+from pynspect.filters import DataObjectFilter
+
 from mentat.const import REPORTING_FILTER_BASIC
 from mentat.datatype.sqldb import FilterModel, GroupModel, ItemChangeLogModel
+from mentat.idea.internal import Idea, IDEAFilterCompiler
 
+import hawat.const
 import hawat.db
 import hawat.events
 from hawat.base import HTMLViewMixin, SQLAlchemyViewMixin, HawatItemListView,\
     HawatItemShowView, HawatItemCreateView, HawatItemCreateForView, HawatItemUpdateView,\
-    HawatItemEnableView, HawatItemDisableView, HawatItemDeleteView, HawatBlueprint
-from hawat.blueprints.filters.forms import BaseFilterForm, AdminFilterForm
+    HawatItemEnableView, HawatItemDisableView, HawatItemDeleteView, HawatRenderableView, HawatBlueprint
+from hawat.blueprints.filters.forms import BaseFilterForm, AdminFilterForm, PlaygroundFilterForm
 
 
-_PARSER = pynspect.gparser.PynspectFilterParser()
+_PARSER = PynspectFilterParser()
 _PARSER.build()
 
+_COMPILER = IDEAFilterCompiler()
+_FILTER = DataObjectFilter()
+
 
 BLUEPRINT_NAME = 'filters'
 """Name of the blueprint as module global constant."""
@@ -99,8 +110,15 @@ def to_tree(rule):
         return _PARSER.parse(rule)
     return None
 
+def tree_compile(rule_tree):
+    """
+    Compile given filtering rule tree.
+    """
+    if rule_tree:
+        return _COMPILER.compile(rule_tree)
+    return None
 
-def to_html(rule_tree):
+def tree_html(rule_tree):
     """
     Render given rule object tree to HTML formatted content.
     """
@@ -108,6 +126,12 @@ def to_html(rule_tree):
         return rule_tree.traverse(pynspect.traversers.HTMLTreeTraverser())
     return None
 
+def tree_check(rule_tree, data):
+    """
+    Check given event against given rule tree.
+    """
+    return _FILTER.filter(rule_tree, data)
+
 
 class ListView(HTMLViewMixin, SQLAlchemyViewMixin, HawatItemListView):
     """
@@ -147,6 +171,12 @@ class ListView(HTMLViewMixin, SQLAlchemyViewMixin, HawatItemListView):
             endpoint = 'filters.create',
             resptitle = True
         )
+        action_menu.add_entry(
+            'endpoint',
+            'playground',
+            endpoint = 'filters.playground',
+            resptitle = True
+        )
         return action_menu
 
     @classmethod
@@ -256,6 +286,11 @@ class ShowView(HTMLViewMixin, SQLAlchemyViewMixin, HawatItemShowView):
             'delete',
             endpoint = 'filters.delete',
         )
+        action_menu.add_entry(
+            'endpoint',
+            'playground',
+            endpoint = 'filters.playground',
+        )
         return action_menu
 
     def do_before_response(self, **kwargs):
@@ -264,9 +299,12 @@ class ShowView(HTMLViewMixin, SQLAlchemyViewMixin, HawatItemShowView):
         """
         item = self.response_context['item']
         filter_tree = to_tree(item.filter)
+        filter_compiled = tree_compile(filter_tree)
         self.response_context.update(
             filter_tree = filter_tree,
-            filter_preview = to_html(filter_tree)
+            filter_compiled = filter_compiled,
+            filter_preview = tree_html(filter_tree),
+            filter_compiled_preview = tree_html(filter_compiled)
         )
 
         if self.can_access_endpoint('filters.update', item) and self.has_endpoint('changelogs.search'):
@@ -372,7 +410,7 @@ class CreateView(HTMLViewMixin, SQLAlchemyViewMixin, HawatItemCreateView):  # py
             filter_tree = to_tree(item.filter)
             self.response_context.update(
                 filter_tree = filter_tree,
-                filter_preview = to_html(filter_tree)
+                filter_preview = tree_html(filter_tree)
             )
 
 
@@ -503,7 +541,7 @@ class CreateForView(HTMLViewMixin, SQLAlchemyViewMixin, HawatItemCreateForView):
             filter_tree = to_tree(item.filter)
             self.response_context.update(
                 filter_tree = filter_tree,
-                filter_preview = to_html(filter_tree)
+                filter_preview = tree_html(filter_tree)
             )
 
 
@@ -607,7 +645,7 @@ class UpdateView(HTMLViewMixin, SQLAlchemyViewMixin, HawatItemUpdateView):  # py
         filter_tree = to_tree(item.filter)
         self.response_context.update(
             filter_tree = filter_tree,
-            filter_preview = to_html(filter_tree)
+            filter_preview = tree_html(filter_tree)
         )
 
 
@@ -785,6 +823,129 @@ class DeleteView(HTMLViewMixin, SQLAlchemyViewMixin, HawatItemDeleteView):  # py
         return gettext('Canceled deleting reporting filter <strong>%(item_id)s</strong> for group <strong>%(parent_id)s</strong>.', item_id = str(kwargs['item']), parent_id = str(kwargs['item'].group))
 
 
+class PlaygroundView(HTMLViewMixin, HawatRenderableView):
+    """
+    Reporting filter playground view.
+    """
+    methods = ['GET','POST']
+
+    authentication = True
+
+    @classmethod
+    def get_view_name(cls):
+        """
+        *Interface implementation* of :py:func:`hawat.base.HawatBaseView.get_view_name`.
+        """
+        return 'playground'
+
+    @classmethod
+    def get_menu_title(cls, item = None):
+        """
+        *Interface implementation* of :py:func:`hawat.base.HawatBaseView.get_menu_title`.
+        """
+        return lazy_gettext('Filter playground')
+
+    @classmethod
+    def get_menu_icon(cls):
+        """
+        *Interface implementation* of :py:func:`hawat.base.HawatBaseView.get_menu_icon`.
+        """
+        return 'playground'
+
+    @classmethod
+    def get_menu_legend(cls, item = None):
+        """
+        *Interface implementation* of :py:func:`hawat.base.HawatBaseView.get_menu_title`.
+        """
+        return lazy_gettext('Reporting filter playground')
+
+    @classmethod
+    def get_view_title(cls, item = None):
+        """
+        *Interface implementation* of :py:func:`hawat.base.HawatBaseView.get_view_title`.
+        """
+        return lazy_gettext('Reporting filter rule playground')
+
+    @classmethod
+    def get_breadcrumbs_menu(cls):
+        """
+        Get breadcrumbs menu.
+        """
+        breadcrumbs_menu = hawat.menu.HawatMenu()
+        breadcrumbs_menu.add_entry(
+            'link',
+            'index',
+            link = flask.url_for('index'),
+            title = gettext('Home')
+        )
+        breadcrumbs_menu.add_entry(
+            'endpoint',
+            'list',
+            endpoint = '{}.{}'.format(cls.module_name, 'list')
+        )
+        breadcrumbs_menu.add_entry(
+            'endpoint',
+            'playground',
+            endpoint = cls.get_view_endpoint()
+        )
+        return breadcrumbs_menu
+
+
+    #---------------------------------------------------------------------------
+
+
+    def dispatch_request(self):
+        """
+        Mandatory interface required by the :py:func:`flask.views.View.dispatch_request`.
+        Will be called by the *Flask* framework to service the request.
+        """
+        form = PlaygroundFilterForm()
+
+        if form.validate_on_submit():
+            form_data = form.data
+
+            try:
+                event                   = Idea.from_json(form.event.data)
+                filter_tree             = to_tree(form.filter.data)
+                filter_preview          = tree_html(filter_tree)
+                filter_compiled         = tree_compile(filter_tree)
+                filter_compiled_preview = tree_html(filter_compiled)
+                filter_result           = tree_check(filter_compiled, event)
+
+                self.response_context.update(
+                    form_data               = form_data,
+                    event                   = event,
+                    filter_tree             = filter_tree,
+                    filter_preview          = filter_preview,
+                    filter_compiled         = filter_compiled,
+                    filter_compiled_preview = filter_compiled_preview,
+                    filter_result           = filter_result,
+                    flag_filtered           = True
+                )
+
+            except Exception as err:  # pylint: disable=locally-disabled,broad-except
+                self.flash(
+                    flask.Markup(gettext(
+                        '<strong>%(error)s</strong>.',
+                        error = str(err)
+                    )),
+                    hawat.const.HAWAT_FLASH_FAILURE
+                )
+
+                tbexc = traceback.TracebackException(*sys.exc_info())
+                self.response_context.update(
+                    filter_exception    = err,
+                    filter_exception_tb = ''.join(tbexc.format())
+                )
+
+
+        self.response_context.update(
+            form_url = flask.url_for(self.get_view_endpoint()),
+            form     = form,
+        )
+        return self.generate_response()
+
+
 #-------------------------------------------------------------------------------
 
 
@@ -842,5 +1003,6 @@ def get_blueprint():
     hbp.register_view_class(EnableView, '/<int:item_id>/enable')
     hbp.register_view_class(DisableView, '/<int:item_id>/disable')
     hbp.register_view_class(DeleteView, '/<int:item_id>/delete')
+    hbp.register_view_class(PlaygroundView, '/playground')
 
     return hbp

lib/hawat/blueprints/filters/forms.py

@@ -18,13 +18,15 @@ __credits__ = "Pavel Kácha <pavel.kacha@cesnet.cz>, Andrea Kropáčová <andrea
 
 
 import pynspect.gparser
+
 import wtforms
 from wtforms.ext.sqlalchemy.fields import QuerySelectField
 
 #
 # Flask related modules.
 #
-from flask_babel import lazy_gettext
+import flask_wtf
+from flask_babel import gettext, lazy_gettext
 
 #
 # Custom modules.
@@ -33,6 +35,7 @@ import hawat.db
 import hawat.forms
 from mentat.datatype.sqldb import GroupModel
 from mentat.const import REPORTING_FILTER_BASIC, REPORTING_FILTER_ADVANCED
+from mentat.idea.internal import Idea
 
 
 def get_available_groups():
@@ -42,13 +45,38 @@ def get_available_groups():
     return hawat.db.db_query(GroupModel).order_by(GroupModel.name).all()
 
 
-def check_filter(form, field):
+def check_filter(form, field):  # pylint: disable=locally-disabled,unused-argument
     """
     Callback for validating Pynspect filter.
     """
     psr = pynspect.gparser.PynspectFilterParser()
     psr.build()
-    rule_tree = psr.parse(field.data)
+    try:
+        psr.parse(field.data)
+    except Exception as err:
+        raise wtforms.validators.ValidationError(
+            gettext(
+                'Filtering rule parse error: "%(error)s".',
+                error = str(err)
+            )
+        )
+
+def check_event(form, field):  # pylint: disable=locally-disabled,unused-argument
+    """
+    Callback for validating IDEA event JSON.
+    """
+    try:
+        Idea.from_json(field.data)
+    except Exception as err:
+        raise wtforms.validators.ValidationError(
+            gettext(
+                'Event JSON parse error: "%(error)s".',
+                error = str(err)
+            )
+        )
+
+
+#-------------------------------------------------------------------------------
 
 
 class BaseFilterForm(hawat.forms.BaseItemForm):
@@ -161,3 +189,25 @@ class AdminFilterForm(BaseFilterForm):
         query_factory = get_available_groups,
         allow_blank = False
     )
+
+class PlaygroundFilterForm(flask_wtf.FlaskForm):
+    """
+    Class representing IP geolocation search form.
+    """
+    filter = wtforms.TextAreaField(
+        lazy_gettext('Filtering rule:'),
+        validators = [
+            wtforms.validators.DataRequired(),
+            check_filter
+        ]
+    )
+    event = wtforms.TextAreaField(
+        lazy_gettext('IDEA event:'),
+        validators = [
+            wtforms.validators.DataRequired(),
+            check_event
+        ]
+    )
+    submit = wtforms.SubmitField(
+        lazy_gettext('Check')
+    )

lib/hawat/blueprints/filters/templates/filters/playground.html

+{%- extends "_layout.html" %}
+
+{%- block content %}
+
+            <div class="row">
+                <div class="col-lg-12">
+                    {{ macros_page.render_breadcrumbs(item) }}
+
+                    <div class="jumbotron" style="margin-top: 1em;">
+                        <h2>{{ hawat_current_view.get_view_title() }}</h2>
+                        <hr>
+                        <form method="POST" class="form-horizontal" id="form-events-simple" action="{{ form_url }}">
+                            {{ macros_form.render_form_item_default(form.filter, placeholder = _('Write your filtering rule here...')) }}
+                            {{ macros_form.render_form_item_default(form.event, placeholder = _('Paste your IDEA event as raw JSON string...')) }}
+
+                            <hr>
+
+                            {{ macros_form.render_form_errors(form.csrf_token.errors) }}
+                            {{ form.csrf_token }}
+
+                            <div class="btn-toolbar" role="toolbar" aria-label="{{ _('Form submission buttons') }}">
+                                <div class="btn-group" role="group">
+                                    {{ form.submit(class_='btn btn-primary') }}
+                                    <a role="button" class="btn btn-default" href="{{ url_for(request.endpoint) }}">
+                                        {{ _('Clear') }}
+                                    </a>
+                                </div>
+                            </div>
+                        </form>
+                    </div> <!-- .jumbotron -->
+
+                </div> <!-- .col-lg-12 -->
+            </div>  <!-- .row -->
+
+    {%- if flag_filtered %}
+
+        {%- if filter_result %}
+            {%- call macros_site.render_alert('success', False) %}
+            {{ _('Given event matches given filtering rule.') }}
+            {%- endcall %}
+        {%- else %}
+            {%- call macros_site.render_alert('info', False) %}
+            {{ _('Given event does NOT match given filtering rule.') }}
+            {%- endcall %}
+        {%- endif %}
+
+            <div class="panel-group" id="result-accordion" role="tablist">
+
+                <div class="panel panel-default">
+                    <div class="panel-heading" role="tab">
+                        <h3 class="panel-title">
+                            <a role="button" data-toggle="collapse" data-parent="#result-accordion" href="#filtering-result">
+                                {{ _('Filtering result') }}
+                            </a>
+                        </h3>
+                    </div>
+                    <div id="filtering-result" class="panel-collapse collapse in" role="tabpanel">
+                        <div class="panel-body">
+<pre>
+{{ filter_result | pprint }}
+</pre>
+                        </div>
+                    </div>
+                </div>
+
+                <div class="panel panel-default">
+                    <div class="panel-heading" role="tab">
+                        <h3 class="panel-title">
+                            <a class="collapsed" role="button" data-toggle="collapse" data-parent="#result-accordion" href="#filtering-filter">
+                                {{ _('Filter tree') }}
+                            </a>
+                        </h3>
+                    </div>
+                    <div id="filtering-filter" class="panel-collapse collapse" role="tabpanel">
+                        <div class="panel-body">
+                            <div class="rule-tree">
+                                {{ filter_compiled_preview | safe }}
+                            </div>
+                            <hr>
+                            <h4>{{ _('Textual representation') }}:</h4>
+<pre>
+{{ filter_compiled.__str__() }}
+</pre>
+                        </div>
+                    </div>
+                </div>
+
+                <div class="panel panel-default">
+                    <div class="panel-heading" role="tab">
+                        <h3 class="panel-title">
+                            <a class="collapsed" role="button" data-toggle="collapse" data-parent="#result-accordion" href="#filtering-event">
+                                {{ _('Filtered event') }}
+                            </a>
+                        </h3>
+                    </div>
+                    <div id="filtering-event" class="panel-collapse collapse" role="tabpanel">
+                        <div class="panel-body">
+<pre>
+{{ event.to_json(indent = 4, sort_keys = True) }}
+</pre>
+                        </div>
+                    </div>
+                </div>
+
+            </div>  <!-- #result-accordion -->
+    {%- endif %}
+
+    {%- if permission_can('developer') %}
+            <hr>
+        {%- if filter_exception %}
+            {{ macros_site.render_raw_var('filter_exception',    filter_exception) }}
+            {{ macros_site.render_raw_var('filter_exception_st', filter_exception_st) }}
+        {%- endif %}
+            {{ macros_site.render_raw_var('form_data',               form_data) }}
+            {{ macros_site.render_raw_var('event',                   event) }}
+            {{ macros_site.render_raw_var('filter_tree',             filter_tree) }}
+            {{ macros_site.render_raw_var('filter_preview',          filter_preview) }}
+            {{ macros_site.render_raw_var('filter_compiled',         filter_compiled) }}
+            {{ macros_site.render_raw_var('filter_compiled_preview', filter_compiled_preview) }}
+            {{ macros_site.render_raw_var('filter_result',           filter_result) }}
+    {%- endif %}
+
+{%- endblock content %}

lib/hawat/blueprints/filters/templates/filters/show.html

@@ -118,6 +118,11 @@
                             <div class="rule-tree">
                                 {{ filter_preview | safe }}
                             </div>
+                            <hr>
+                            <h4>{{ _('Textual representation') }}:</h4>
+<pre>
+{{ filter_tree.__str__() }}
+</pre>
                         </div>
                         {%- endif %}
 
@@ -143,5 +148,12 @@
 
                 </div><!-- /.col-lg-12 -->
             </div><!-- /.row -->
+    {%- if permission_can('developer') %}
+            <hr>
+            {{ macros_site.render_raw_var('filter_tree',             filter_tree) }}
+            {{ macros_site.render_raw_var('filter_compiled',         filter_compiled) }}
+            {{ macros_site.render_raw_var('filter_preview',          filter_preview) }}
+            {{ macros_site.render_raw_var('filter_compiled_preview', filter_compiled_preview) }}
+    {%- endif %}
 
 {%- endblock content %}

lib/hawat/const.py

@@ -278,7 +278,8 @@ FA_ICONS = {
     'collapse':    '<i class="fas fa-fw fa-angle-down" aria-hidden="true"></i>',
     'form-error':  '<i class="fas fa-fw fa-exclamation-triangle" aria-hidden="true"></i>',
     'table':       '<i class="fas fa-fw fa-table"></i>',
-    'quicksearch': '<i class="fab fa-fw fa-searchengin"></i>'
+    'quicksearch': '<i class="fab fa-fw fa-searchengin"></i>',
+    'playground':  '<i class="fas fa-fw fa-gamepad"></i>'
 }
 """
 Predefined list of selected `font-awesome <http://fontawesome.io/icons/>`__ icons