Initial implementation of DNS resolver service.

This new feature will be used to enhance various event views. (Redmine issue: #5067)

conf/core/services.json.conf

@@ -4,6 +4,14 @@
     #
     "__core__services": {
 
+        #
+        # DNS service settings.
+        #
+        "dns": {
+            "timeout": 1,
+            "lifetime": 3
+        },
+
         #
         # GeoIP service settings.
         #

conf/mentat-hawat.py.conf

@@ -26,6 +26,7 @@ ENABLED_BLUEPRINTS = [
     'hawat.blueprints.reports',
     'hawat.blueprints.events',
     'hawat.blueprints.timeline',
+    'hawat.blueprints.dnsr',
     'hawat.blueprints.geoip',
     #'hawat.blueprints.nerd',
     'hawat.blueprints.whois',

conf/requirements-latest.pip

@@ -20,6 +20,7 @@ flask-script
 flask-sqlalchemy
 flask-debugtoolbar
 flask-jsglue
+dnspython
 geoip2
 requests
 rrdtool

conf/requirements.pip

@@ -20,6 +20,7 @@ flask-script==2.0.6
 flask-sqlalchemy==2.3.2
 flask-debugtoolbar==0.10.1
 flask-jsglue==0.3.1
+dnspython==1.16.0
 geoip2==2.9.0
 requests==2.21.0
 rrdtool==0.1.14

lib/hawat/base.py

@@ -524,6 +524,11 @@ class AJAXMixin:
         :return: Possibly updated response context.
         :rtype: dict
         """
+        for key in ('search_form', 'item_form'):
+            try:
+                del response_context[key]
+            except KeyError:
+                pass
         return response_context
 
     @staticmethod

lib/hawat/blueprints/dnsr/__init__.py

+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#-------------------------------------------------------------------------------
+# This file is part of Mentat system (https://mentat.cesnet.cz/).
+#
+# Copyright (C) since 2011 CESNET, z.s.p.o (http://www.ces.net/)
+# Use of this source is governed by the MIT license, see LICENSE file.
+#-------------------------------------------------------------------------------
+
+
+"""
+Description
+-----------
+
+This pluggable module provides access to external DNS service. It is built upon
+custom :py:mod:`mentat.services.dnsr` module.
+
+
+Provided endpoints
+------------------
+
+``/dnsr/search``
+    Page providing search form and displaying search results.
+
+    * *Authentication:* login required
+    * *Methods:* ``GET``
+"""
+
+
+__author__ = "Jan Mach <jan.mach@cesnet.cz>"
+__credits__ = "Pavel Kácha <pavel.kacha@cesnet.cz>, Andrea Kropáčová <andrea.kropacova@cesnet.cz>"
+
+#
+# Flask related modules.
+#
+import flask
+from flask_babel import lazy_gettext
+
+#
+# Custom modules.
+#
+import mentat.services.dnsr
+from mentat.const import tr_
+
+import hawat.const
+import hawat.db
+import hawat.acl
+from hawat.base import HTMLMixin, AJAXMixin, RenderableView, HawatBlueprint, URLParamsBuilder
+from hawat.blueprints.dnsr.forms import DnsrSearchForm
+
+
+BLUEPRINT_NAME = 'dnsr'
+"""Name of the blueprint as module global constant."""
+
+
+class AbstractSearchView(RenderableView):
+    """
+    Application view providing search form for internal IP geolocation service
+    and appropriate result page.
+
+    The geolocation is implemented using :py:mod:`mentat.services.dnsr` module.
+    """
+    authentication = True
+
+    authorization = [hawat.acl.PERMISSION_ANY]
+
+    @classmethod
+    def get_menu_title(cls, item = None):
+        """*Implementation* of :py:func:`hawat.base.BaseView.get_menu_title`."""
+        return lazy_gettext('Search DNS')
+
+    @classmethod
+    def get_view_title(cls, item = None):
+        """*Implementation* of :py:func:`hawat.base.BaseView.get_view_title`."""
+        return lazy_gettext('Search DNS')
+
+    #---------------------------------------------------------------------------
+
+    def dispatch_request(self):
+        """
+        Mandatory interface required by the :py:func:`flask.views.View.dispatch_request`.
+        Will be called by the *Flask* framework to service the request.
+        """
+        form = DnsrSearchForm(flask.request.args, meta = {'csrf': False})
+
+        if hawat.const.HAWAT_FORM_ACTION_SUBMIT in flask.request.args:
+            if form.validate():
+                form_data = form.data
+                dnsr_service = mentat.services.dnsr.service()
+                self.response_context.update(
+                    search_item   = form.search.data,
+                    search_result = dnsr_service.lookup(form.search.data),
+                    form_data     = form_data
+                )
+
+        self.response_context.update(
+            search_form  = form,
+            request_args = flask.request.args,
+        )
+        return self.generate_response()
+
+
+class SearchView(HTMLMixin, AbstractSearchView):  # pylint: disable=locally-disabled,too-many-ancestors
+    """
+    View responsible for querying `IDEA <https://idea.cesnet.cz/en/index>`__
+    event database and presenting the results in the form of HTML page.
+    """
+    methods = ['GET']
+
+    @classmethod
+    def get_view_name(cls):
+        """*Implementation* of :py:func:`hawat.base.BaseView.get_view_name`."""
+        return 'search'
+
+
+class APISearchView(AJAXMixin, AbstractSearchView):  # pylint: disable=locally-disabled,too-many-ancestors
+    """
+    View responsible for querying `IDEA <https://idea.cesnet.cz/en/index>`__
+    event database and presenting the results in the form of JSON document.
+    """
+    methods = ['GET','POST']
+
+    @classmethod
+    def get_view_name(cls):
+        """*Implementation* of :py:func:`hawat.base.BaseView.get_view_name`."""
+        return 'apisearch'
+
+
+#-------------------------------------------------------------------------------
+
+
+class DnsrBlueprint(HawatBlueprint):
+    """
+    Hawat pluggable module - DNS database service.
+    """
+
+    @classmethod
+    def get_module_title(cls):
+        """*Implementation* of :py:func:`hawat.base.HawatBlueprint.get_module_title`."""
+        return lazy_gettext('External DNS pluggable module')
+
+    def register_app(self, app):
+        """
+        *Callback method*. Will be called from :py:func:`hawat.base.HawatApp.register_blueprint`
+        method and can be used to customize the Flask application object. Possible
+        use cases:
+
+        * application menu customization
+
+        :param hawat.base.HawatApp app: Flask application to be customize.
+        """
+
+        mentat.services.dnsr.init(app.mconfig)
+
+        app.menu_main.add_entry(
+            'view',
+            'more.{}'.format(BLUEPRINT_NAME),
+            position = 30,
+            view = SearchView
+        )
+
+        # Register context actions provided by this module.
+        app.set_csag(
+            hawat.const.HAWAT_CSAG_ADDRESS,
+            tr_('Search for address <strong>%(name)s</strong> in DNS'),
+            SearchView,
+            URLParamsBuilder({'submit': tr_('Search')}).add_rule('search')
+        )
+
+
+#-------------------------------------------------------------------------------
+
+
+def get_blueprint():
+    """
+    Mandatory interface and factory function. This function must return a valid
+    instance of :py:class:`hawat.base.HawatBlueprint` or :py:class:`flask.Blueprint`.
+    """
+
+    hbp = DnsrBlueprint(
+        BLUEPRINT_NAME,
+        __name__,
+        template_folder = 'templates'
+    )
+
+    hbp.register_view_class(SearchView,    '/{}/search'.format(BLUEPRINT_NAME))
+    hbp.register_view_class(APISearchView, '/api/{}/search'.format(BLUEPRINT_NAME))
+
+    return hbp

lib/hawat/blueprints/dnsr/forms.py

+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#-------------------------------------------------------------------------------
+# This file is part of Mentat system (https://mentat.cesnet.cz/).
+#
+# Copyright (C) since 2011 CESNET, z.s.p.o (http://www.ces.net/)
+# Use of this source is governed by the MIT license, see LICENSE file.
+#-------------------------------------------------------------------------------
+
+
+"""
+This module contains custom external DNS database search form for Hawat.
+"""
+
+
+__author__ = "Jan Mach <jan.mach@cesnet.cz>"
+__credits__ = "Pavel Kácha <pavel.kacha@cesnet.cz>, Andrea Kropáčová <andrea.kropacova@cesnet.cz>"
+
+
+import wtforms
+import flask_wtf
+from flask_babel import lazy_gettext
+
+
+class DnsrSearchForm(flask_wtf.FlaskForm):
+    """
+    Class representing DNS search form.
+    """
+    search = wtforms.StringField(
+        lazy_gettext('Search DNS:'),
+        validators = [
+            wtforms.validators.DataRequired()
+        ]
+    )
+    submit = wtforms.SubmitField(
+        lazy_gettext('Search')
+    )

lib/hawat/blueprints/dnsr/templates/dnsr/search.html

+{%- extends "_layout.html" %}
+
+{%- block content %}
+
+            <div class="row">
+                <div class="col-lg-12">
+
+                    <div class="jumbotron" style="margin-top: 1em;">
+                        <h2>{{ hawat_current_view.get_view_title() }}</h2>
+                        <form method="GET" class="form-inline" action="{{ url_for('dnsr.search') }}">
+                            <div class="form-group{% if search_form.search.errors %}{{ ' has-error' }}{% endif %}">
+                                {{ search_form.search.label(class_='sr-only') }}
+                                <div class="input-group">
+                                    <div data-toggle="tooltip" class="input-group-addon" title="{{ _('Search DNS for:') }}">{{ get_icon('action-search') }}</div>
+                                    {{ search_form.search(class_='form-control', placeholder=_('Hostname, IPv4 or IPv6 address'), size='50') }}
+                                </div>
+
+                                <div class="btn-group" role="group">
+                                    {{ search_form.submit(class_='btn btn-primary') }}
+                                    <a role="button" class="btn btn-default" href="{{ url_for('dnsr.search') }}">
+                                        {{ _('Clear') }}
+                                    </a>
+                                </div>
+                            </div>
+                            {{ macros_form.render_form_errors(search_form.search.errors) }}
+                        </form>
+                    </div> <!-- .jumbotron -->
+
+                </div> <!-- .col-lg-12 -->
+            </div>  <!-- .row -->
+
+    {%- if search_item %}
+            <div class="row">
+                <div class="col-lg-12">
+        {%- if search_result %}
+
+                    <div class="panel panel-default">
+                        <div class="panel-heading">
+                            <h3 class="panel-title">{{ _('DNS lookup raw result') }}</h3>
+                        </div>
+                        <div class="panel-body">
+<pre>
+{{ search_result | pprint }}
+</pre>
+                        </div>
+                    </div>
+        {%- else %}
+
+                    {%- call macros_site.render_alert('info', False) %}
+                    {{ _('There are no records for <strong>%(item_id)s</strong> in <em>DNS</em>.', item_id = search_item) | safe }}
+                    {%- endcall %}
+
+        {%- endif %}
+
+                </div> <!-- .col-lg-12 -->
+            </div>  <!-- .row -->
+
+        {%- if permission_can('developer') %}
+            <hr>
+            {{ macros_site.render_raw_var('form_data', form_data) }}
+            {{ macros_site.render_raw_var('search_result', search_result) }}
+            {{ macros_site.render_raw_var('request_args', request_args) }}
+            {{ macros_site.render_raw_var('query_params', query_params) }}
+        {%- endif %}
+
+    {%- endif %}
+
+{%- endblock content %}

lib/hawat/config.py

@@ -135,6 +135,7 @@ class Config:  # pylint: disable=locally-disabled,too-few-public-methods
         'hawat.blueprints.reports',
         'hawat.blueprints.events',
         'hawat.blueprints.timeline',
+        'hawat.blueprints.dnsr',
         'hawat.blueprints.geoip',
         #'hawat.blueprints.nerd',
         'hawat.blueprints.whois',
@@ -267,6 +268,7 @@ class DevelopmentConfig(Config):  # pylint: disable=locally-disabled,too-few-pub
         'hawat.blueprints.reports',
         'hawat.blueprints.events',
         'hawat.blueprints.timeline',
+        'hawat.blueprints.dnsr',
         'hawat.blueprints.geoip',
         #'hawat.blueprints.nerd',
         'hawat.blueprints.whois',

lib/hawat/const.py

@@ -179,6 +179,7 @@ FA_ICONS = {
     'module-dbstatus':           '<i class="fas fa-fw fa-database"></i>',
     'module-design':             '<i class="fas fa-fw fa-palette"></i>',
     'module-devtools':           '<i class="fas fa-fw fa-bug"></i>',
+    'module-dnsr':               '<i class="fas fa-fw fa-directions"></i>',
     'module-events':             '<i class="fas fa-fw fa-bell"></i>',
     'module-filters':            '<i class="fas fa-fw fa-filter"></i>',
     'module-geoip':              '<i class="fas fa-fw fa-map-marked-alt"></i>',

lib/mentat/__init__.py

@@ -20,4 +20,4 @@ open-source project.
 
 __author__  = "Jan Mach <jan.mach@cesnet.cz>"
 __credits__ = "Pavel Kácha <pavel.kacha@cesnet.cz>, Andrea Kropáčová <andrea.kropacova@cesnet.cz>"
-__version__ = "2.4.4"
+__version__ = "2.4.5"

lib/mentat/const.py

@@ -85,6 +85,8 @@ CKEY_CORE_DATABASE_SCHEMA = 'schema'
 
 CKEY_CORE_SERVICES = '__core__services'
 """Name of the configuration key for ``core services`` configurations."""
+CKEY_CORE_SERVICES_DNS = 'dns'
+"""Name of the configuration subkey key for ``dns`` configuration in ``core services`` configurations."""
 CKEY_CORE_SERVICES_GEOIP = 'geoip'
 """Name of the configuration subkey key for ``geoip`` configuration in ``core services`` configurations."""
 CKEY_CORE_SERVICES_NERD = 'nerd'

lib/mentat/services/dnsr.py

+# -*- coding: utf-8 -*-
+#-------------------------------------------------------------------------------
+# This file is part of Mentat system (https://mentat.cesnet.cz/).
+#
+# Copyright (C) since 2011 CESNET, z.s.p.o (http://www.ces.net/)
+# Use of this source is governed by the MIT license, see LICENSE file.
+#-------------------------------------------------------------------------------
+
+
+"""
+Implementation of internal **DNS** service library.
+"""
+
+
+__author__ = "Jan Mach <jan.mach@cesnet.cz>"
+__credits__ = "Pavel Kácha <pavel.kacha@cesnet.cz>, Andrea Kropáčová <andrea.kropacova@cesnet.cz>"
+
+
+import copy
+import dns.resolver
+import dns.exception
+import ipranges
+
+from mentat.const import CKEY_CORE_SERVICES, CKEY_CORE_SERVICES_DNS
+
+
+_MANAGER = None
+
+
+class DnsService:
+    """
+    Implementation of internal **DNS** database service.
+    """
+
+    def __init__(self, timeout = 1, lifetime = 3):
+        """
+        Initialize geolocation service with paths to desired database files.
+        """
+        self.resolver = None
+        self.timeout  = timeout
+        self.lifetime = lifetime
+
+    def setup(self):
+        """
+        Setup internal DNS service resolver.
+        """
+        self.resolver = dns.resolver.Resolver()
+        self.resolver.timeout  = self.timeout
+        self.resolver.lifetime = self.lifetime
+
+    def status(self):
+        """
+        Display status of internal geolocation readers.
+        """
+        return {
+            'timeout':  self.timeout,
+            'lifetime': self.lifetime
+        }
+
+    def lookup_ip(self, ipaddr):
+        """
+        Lookup given IP address in DNS.
+        """
+        result = []
+        revipaddr = dns.reversename.from_address(ipaddr) # create .in-addr.arpa address
+        try:
+            answer = self.resolver.query(revipaddr, "PTR")
+            for res in answer.rrset:
+                res = str(res)
+                if res[-1] == '.':
+                    res = res[:-1] # trim trailing '.'
+                result.append(res)
+        except dns.exception.Timeout as exc:
+            raise RuntimeError("DNS query for {} timed out".format(ipaddr))
+        except dns.exception.DNSException as exc:
+            pass
+        return result
+
+    def lookup_hostname(self, hname):
+        """
+        Lookup given hostname in DNS.
+        """
+        result = []
+        try:
+            for qtype in ('A', 'AAAA'):
+                answer = self.resolver.query(hname, qtype)
+                for res in answer.rrset:
+                    res = str(res)
+                    result.append(res)
+        except dns.exception.Timeout as exc:
+            raise RuntimeError("DNS query for {} timed out".format(hname))
+        except dns.exception.DNSException as exc:
+            pass
+        return result
+
+    def lookup(self, thing):
+        """
+        Lookup given object in DNS.
+        """
+        for tconv in ipranges.IP4, ipranges.IP6:
+            try:
+                tconv(thing)
+                return self.lookup_ip(thing)
+            except ValueError:
+                pass
+        return self.lookup_hostname(thing)
+
+
+class DnsServiceManager:
+    """
+    Class representing a custom DnsServiceManager capable of understanding and
+    parsing Mentat system core configurations and enabling easy way of unified
+    bootstrapping of :py:class:`mentat.services.dns.DnsService` service.
+    """
+
+    def __init__(self, core_config, updates = None):
+        """
+        Initialize DnsServiceManager object with full core configuration tree structure.
+
+        :param dict core_config: Mentat core configuration structure.
+        :param dict updates: Optional configuration updates (same structure as ``core_config``).
+        """
+        self._dnsconfig = {}
+
+        self._service = None
+
+        self._configure_dns(core_config, updates)
+
+    def _configure_dns(self, core_config, updates):
+        """
+        Internal sub-initialization helper: Configure database structure parameters
+        and optionally merge them with additional updates.
+
+        :param dict core_config: Mentat core configuration structure.
+        :param dict updates: Optional configuration updates (same structure as ``core_config``).
+        """
+        self._dnsconfig = copy.deepcopy(core_config[CKEY_CORE_SERVICES][CKEY_CORE_SERVICES_DNS])
+
+        if updates and CKEY_CORE_SERVICES in updates and CKEY_CORE_SERVICES_DNS in updates[CKEY_CORE_SERVICES]:
+            self._dnsconfig.update(
+                updates[CKEY_CORE_SERVICES][CKEY_CORE_SERVICES_DNS]
+            )
+
+    def service(self):
+        """
+        Return handle to DNS service according to internal configurations.
+
+        :return: Reference to DNS service object.
+        :rtype: mentat.services.dnsr.DnsService
+        """
+        if not self._service:
+            self._service = DnsService(**self._dnsconfig)
+            self._service.setup()
+        return self._service
+
+
+#-------------------------------------------------------------------------------
+
+
+def init(core_config, updates = None):
+    """
+    (Re-)Initialize :py:class:`DnsServiceManager` instance at module level and
+    store the refence within module.
+    """
+    global _MANAGER  # pylint: disable=locally-disabled,global-statement
+    _MANAGER = DnsServiceManager(core_config, updates)
+
+
+def manager():
+    """
+    Obtain reference to :py:class:`DnsServiceManager` instance stored at module
+    level.
+    """
+    return _MANAGER
+
+
+def service():
+    """
+    Obtain reference to :py:class:`DnsService` instance from module level manager.
+    """
+    return manager().service()

lib/mentat/services/test_dnsr.py

+# -*- coding: utf-8 -*-
+#-------------------------------------------------------------------------------
+# This file is part of Mentat system (https://mentat.cesnet.cz/).
+#
+# Copyright (C) since 2011 CESNET, z.s.p.o (http://www.ces.net/)
+# Use of this source is governed by the MIT license, see LICENSE file.
+#-------------------------------------------------------------------------------
+
+"""
+Unit test module for testing the :py:mod:`mentat.services.dnsr` module.
+"""
+
+
+__author__  = "Jan Mach <jan.mach@cesnet.cz>"
+__credits__ = "Pavel Kácha <pavel.kacha@cesnet.cz>, Andrea Kropáčová <andrea.kropacova@cesnet.cz>"
+
+
+import os
+import unittest
+import subprocess
+
+#
+# Custom libraries
+#
+import mentat.services.dnsr
+
+
+#-------------------------------------------------------------------------------
+# NOTE: Sorry for the long lines in this file. They are deliberate, because the
+# assertion permutations are (IMHO) more readable this way.
+#-------------------------------------------------------------------------------
+
+
+class TestMentatDns(unittest.TestCase):
+    """
+    Unit test class for testing the :py:mod:`mentat.services.dnsr` module.
+    """
+
+    #
+    # Turn on more verbose output, which includes print-out of constructed
+    # objects. This will really clutter your console, usable only for test
+    # debugging.
+    #
+    verbose = True
+
+    rdns_test_list = [
+        [
+            '195.113.144.233',
+            ['ns.ces.net']
+        ],
+        [
+            '195.113.144.230',
+            ['www.cesnet.cz']
+        ]
+    ]
+
+    dns_test_list = [
+        [
+            'ns.ces.net',
+            ['195.113.144.233', '2001:718:1:101::3']
+        ],
+        [
+            'www.cesnet.cz',
+            ['195.113.144.230', '2001:718:1:101::4']
+        ]
+    ]
+
+    def test_01_lookup_ip(self):
+        """
+        Perform lookup tests by IP address.
+        """
+        self.maxDiff = None
+
+        dns = mentat.services.dnsr.DnsService(timeout = 1, lifetime = 3)
+        dns.setup()
+
+        for test in self.rdns_test_list:
+            self.assertEqual(dns.lookup_ip(test[0]), test[1])
+
+    def test_02_lookup_hostname(self):
+        """
+        Perform lookup tests by hostname.
+        """
+        self.maxDiff = None
+
+        dns = mentat.services.dnsr.DnsService(timeout = 1, lifetime = 3)
+        dns.setup()
+
+        for test in self.dns_test_list:
+            self.assertEqual(dns.lookup_hostname(test[0]), test[1])
+
+    def test_03_lookup(self):
+        """
+        Perform lookup tests by hostname.
+        """
+        self.maxDiff = None
+
+        dns = mentat.services.dnsr.DnsService(timeout = 1, lifetime = 3)
+        dns.setup()
+
+        for test in self.rdns_test_list:
+            self.assertEqual(dns.lookup(test[0]), test[1])
+        for test in self.dns_test_list:
+            self.assertEqual(dns.lookup(test[0]), test[1])
+
+    def test_04_service_manager(self):
+        """
+        Perform full lookup tests with service obtained by manually configured service manager.
+        """
+        self.maxDiff = None
+
+        manager = mentat.services.dnsr.DnsServiceManager(
+            {
+                "__core__services": {
+                    "dns": {
+                        "timeout": 1,
+                        "lifetime": 3
+                    }
+                }
+            }
+        )
+
+        dns = manager.service()
+        for test in self.rdns_test_list:
+            self.assertEqual(dns.lookup_ip(test[0]), test[1])
+
+    def test_05_module_service(self):
+        """
+        Perform full lookup tests with service obtained by module interface.
+        """
+        self.maxDiff = None
+
+        mentat.services.dnsr.init(
+            {
+                "__core__services": {
+                    "dns": {
+                        "timeout": 1,
+                        "lifetime": 3
+                    }
+                }
+            }
+        )
+
+        dns = mentat.services.dnsr.service()
+        for test in self.rdns_test_list:
+            self.assertEqual(dns.lookup_ip(test[0]), test[1])
+
+
+#-------------------------------------------------------------------------------
+
+
+if __name__ == '__main__':
+    unittest.main()