Feature: Load detectors from reference file. (Redmine issue: #7577)

lib/mentat/datatype/internal.py

@@ -397,6 +397,33 @@ def t_saved_query(val):
     """
     return val
 
+def t_detector_record(val, source):
+    """
+    Convert/validate: Detector record.
+
+    :param any val: Value to be converted/validated
+    :return: detector record object
+    :rtype: DetectorRecord
+    :raises ValueError: if the value is not valid detector record
+    """
+    if isinstance(val, Detector):
+        return val
+
+    record = {}
+
+    try:
+        record['_id'] = val.get('id', gen_sid())
+        record['source'] = source
+        record['name'] = val['name']
+        if 'credibility' in val:
+            record['credibility'] = val['credibility']
+        else:
+            record['credibility'] = 1.0
+        #record['credibility']
+    except:
+        raise ValueError('Unknown detector record {}'.format(pprint.pformat(val)))
+    return Detector(record)
+
 #-------------------------------------------------------------------------------
 # DATATYPE DEFINITIONS
 #-------------------------------------------------------------------------------

lib/mentat/module/detmngr.py

@@ -81,7 +81,7 @@ import pyzenkit.jsonconf
 import mentat.script.fetcher
 import mentat.const
 import mentat.datatype.internal
-from mentat.datatype.sqldb import GroupModel, NetworkModel, SettingsReportingModel, \
+from mentat.datatype.sqldb import DetectorModel, NetworkModel, SettingsReportingModel, \
     networkmodel_from_typeddict
 
 DETECTORS_FILE_GENERIC = 'detectors-file'
@@ -219,4 +219,54 @@ class MentatDetmngrScript(mentat.script.fetcher.FetcherScript):
         :return: Structure containing information about changes.
         :rtype: dict
         """
-        raise NotImplementedError
+        result = {'create': {}, 'delete': {}, 'update': {}}
+
+        det_file = self.c(self.CONFIG_DETECTORS_FILE)
+        det_file_type, det_file_data_raw = self._load_detectors_file(det_file)
+        self.logger.debug("Raw data: %s", str(det_file_data_raw))
+        det_file_data = self._process_detectors_data(det_file_data_raw, det_file_type)
+        self.logger.info("Number of detectors in reference detectors file: %d", len(det_file_data.keys()))
+
+        detectors = self.sqlservice.session.query(DetectorModel).order_by(DetectorModel.name).all()
+        self.sqlservice.session.commit()
+        self.logger.info("Number of detectors in the database: %d", len(detectors))
+
+        return result
+
+    def _load_detectors_file(self, detectors_file):
+        """
+        Load reference detectors file.
+
+        :param str detectors_file: Name of the reference detectors file.
+        :return: Data content of detectors file.
+        :rtype: dict
+        """
+        try:
+            with open(detectors_file, 'r') as jsf:
+                json_data = jsf.read()
+            detectors_file_data = json.loads(json_data)
+        except:
+            raise pyzenkit.zenscript.ZenScriptException("Invalid detectors file '{}', expected JSON formated file".format(detectors_file))
+
+        detectors_file_type = self.c(self.CONFIG_DETECTORS_SOURCE)
+        self.logger.info("Loaded reference detectors file '%s :: %s'", detectors_file, detectors_file_type)
+        return (detectors_file_type, detectors_file_data)
+
+    def _process_detectors_data(self, detectors_file_data, detectors_file_type):
+        """
+        Process reference detectors file data into format more appropriate for searching
+        and comparisons.
+
+        :param dict whois_file_data: Whois data as loaded by :py:func:`_load_whois_file`.
+        :param str whois_file_type: Type of the whois file (value of ``__whois_type__`` meta attribute).
+        :return: Processed whois file data into format more appropriate for searching.
+        :rtype: dict
+        """
+        if 'clients' not in detectors_file_data:
+            raise pyzenkit.zenscript.ZenScriptException("Invalid detectors file '{}' format, expected 'clients' key.".format(detectors_file))
+        processed_data = collections.defaultdict(dict)
+        for client in detectors_file_data['clients']:
+            det = mentat.datatype.internal.t_detector_record(client, source = detectors_file_type)
+            self.logger.debug(det)
+            processed_data[det['name']] = det
+        return processed_data