From 1184eb55a57cc991e1bcd101fe071e3254b1836c Mon Sep 17 00:00:00 2001
From: Pavel Eis <xeispa00@stud.fit.vutbr.cz>
Date: Wed, 30 Jan 2019 14:39:00 +0100
Subject: [PATCH] Update of misp.py converter
MISP to IDEA converted moved to class MispToIdea, some semantic
conversion fixes (Email in Source is array etc.). Now using RSIT and
ECSIRT taxonomy, mapping table can be found in 'category_to_taxonomy'
variable. Added support of custom objects, which were defined for
IDEA to MISP conversion. Conversion can be made by calling method
to_idea on MispToIdea instance.
Also IDEA to MISP conversion finished and placed to IdeaToMisp class.
Conversion is using custom object templates, which can be found in
/misp_object_templates. Conversion can be made by calling method
to_misp on IdeaToMisp instance.
Added custom object templates to /misp_object_templates. Their name
will probably change in the future.
---
idea/misp.py | 826 ++++++++++++-------
misp_object_templates/attach/definition.json | 121 +++
misp_object_templates/source/definition.json | 104 +++
misp_object_templates/target/definition.json | 104 +++
4 files changed, 867 insertions(+), 288 deletions(-)
create mode 100644 misp_object_templates/attach/definition.json
create mode 100644 misp_object_templates/source/definition.json
create mode 100644 misp_object_templates/target/definition.json
diff --git a/idea/misp.py b/idea/misp.py
index d9ebd95..a744ed2 100644
--- a/idea/misp.py
+++ b/idea/misp.py
@@ -3,307 +3,557 @@
# Copyright (c) 2018, CESNET, z. s. p. o.
# Use of this source is governed by an ISC license, see LICENSE file.
+from pymisp import MISPEvent, MISPObject, NewAttributeError
from uuid import uuid4
import time
import itertools
from _datetime import datetime
-
-misp_to_idea_dictionary = {}
-
-# Every list on current index corresponds to idea_category on the same index
-misp_tags = [['circl:incident-classification="spam"'],
- ['veris:action:malware:variety="Exploit vuln"', 'veris:action:malware:variety="Backdoor"',
- 'ecsirt:intrusions="backdoor"'],
- ['circl:incident-classification="phishing"', 'europol-incident:information-gathering="phishing"',
- 'enisa:nefarious-activity-abuse="phishing-attacks"'],
- ['circl:incident-classification="scan"'],
- ['veris:action:malware:variety="Rootkit"'],
- ['europol-incident:availability="dos-ddos"', 'ms-caro-malware:malware-type="DDoS"'],
- ['circl:incident-classification="denial-of-service"'],
- ['circl:incident-classification="malware"']]
-
-idea_categories = ["Abusive.Spam", "Attempt.Exploit", "Fraud.Phishing", "Recon.Scanning", "Malware.Rootkit",
- "Availability.DDoS", "Availability.DoS", "Malware"]
-
-# prepare category translation dictionary
-for index, tag_list in enumerate(misp_tags):
- for tag in tag_list:
- misp_to_idea_dictionary[tag] = idea_categories[index]
-
-useful_hashes = ["md5", "sha1", "sha224", "sha256", "sha384", "sha512", "sha512/224", "sha512/256", "filename|md5",
- "filename|sha1", "filename|sha224", "filename|sha256", "filename|sha384", "filename|sha512",
- "filename|sha512/224", "filename|sha512/256"]
-
-useful_ip_objects = ["ip-port", "domain-ip", "netflow", "network-connection", "network-socket"]
-
-
-def get_date_from_timestamp(timestamp):
- date_and_time = convert_epoch_to_utc(timestamp)
- # returns only date from datetime, date stops at index 9 (2018-05-16T06:20:35Z)
- return date_and_time[0:9]
-
-
-def convert_epoch_to_utc(timestamp):
+import re
+
+
+category_to_taxonomy = {
+ 'Abusive.Spam': ['ecsirt:abusive-content="spam"', 'rsit:abusive-content="spam"'],
+ 'Abusive.Harassment': ['ecsirt:abusive-content="harmful-speech"', 'rsit:abusive-content="harmful-speech"'],
+ 'Abusive.Child': ['ecsirt:abusive-content="violence"', 'rsit:abusive-content="violence"'],
+ 'Abusive.Sexual': ['ecsirt:abusive-content="violence"', 'rsit:abusive-content="violence"'],
+ 'Abusive.Violence': ['ecsirt:abusive-content="violence"', 'rsit:abusive-content="violence"'],
+ 'Malware.Virus': ['ecsirt:malicious-code="virus"', 'rsit:malicious-code="virus"'],
+ 'Malware.Worm': ['ecsirt:malicious-code="worm"', 'rsit:malicious-code="worm"'],
+ 'Malware.Trojan': ['ecsirt:malicious-code="trojan"', 'rsit:malicious-code="trojan"'],
+ 'Malware.Spyware': ['ecsirt:malicious-code="spyware"', 'rsit:malicious-code="spyware"'],
+ 'Malware.Dialer': ['ecsirt:malicious-code="dialer"', 'rsit:malicious-code="dialer"'],
+ 'Malware.Rootkit': ['ecsirt:malicious-code="rootkit"', 'rsit:malicious-code="rootkit"'],
+ 'Recon.Scanning': ['ecsirt:information-gathering="scanner"', 'rsit:information-gathering="scanner"'],
+ 'Recon.Sniffing': ['ecsirt:information-gathering="sniffing"', 'rsit:information-gathering="sniffing"'],
+ 'Recon.SocialEngineering': ['ecsirt:information-gathering="social-engineering"',
+ 'rsit:information-gathering="social-engineering"'],
+ 'Recon.Searching': ['ecsirt:information-gathering', 'rsit:information-gathering'],
+ 'Attempt.Exploit': ['ecsirt:intrusion-attempts="exploit"', 'rsit:intrusion-attempts="exploit"'],
+ 'Attempt.Login': ['ecsirt:intrusion-attempts="brute-force"', 'rsit:intrusion-attempts="brute-force"'],
+ 'Attempt.NewSignature': ['ecsirt:intrusion-attempts', 'rsit:intrusion-attempts'],
+ 'Intrusion.AdminCompromise': ['ecsirt:intrusions="unprivileged-account-compromise"',
+ 'rsit:intrusions="unprivileged-account-compromise"'],
+ 'Intrusion.UserCompromise': ['ecsirt:intrusions="unprivileged-account-compromise"',
+ 'rsit:intrusions="unprivileged-account-compromise"'],
+ 'Intrusion.AppCompromise': ['ecsirt:intrusions="application-compromise"',
+ 'rsit:intrusions="application-compromise"'],
+ 'Intrusion.Botnet': ['ecsirt:intrusions="bot"', 'rsit:intrusions="bot"'],
+ 'Availability.DoS': ['ecsirt:availability="dos"', 'rsit:availability="dos"'],
+ 'Availability.DDoS': ['ecsirt:availability="ddos"', 'rsit:availability="ddos"'],
+ 'Availability.Sabotage': ['ecsirt:availability="sabotage"', 'rsit:availability="sabotage"'],
+ 'Availability.Outage': ['ecsirt:availability="outage"', 'rsit:availability="outage"'],
+ 'Information.UnauthorizedAccess': ['ecsirt:information-content-security="Unauthorised-information-access"',
+ 'rsit:information-content-security="Unauthorised-information-access"'],
+ 'Information.UnauthorizedModification':
+ ['ecsirt:information-content-security="Unauthorised-information-modification"',
+ 'rsit:information-content-security="Unauthorised-information-modification"'],
+ 'Fraud.UnauthorizedUsage': ['ecsirt:fraud="unauthorized-use-of-resources"',
+ 'rsit:fraud="unauthorized-use-of-resources"'],
+ 'Fraud.Copyright': ['ecsirt:fraud="copyright"', 'rsit:fraud="copyright"'],
+ 'Fraud.Masquerade': ['ecsirt:fraud="masquerade"', 'rsit:fraud="masquerade"'],
+ 'Fraud.Phishing': ['ecsirt:fraud="phishing"', 'ecsirt:fraud="phishing"'],
+ 'Fraud.Scam': ['ecsirt:fraud', 'rsit:fraud'],
+ 'Vulnerable.Open': ['ecsirt:vulnerable="vulnerable-service"', 'rsit:vulnerable="vulnerable-service"'],
+ 'Vulnerable.Config': ['ecsirt:vulnerable="vulnerable-service"', 'rsit:vulnerable="vulnerable-service"'],
+ 'Anomaly.Traffic': ['ecsirt:other="other"', 'rsit:other="other"'],
+ 'Anomaly.Connection': ['ecsirt:other="other"', 'rsit:other="other"'],
+ 'Anomaly.Protocol': ['ecsirt:other="other"', 'rsit:other="other"'],
+ 'Anomaly.System': ['ecsirt:other="other"', 'rsit:other="other"'],
+ 'Anomaly.Application': ['ecsirt:other="other"', 'rsit:other="other"'],
+ 'Anomaly.Behaviour': ['ecsirt:other="other"', 'rsit:other="other"'],
+ 'Other': ['ecsirt:other="other"', 'rsit:other="other"'],
+ 'Test': ['ecsirt:test="test"', 'rsit:test="test"']
+}
+
+
+class MispToIdea(object):
"""
- Converts Unix timestamp to to UTC datetime
- :param timestamp: Unix timestamp
- :return: UTC datetime in string
+ Converts MISP event in MISP core format to IDEA event
"""
- return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(int(timestamp)))
-
-def get_ip_version(ip_addr):
- """
- Gets verstion of IP address
- :param ip_addr: the IP address
- :return: version of IP address
- """
- return "IP6" if ":" in ip_addr else "IP4"
-
-
-def get_ip_addr(attrib, ip_only=True):
- """
- Gets IP address value from event's attribute
- :param attrib: the attribute
- :param ip_only: gets only IP address
- :return: returns IP address, role of IP address (Source|Target), port, hostname or IP address only if ip_only == True
- """
- role = "Source" if "src" in attrib['type'] else "Target"
- if "ip-src" == attrib['type'] or "ip-dst" == attrib['type']:
- return attrib['value'] if ip_only is True else (attrib['value'], role, None, None)
- elif "ip-src|port" == attrib['type'] or "ip-dst|port" == attrib['type']:
- split_attrib = attrib['value'].split('|')
- if len(split_attrib) == 1:
- split_attrib = attrib['value'].split(':')
- return split_attrib[0] if ip_only else (split_attrib[0], role, split_attrib[1], None)
- else:
- # attrib['type'] == "domain|ip"
- split_attrib = attrib['value'].split('|')
- return split_attrib[1] if ip_only else (split_attrib[1], "Target", None, split_attrib[0])
-
-
-# https://stackoverflow.com/questions/9807634/find-all-occurrences-of-a-key-in-nested-python-dictionaries-and-lists
-def find(key, value):
- """
- find gradually all values by key
- :param key: searched key
- :param value: searched object (dict|list)
- :return: found value by key
- """
- if isinstance(value, dict):
- for k, v in value.items():
- if k == key:
- yield v
- elif isinstance(v, dict):
- for result in find(key, v):
- yield result
- elif isinstance(v, list):
- for d in v:
- for result in find(key, d):
+ useful_hashes = ["md5", "sha1", "sha224", "sha256", "sha384", "sha512", "sha512/224", "sha512/256", "filename|md5",
+ "filename|sha1", "filename|sha224", "filename|sha256", "filename|sha384", "filename|sha512",
+ "filename|sha512/224", "filename|sha512/256"]
+ useful_ip_objects = ["ip-port", "domain-ip", "netflow", "network-connection", "network-socket"]
+ _taxonomy_to_category_created = False
+ taxonomy_to_category = {}
+
+ def __init__(self):
+ if not self._taxonomy_to_category_created:
+ # create taxonomy_to_category conversion dictionary
+ for idea_category, misp_taxonomies in category_to_taxonomy.items():
+ # take ECSIRT category and assign corresponding IDEA category
+ self.taxonomy_to_category[misp_taxonomies[0]] = idea_category
+ # take RSIT category and assign corresponding IDEA category
+ self.taxonomy_to_category[misp_taxonomies[1]] = idea_category
+ self._taxonomy_to_category_created = True
+
+ self._attach_counter = 0
+ self._source = []
+ self._target = []
+ self._attach = []
+
+ @staticmethod
+ def convert_epoch_to_utc(timestamp):
+ """
+ Converts Unix timestamp to to UTC datetime
+ :param timestamp: Unix timestamp
+ :return: UTC datetime in string
+ """
+ return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(int(timestamp)))
+
+ @staticmethod
+ def get_date_from_timestamp(timestamp):
+ date_and_time = __class__.convert_epoch_to_utc(timestamp)
+ # returns only date from datetime, date stops at index 9 (2018-05-16T06:20:35Z)
+ return date_and_time[0:9]
+
+ @staticmethod
+ def get_ip_version(ip_addr):
+ """
+ Gets verstion of IP address
+ :param ip_addr: the IP address
+ :return: version of IP address
+ """
+ return "IP6" if ":" in ip_addr else "IP4"
+
+ @staticmethod
+ def get_ip_addr(attrib, ip_only=True):
+ """
+ Gets IP address value from event's attribute
+ :param attrib: the attribute
+ :param ip_only: gets only IP address
+ :return: returns IP address, role of IP address (Source|Target), port, hostname or IP address only if
+ ip_only == True
+ """
+ role = "Source" if "src" in attrib['type'] else "Target"
+ if "ip-src" == attrib['type'] or "ip-dst" == attrib['type']:
+ return attrib['value'] if ip_only is True else (attrib['value'], role, None, None)
+ elif "ip-src|port" == attrib['type'] or "ip-dst|port" == attrib['type']:
+ split_attrib = attrib['value'].split('|')
+ if len(split_attrib) == 1:
+ split_attrib = attrib['value'].split(':')
+ return split_attrib[0] if ip_only else (split_attrib[0], role, split_attrib[1], None)
+ else:
+ # attrib['type'] == "domain|ip"
+ split_attrib = attrib['value'].split('|')
+ return split_attrib[1] if ip_only else (split_attrib[1], "Target", None, split_attrib[0])
+
+ @classmethod
+ # https://stackoverflow.com/questions/9807634/find-all-occurrences-of-a-key-in-nested-python-dictionaries-and-lists
+ def find(cls, key, value):
+ """
+ find gradually all values by key
+ :param key: searched key
+ :param value: searched object (dict|list)
+ :return: found value by key
+ """
+ if isinstance(value, dict):
+ for k, v in value.items():
+ if k == key:
+ yield v
+ elif isinstance(v, dict):
+ for result in cls.find(key, v):
yield result
- elif isinstance(value, list):
- for v in value:
- for result in find(key, v):
- yield result
+ elif isinstance(v, list):
+ for d in v:
+ for result in cls.find(key, d):
+ yield result
+ elif isinstance(value, list):
+ for v in value:
+ for result in cls.find(key, v):
+ yield result
+ def process_misp_attribute(self, attrib):
+ """
+ Process MISP attribute and get all useful data from it and insert it into corresponding IDEA object list
+ :param attrib: the attribute, which will be processed
+ :return: None
+ """
+ # if attribute contains ip address, get it with other potential attributes(port, hostname) and append it to
+ # Source or Target based on IP address type
+ if "ip" in attrib['type']:
+ ip_addr, role, port, hostname = self.get_ip_addr(attrib, False)
+ new_description = {self.get_ip_version(ip_addr): [ip_addr]}
+ if port:
+ new_description['Port'] = [int(port)]
+ if hostname:
+ new_description['Hostname'] = [hostname]
+ if role == "Source":
+ self._source.append(new_description)
+ if role == "Target":
+ self._target.append(new_description)
+
+ # if attrib is hash, create new attach
+ elif attrib['type'] in __class__.useful_hashes:
+ new_attach = {'Handle': "attach" + str(self._attach_counter)}
+ self._attach_counter += 1
+ if "|" in attrib['type']:
+ new_attach['FileName'] = [attrib['value'].split("|")[0]]
+ new_attach['Hash'] = [attrib['type'].split("|")[1] + ":" + attrib['value'].split("|")[1]]
+ else:
+ new_attach['Hash'] = [attrib['type'] + ":" + attrib['value']]
+ self._attach.append(new_attach)
+
+ elif attrib['type'] == "url":
+ new_attach_name = "attach" + str(self._attach_counter)
+ self._attach_counter += 1
+ self._attach.append({'Handle': new_attach_name, 'Ref': [attrib['value']]})
+
+ elif attrib['type'] == "email-src":
+ self._source.append({'Email': [attrib['value']]})
+
+ elif attrib['type'] == "email-dst":
+ self._target.append({'Email': [attrib['value']]})
+
+ @staticmethod
+ def append_value_or_create_list(key, value, idea_object):
+ """
+ Append value to array retrieved by key from idea_object, if key does not exist, create key in idea_object
+ and assign array with inserted value to this key
+ :param key: search key
+ :param value: appended value
+ :param idea_object: updated object
+ :return: None
+ """
+ try:
+ idea_object[key].append(value)
+ except KeyError:
+ idea_object[key] = [value]
+
+ def process_misp_object(self, misp_object):
+ """
+ Process MISP object and get all useful data from it and insert it into corresponding IDEA object array
+ :param misp_object: the object, which will be processed
+ :return: None
+ """
+ # if object name is in useful_ip_objects, all its attributes will be filled into source or target
+ if misp_object['name'] in __class__.useful_ip_objects:
+ source = {}
+ target = {}
+ proto_dict = {}
+ for attrib in misp_object['Attribute']:
+ if attrib['type'] == "ip-dst":
+ ip_addr = self.get_ip_addr(attrib)
+ self.append_value_or_create_list(self.get_ip_version(ip_addr), ip_addr, target)
+ elif attrib['type'] == "ip-src":
+ ip_addr = self.get_ip_addr(attrib)
+ self.append_value_or_create_list(self.get_ip_version(ip_addr), ip_addr, source)
+ elif attrib['type'] == "src-port" or attrib['object_relation'] == "src-port":
+ source['Port'] = [int(attrib['value'])]
+ elif attrib['type'] == "dst-port" or attrib['object_relation'] == "dst-port":
+ target['Port'] = [int(attrib['value'])]
+ elif attrib['object_relation'] == "hostname-src":
+ source['Hostname'] = [attrib['value']]
+ elif attrib['object_relation'] == "hostname-dst" or attrib['type'] == "hostname":
+ target['Hostname'] = [attrib['value']]
+ elif attrib['object_relation'] == "protocol":
+ source['Proto'] = [attrib['value']]
+ target['Proto'] = [attrib['value']]
+ elif attrib['object_relation'].startswith("layer"):
+ # means layer3-protocol, layer4-protocol or layer7-protocol
+ # save its value to number of layer
+ proto_dict[attrib['object_relation'][5]] = attrib['value']
+ elif attrib['type'] == "src-as" or attrib['object_relation'] == "src-as":
+ source['ASN'] = [int(attrib['value'])]
+ elif attrib['type'] == "dst-as" or attrib['object_relation'] == "dst-as":
+ target['ASN'] = [int(attrib['value'])]
+ elif attrib['type'] == "domain":
+ # won't overwrite previous value of hostname-dst, hostname-dst can occur in object of type
+ # network-connection and network-socket, but domain occurs only in other object types
+ target['Hostname'] = [attrib['value']]
+
+ if proto_dict:
+ # sort protocols by layer
+ proto_list = [proto_dict.get("3", "")] + [proto_dict.get("4", "")] + [proto_dict.get("7", "")]
+ # remove empty strings from proto list
+ proto_list = [proto for proto in proto_list if proto != ""]
+ source['Proto'] = proto_list
+ target['Proto'] = proto_list
+
+ if source:
+ self._source.append(source)
+ if target:
+ self._target.append(target)
+
+ elif misp_object['name'] == "file":
+ new_attach = {'Handle': "attach" + str(self._attach_counter)}
+ self._attach_counter += 1
+ for attrib in misp_object['Attribute']:
+ if attrib['type'] in __class__.useful_hashes:
+ try:
+ new_attach['Hash'].append(attrib['type'] + ":" + attrib['value'])
+ except KeyError:
+ new_attach['Hash'] = [attrib['type'] + ":" + attrib['value']]
+ elif attrib['type'] == "filename":
+ try:
+ new_attach['FileName'].append(attrib['value'])
+ except KeyError:
+ new_attach['FileName'] = [attrib['value']]
+ if new_attach.get('Hash') or new_attach.get('FileName'):
+ self._attach.append(new_attach)
+
+ def process_source_or_target_object(self, misp_object):
+ """
+ Process source or target object, and retrieved data insert to IDEA source or target object and append it to
+ corresponding object array
+ :param misp_object: the object, which will be processed
+ :return: None
+ """
+ converted_object = {}
+ for misp_attrib in misp_object['Attribute']:
+ if misp_attrib['object_relation'] not in ("Note", "Vulnerability", "Reference"):
+ self.append_value_or_create_list(misp_attrib['object_relation'], misp_attrib['value'], converted_object)
+ else:
+ if misp_attrib['object_relation'] == "Note":
+ converted_object['Note'] = misp_attrib['value']
+ elif misp_attrib['object_relation'] == "Vulnerability":
+ self.append_value_or_create_list("Ref", "cve:" + misp_attrib['value'],
+ converted_object)
+ elif misp_attrib['object_relation'] == "Reference":
+ self.append_value_or_create_list("Ref", misp_attrib['value'], converted_object)
+ if misp_object['name'] == "source":
+ self._source.append(converted_object)
+ else:
+ self._target.append(converted_object)
+
+ def process_attach_object(self, misp_object):
+ """
+ Process attach object, and retrieved data insert to IDEA attach object and append it to attach array
+ :param misp_object: the object, which will be processed
+ :return: None
+ """
+ converted_object = {"Handle": "att" + str(self._attach_counter)}
+ self._attach_counter += 1
+ for attach_attrib in misp_object['Attribute']:
+ if attach_attrib['object_relation'] in ("Note", "ContentType", "ContentCharset", "ContentEncoding",
+ "Content"):
+ converted_object[attach_attrib['object_relation']] = attach_attrib['value']
+ elif attach_attrib['object_relation'] == "FileName":
+ self.append_value_or_create_list("FileName", attach_attrib['value'], converted_object)
+ elif attach_attrib['object_relation'] == "Reference":
+ self.append_value_or_create_list("Ref", attach_attrib['value'], converted_object)
+ elif attach_attrib['object_relation'] == "Vulnerability":
+ self.append_value_or_create_list("Ref", "cve" + attach_attrib['value'], converted_object)
+ elif attach_attrib['object_relation'] == "Size":
+ converted_object['Size'] = int(attach_attrib['value'])
+ else:
+ self.append_value_or_create_list("Hash", attach_attrib['type'] + ":" + attach_attrib['value'],
+ converted_object)
+ self._attach.append(converted_object)
+
+ def to_idea(self, misp_event, idea_id=None, test=False, origdata=False):
+ """
+ Creates whole IDEA message from MISP event
+ :param misp_event: the misp event
+ :param idea_id: uuid of IDEA message (when needs to be preset)
+ :param test: add Test into IDEA['Category']
+ :param origdata: add original data to attachment
+ :return: new converted IDEA message
+ """
+ idea_event = {
+ 'Format': "IDEA0",
+ 'ID': str(idea_id) if idea_id is not None else str(uuid4()),
+ 'Category': [],
+ 'Description': misp_event['info']
+ }
+
+ # fill in the IDEA event Category
+ for tag in misp_event.get('Tag', []):
+ try:
+ new_category = self.taxonomy_to_category[tag['name']]
+ if new_category not in idea_event['Category']:
+ idea_event['Category'].append(new_category)
+ except KeyError:
+ pass
+
+ # cannot determine IDEA Category, cannot convert
+ if not idea_event['Category']:
+ return None
+ if test and "Test" not in idea_event['Category']:
+ idea_event['Category'].append("Test")
+
+ if not int(misp_event['publish_timestamp']):
+ idea_event['CreateTime'] = datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ")
+ else:
+ idea_event['CreateTime'] = self.convert_epoch_to_utc(misp_event['publish_timestamp'])
-def process_misp_attribute(attrib, attach_counter):
+ timestamps = itertools.chain(self.find('timestamp', misp_event['Attribute']),
+ self.find('timestamp', misp_event['Object']))
+ try:
+ oldest_timestamp = min(map(int, timestamps))
+ idea_event['DetectTime'] = self.convert_epoch_to_utc(oldest_timestamp)
+ except ValueError:
+ # no object and attributes --> min() ValueError for empty sequence
+ idea_event['DetectTime'] = misp_event['date'] + "T00:00:00Z"
+
+ # fill in info about organizations
+ idea_event['Node'] = [
+ {
+ 'Name': misp_event['Orgc']['name'],
+ 'Note': "MISP organization id (created event): " + misp_event['Orgc']['id']
+ },
+ {
+ 'Name': misp_event['Org']['name'],
+ 'Note': "MISP organization id (reported event): " + misp_event['Org']['id']
+ }]
+
+ # check all attributes for all potentially useful data
+ for attrib in misp_event['Attribute']:
+ self.process_misp_attribute(attrib)
+
+ # check all objects for all potentially useful data
+ for misp_object in misp_event['Object']:
+ if misp_object['name'] in ("source", "target"):
+ # process the object
+ self.process_source_or_target_object(misp_object)
+ elif misp_object['name'] == "attach":
+ self.process_attach_object(misp_object)
+ else:
+ self.process_misp_object(misp_object)
+
+ if origdata:
+ idea_event['Attach'].append({
+ 'Handle': "att" + str(self._attach_counter),
+ 'Note': "original data",
+ 'Content': misp_event
+ })
+
+ if self._source:
+ idea_event['Source'] = self._source
+ if self._target:
+ idea_event['Target'] = self._target
+ if self._attach:
+ idea_event['Attach'] = self._attach
+
+ # prepare instance for another conversion
+ self.__init__()
+
+ return idea_event
+
+
+class IdeaToMisp(object):
"""
- Process MISP attribute and get all useful data from it
- :param attrib: the attribute
- :param attach_counter: IDEA attachment counter
- :return: IDEA key (Source| Target | Attach) and its data
+ Converts IDEA event to MISP event in MISP core format
"""
- # if attribute contains ip address, get it with other potential attributes(port, hostname) and append it to
- # Source or Target based on IP address type
- if "ip" in attrib['type']:
- ip_addr, role, port, hostname = get_ip_addr(attrib, False)
- new_description = {get_ip_version(ip_addr): ip_addr}
- if port:
- new_description['Port'] = [int(port)]
- if hostname:
- new_description['Hostname'] = [hostname]
- return role, new_description
-
- # if attrib is hash, create new attach
- elif attrib['type'] in useful_hashes:
- new_attach = {'Handle': "attach" + str(attach_counter)}
- if "|" in attrib['type']:
- new_attach['Filename'] = [attrib['value'].split("|")[0]]
- new_attach['Hash'] = [attrib['type'].split("|")[1] + ":" + attrib['value'].split("|")[1]]
- else:
- new_attach['Hash'] = [attrib['type'] + ":" + attrib['value']]
- return "Attach", new_attach
- elif attrib['type'] == "url":
- return "Attach", {'Handle': "attach" + str(attach_counter), 'Ref': [attrib['value']]}
+ re_cve = re.compile("cve:", re.IGNORECASE)
+ severity_conversion = ["", "low", "medium", "high"]
+
+ def __init__(self):
+ """
+ Initializes class and prepares instance variables
+ """
+ self._idea_event = None
+ self._new_event = None
+
+ def process_basic_info(self):
+ """
+ Process basic info of MISP event such distribution, threat_level_id, info, Tags, and standalone attributes
+ :return: None
+ """
+ self._new_event = MISPEvent()
+ # add basic info about event
+ self._new_event.distribution = 1
- elif attrib['type'] == "email-src":
- return "Source", [attrib['value']]
-
- elif attrib['type'] == "email-dst":
- return "Target", [attrib['value']]
-
- return None, None
-
-
-def process_misp_object(misp_object, attach_counter):
- """
- Process MISP object and get all useful data from it
- :param misp_object: the object
- :param attach_counter: IDEA attachment counter
- :return: IDEA key (Source| Target | Attach) and its data
- """
- # if object name is in useful_ip_objects, all its attributes will be filled into source or target
- if misp_object['name'] in useful_ip_objects:
- source = {}
- target = {}
- proto_dict = {}
- for attrib in misp_object['Attribute']:
- if attrib['type'] == "ip-dst":
- ip_addr = get_ip_addr(attrib)
- target[get_ip_version(ip_addr)] = ip_addr
- elif attrib['type'] == "ip-src":
- ip_addr = get_ip_addr(attrib)
- source[get_ip_version(ip_addr)] = ip_addr
- elif attrib['type'] == "src-port" or attrib['object_relation'] == "src-port":
- source['Port'] = [int(attrib['value'])]
- elif attrib['type'] == "dst-port" or attrib['object_relation'] == "dst-port":
- target['Port'] = [int(attrib['value'])]
- elif attrib['type'] == "hostname-src":
- source['Hostname'] = [attrib['value']]
- elif attrib['type'] == "hostname-dst":
- target['Hostname'] = [attrib['value']]
- elif attrib['type'] == "protocol":
- source['Proto'] = [attrib['value']]
- target['Proto'] = [attrib['value']]
- elif attrib['type'].startswith("layer"):
- # means layer3-protocol, layer4-protocol or layer7-protocol
- # save its value to number of layer
- proto_dict[attrib['type'][5]] = attrib['value']
- elif attrib['type'] == "src-as" or attrib['object_relation'] == "src-as":
- source['ASN'] = [int(attrib['value'])]
- elif attrib['type'] == "dst-as" or attrib['object_relation'] == "dst-as":
- target['ASN'] = [int(attrib['value'])]
- elif attrib['type'] == "domain":
- # won't be overwrite previous value of hostname-dst, hostname-dst can occur in object of type
- # network-connection and network-socket, but domain occurs only in other object types
- target['Hostname'] = [attrib['value']]
-
- if proto_dict:
- # sort protocols by layer
- proto_list = [proto_dict.get("3", "")] + [proto_dict.get("4", "")] + [proto_dict.get("7", "")]
- # remove empty strings from proto list
- proto_list = [proto for proto in proto_list if proto != ""]
- source['Proto'] = proto_list
- target['Proto'] = proto_list
-
- if source:
- yield "Source", source
- if target:
- yield "Target", target
-
- elif misp_object['name'] == "file":
- new_attach = {'Handle': "attach" + str(attach_counter)}
- for attrib in misp_object['Attribute']:
- if attrib['type'] in useful_hashes:
- try:
- new_attach['Hash'].append(attrib['type'] + ":" + attrib['value'])
- except KeyError:
- new_attach['Hash'] = [attrib['type'] + ":" + attrib['value']]
- elif attrib['type'] == "filename":
- try:
- new_attach['Filename'].append(attrib['value'])
- except KeyError:
- new_attach['Filename'] = [attrib['value']]
- if new_attach.get('Hash') or new_attach.get('Filename'):
- yield "Attach", new_attach
- else:
- return None, None
-
-
-def to_idea(misp_event, idea_id=None, test=False, origdata=False):
- """
- Creates whole IDEA message from MISP event
- :param misp_event: the misp event
- :param idea_id: uuid of IDEA message (when needs to be preset)
- :param test: add Test into IDEA['Category']
- :param origdata: add original data to attachment
- :return: new converted IDEA message
- """
- idea_event = {
- 'Format': "IDEA0",
- 'ID': str(idea_id) if idea_id is not None else str(uuid4()),
- 'Category': [],
- 'Description': misp_event['info'],
- 'Source': [],
- 'Target': [],
- 'Attach': []
- }
-
- # fill in the IDEA event Category
- for tag in misp_event.get('Tag', []):
try:
- new_category = misp_to_idea_dictionary[tag['name']]
- if new_category not in idea_event['Category']:
- idea_event['Category'].append(new_category)
+ self._new_event.threat_level_id = __class__.severity_conversion.index(
+ self._idea_event['_CESNET']['EventSeverity'])
except KeyError:
- pass
-
- # cannot determine IDEA Category, cannot convert
- if not idea_event['Category']:
- return None
- if test:
- idea_event['Category'].append("Test")
-
- if not int(misp_event['publish_timestamp']):
- idea_event['CreateTime'] = datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ")
- else:
- idea_event['CreateTime'] = convert_epoch_to_utc(misp_event['publish_timestamp'])
-
- attach_counter = 0
-
- timestamps = itertools.chain(find('timestamp', misp_event['Attribute']),
- find('timestamp', misp_event['Object']))
- try:
- oldest_timestamp = min(map(int, timestamps))
- idea_event['DetectTime'] = convert_epoch_to_utc(oldest_timestamp)
- except ValueError:
- # no object and attributes --> min() ValueError for empty sequence
- idea_event['DetectTime'] = misp_event['date'] + "T00:00:00Z"
-
- # fill in info about organizations
- idea_event['Node'] = [
- {
- 'Name': misp_event['Orgc']['name'],
- 'Note': "MISP organization id (created event): " + misp_event['Orgc']['id']
- },
- {
- 'Name': misp_event['Org']['name'],
- 'Note': "MISP organization id (reported event): " + misp_event['Org']['id']
- }]
-
- # check all attributes for all potentially useful data
- for attrib in misp_event['Attribute']:
- key, new_data = process_misp_attribute(attrib, attach_counter)
- if key and new_data:
- idea_event[key].append(new_data)
- if key == "Attach":
- attach_counter += 1
-
- # check all objects for all potentially useful data
- for misp_object in misp_event['Object']:
- for key, new_data in process_misp_object(misp_object, attach_counter):
- idea_event[key].append(new_data)
- if key == "Attach":
- attach_counter += 1
-
- if origdata:
- idea_event['Attach'].append({
- 'Handle': "att" + str(attach_counter),
- 'Note': "original data",
- 'Content': misp_event
- })
- attach_counter += 1
-
- return idea_event
+ # EventSeverity not in IDEA message --> 0 = Undefined
+ self._new_event.threat_level_id = 1
+
+ self._new_event.analysis = 2
+ # [0:10] is for taking date only from DetectTime (2019-01-26xxxxxxx)
+ self._new_event.date = datetime.strptime(self._idea_event['DetectTime'][0:10], "%Y-%m-%d")
+
+ if self._idea_event.get('Description'):
+ # add Description as event info and add Note as well if present
+ try:
+ self._new_event.info = self._idea_event['Description'] + ". " + self._idea_event['Note']
+ except KeyError:
+ self._new_event.info = self._idea_event['Description']
+ elif self._idea_event.get('Note'):
+ # or atleast Note if Description is not present
+ self._new_event.info = self._idea_event['Note']
+
+ # convert all categories in IDEA to taxonomies
+ for idea_category in self._idea_event['Category']:
+ # add both taxonomy ECSIRT and RSIT
+ self._new_event.add_tag(category_to_taxonomy[idea_category][0])
+ self._new_event.add_tag(category_to_taxonomy[idea_category][1])
+
+ for reference in self._idea_event.get('Ref', []):
+ # if reference is cve, add MISP attribute as vulnerability, otherwise add MISP attribute as link
+ if __class__.re_cve.search(reference):
+ self._new_event.add_attribute(category="External analysis", type="vulnerability",
+ value=__class__.re_cve.split(reference)[1])
+ else:
+ self._new_event.add_attribute(category="External analysis", type="link",
+ value=reference)
+
+ @classmethod
+ def add_attribute_to_object(cls, key, value, misp_object):
+ """
+ Add one value (new MISP attribute) to source or target or attach object
+ :param key: name of attribute relation
+ :param value: inserted value
+ :param misp_object: object, which will the attribute be inserted into
+ :return: the object with inserted attribute
+ """
+ try:
+ misp_object.add_attribute(key, value=value)
+ except NewAttributeError:
+ if key == "Ref":
+ if __class__.re_cve.search(value):
+ misp_object.add_attribute("Vulnerability", value=cls.re_cve.split(value)[1])
+ else:
+ misp_object.add_attribute("Reference", value=value)
+ if misp_object.name == "attach" and key == "Hash":
+ hash_method = value.split(":", 1)[0]
+ # FIXME may not be fixed properly
+ hash_value = value.split(":", 1)[1]
+ misp_object.add_attribute(hash_method.lower(), value=hash_value)
+ return misp_object
+
+ def process_one_idea_object(self, object_name):
+ """
+ Process IDEA's Source or Target or Attach and convert it into MISP object
+ :param object_name: defines processing of IDEA's 'Source' or 'Target' or 'Attach'
+ :return: None
+ """
+ for idea_obj in self._idea_event.get(object_name, []):
+ new_object = MISPObject(name=object_name.lower(), strict=True, standalone=False,
+ misp_objects_path_custom='object_templates')
+ for key, value in idea_obj.items():
+ if isinstance(value, list):
+ for list_value in value:
+ new_object = self.add_attribute_to_object(key, list_value, new_object)
+ else:
+ new_object = self.add_attribute_to_object(key, value, new_object)
+
+ self._new_event.add_object(new_object)
+
+ def process_all_idea_objects(self):
+ """
+ Calls processing of Source, Target and Attach objects
+ :return: None
+ """
+ self.process_one_idea_object('Source')
+ self.process_one_idea_object('Target')
+ self.process_one_idea_object('Attach')
+
+ def to_misp(self, idea_event):
+ """
+ Converts IDEA message into MISP core format message
+ :param idea_event: the idea event, which will be converted
+ :return: instance of MISPEvent (with converted data)
+ """
+ self._idea_event = idea_event
+ # fill new event with basic info and set event as published
+ self.process_basic_info()
+ self._new_event.publish()
+
+ self.process_all_idea_objects()
+
+ return self._new_event
diff --git a/misp_object_templates/attach/definition.json b/misp_object_templates/attach/definition.json
new file mode 100644
index 0000000..b559e02
--- /dev/null
+++ b/misp_object_templates/attach/definition.json
@@ -0,0 +1,121 @@
+{
+ "name": "attach",
+ "meta-category": "misc",
+ "description": "Event attachment",
+ "uuid": "f5a964ac-5782-4c3e-8056-9b2783c987a8",
+ "version": 1,
+ "attributes": {
+ "FileName": {
+ "misp-attribute": "filename",
+ "description": "Name of file",
+ "ui-priority": 1,
+ "categories": ["Payload delivery"],
+ "multiple": true
+ },
+ "md5": {
+ "misp-attribute": "md5",
+ "description": "[Insecure] MD5 hash (128 bits)",
+ "ui-priority": 0,
+ "categories": ["Payload delivery"],
+ "multiple": true
+ },
+ "sha1": {
+ "misp-attribute": "sha1",
+ "description": "[Insecure] Secure Hash Algorithm 1 (160 bits)",
+ "ui-priority": 0,
+ "categories": ["Payload delivery"],
+ "multiple": true
+ },
+ "sha224": {
+ "misp-attribute": "sha224",
+ "description": "Secure Hash Algorithm 2 (224 bits)",
+ "ui-priority": 0,
+ "categories": ["Payload delivery"],
+ "multiple": true
+ },
+ "sha256": {
+ "misp-attribute": "sha256",
+ "description": "Secure Hash Algorithm 2 (256 bits)",
+ "ui-priority": 0,
+ "categories": ["Payload delivery"],
+ "multiple": true
+ },
+ "sha384": {
+ "misp-attribute": "sha384",
+ "description": "Secure Hash Algorithm 2 (384 bits)",
+ "ui-priority": 0,
+ "categories": ["Payload delivery"],
+ "multiple": true
+ },
+ "sha512": {
+ "misp-attribute": "sha512",
+ "description": "Secure Hash Algorithm 2 (512 bits)",
+ "ui-priority": 0,
+ "categories": ["Payload delivery"],
+ "multiple": true
+ },
+ "sha512/224": {
+ "misp-attribute": "sha512/224",
+ "description": "Secure Hash Algorithm 2 (224 bits)",
+ "ui-priority": 0,
+ "categories": ["Payload delivery"],
+ "multiple": true
+ },
+ "Size": {
+ "misp-attribute": "size-in-bytes",
+ "description": "Length of the content",
+ "ui-priority": 0,
+ "categories": ["Other"],
+ "multiple": false
+ },
+ "Vulnerability": {
+ "misp-attribute": "vulnerability",
+ "description": "Reference to known source, related to vulnerability, specific to this attachment",
+ "ui-priority": 0,
+ "categories": ["External analysis"],
+ "multiple": true
+ },
+ "Reference": {
+ "misp-attribute": "other",
+ "description": "Reference to known source, related to attack, specific to this attachment",
+ "ui-priority": 0,
+ "categories": ["External analysis"],
+ "multiple": true
+ },
+ "Note": {
+ "misp-attribute": "text",
+ "description": "Free text human readable additional note",
+ "ui-priority": 0,
+ "categories": ["Payload delivery"],
+ "multiple": false
+ },
+ "ContentType": {
+ "misp-attribute": "text",
+ "description": "Internet Media Type of the attachment",
+ "ui-priority": 1,
+ "categories": ["Payload delivery"],
+ "multiple": false
+ },
+ "ContentCharset": {
+ "misp-attribute": "text",
+ "description": "Name of the content character set",
+ "ui-priority": 0,
+ "categories": ["Payload delivery"],
+ "multiple": false
+ },
+ "ContentEncoding": {
+ "misp-attribute": "text",
+ "description": "Encoding of the content",
+ "ui-priority": 0,
+ "categories": ["Payload delivery"],
+ "multiple": false
+ },
+ "Content": {
+ "misp-attribute": "text",
+ "description": "Attachment content",
+ "ui-priority": 1,
+ "categories": ["Payload delivery"],
+ "multiple": false
+ }
+ }
+}
\ No newline at end of file
diff --git a/misp_object_templates/source/definition.json b/misp_object_templates/source/definition.json
new file mode 100644
index 0000000..1df52b3
--- /dev/null
+++ b/misp_object_templates/source/definition.json
@@ -0,0 +1,104 @@
+{
+ "name": "source",
+ "meta-category": "network",
+ "description": "Description of the source of the event",
+ "uuid": "63cf1c78-4afe-49be-baff-2c101a942000",
+ "version": 1,
+ "attributes": {
+ "Hostname": {
+ "misp-attribute": "hostname",
+ "description": "Participating hostname",
+ "ui-priority": 0,
+ "categories": ["Network activity"],
+ "multiple": true
+ },
+ "IP4": {
+ "misp-attribute": "ip-src",
+ "description": "Source IPv4 address",
+ "ui-priority": 1,
+ "categories": ["Network activity"],
+ "multiple": true
+ },
+ "MAC": {
+ "misp-attribute": "mac-address",
+ "description": "Source MAC adress",
+ "ui-priority": 0,
+ "categories": ["Network activity"],
+ "multiple": true
+ },
+ "IP6": {
+ "misp-attribute": "ip-src",
+ "description": "Source IPv6 address",
+ "ui-priority": 1,
+ "categories": ["Network activity"],
+ "multiple": true
+ },
+ "Port": {
+ "misp-attribute": "port",
+ "description": "Source port",
+ "ui-priority": 1,
+ "categories": ["Other"],
+ "multiple": true
+ },
+ "Proto": {
+ "misp-attribute": "text",
+ "description": "Name of used protocol",
+ "ui-priority": 1,
+ "categories": ["Network activity"],
+ "multiple": true,
+ "values_list": [
+ "tcp",
+ "udp",
+ "icmp",
+ "ip",
+ "arp",
+ "http",
+ "https",
+ "ftp"
+ ],
+ "required": false
+ },
+ "URL": {
+ "misp-attribute": "url",
+ "description": "Source URL",
+ "ui-priority": 0,
+ "categories": ["Network activity"],
+ "multiple": true
+ },
+ "Email": {
+ "misp-attribute": "email-src",
+ "description": "Source email adress",
+ "ui-priority": 0,
+ "categories": ["Payload delivery"],
+ "multiple": true
+ },
+ "Note": {
+ "misp-attribute": "text",
+ "description": "Free text human readable additional note",
+ "ui-priority": 0,
+ "categories": ["Payload delivery"],
+ "multiple": false
+ },
+ "ASN": {
+ "misp-attribute": "AS",
+ "description": "Autonomous system number of this source",
+ "ui-priority": 0,
+ "categories": ["Network activity"],
+ "multiple": true
+ },
+ "Vulnerability": {
+ "misp-attribute": "vulnerability",
+ "description": "Reference to known source, related to vulnerability, specific to this source",
+ "ui-priority": 0,
+ "categories": ["External analysis"],
+ "multiple": true
+ },
+ "Reference": {
+ "misp-attribute": "other",
+ "description": "Reference to known source, related to attack, specific to this source",
+ "ui-priority": 0,
+ "categories": ["External analysis"],
+ "multiple": true
+ }
+ }
+}
\ No newline at end of file
diff --git a/misp_object_templates/target/definition.json b/misp_object_templates/target/definition.json
new file mode 100644
index 0000000..7336da9
--- /dev/null
+++ b/misp_object_templates/target/definition.json
@@ -0,0 +1,104 @@
+{
+ "name": "target",
+ "meta-category": "network",
+ "description": "Description of the target of the event",
+ "uuid": "b251672f-3463-47d6-a20f-19d04c383195",
+ "version": 1,
+ "attributes": {
+ "Hostname": {
+ "misp-attribute": "hostname",
+ "description": "Participating hostname",
+ "ui-priority": 0,
+ "categories": ["Network activity"],
+ "multiple": true
+ },
+ "IP4": {
+ "misp-attribute": "ip-dst",
+ "description": "Target IPv4 address",
+ "ui-priority": 1,
+ "categories": ["Network activity"],
+ "multiple": true
+ },
+ "MAC": {
+ "misp-attribute": "mac-address",
+ "description": "Target MAC adress",
+ "ui-priority": 0,
+ "categories": ["Network activity"],
+ "multiple": true
+ },
+ "IP6": {
+ "misp-attribute": "ip-dst",
+ "description": "Target IPv6 address",
+ "ui-priority": 1,
+ "categories": ["Network activity"],
+ "multiple": true
+ },
+ "Port": {
+ "misp-attribute": "port",
+ "description": "Target port",
+ "ui-priority": 1,
+ "categories": ["Other"],
+ "multiple": true
+ },
+ "Proto": {
+ "misp-attribute": "text",
+ "description": "Name of used protocol",
+ "ui-priority": 1,
+ "categories": ["Network activity"],
+ "multiple": true,
+ "values_list": [
+ "tcp",
+ "udp",
+ "icmp",
+ "ip",
+ "arp",
+ "http",
+ "https",
+ "ftp"
+ ],
+ "required": false
+ },
+ "URL": {
+ "misp-attribute": "url",
+ "description": "Target URL",
+ "ui-priority": 0,
+ "categories": ["Network activity"],
+ "multiple": true
+ },
+ "Email": {
+ "misp-attribute": "email-dst",
+ "description": "Target email adress",
+ "ui-priority": 0,
+ "categories": ["Payload delivery"],
+ "multiple": true
+ },
+ "Note": {
+ "misp-attribute": "text",
+ "description": "Free text human readable additional note",
+ "ui-priority": 0,
+ "categories": ["Payload delivery"],
+ "multiple": false
+ },
+ "ASN": {
+ "misp-attribute": "AS",
+ "description": "Autonomous system number of this target",
+ "ui-priority": 0,
+ "categories": ["Network activity"],
+ "multiple": true
+ },
+ "Vulnerability": {
+ "misp-attribute": "vulnerability",
+ "description": "Reference to known source, related to vulnerability, specific to this target",
+ "ui-priority": 0,
+ "categories": ["External analysis"],
+ "multiple": true
+ },
+ "Reference": {
+ "misp-attribute": "other",
+ "description": "Reference to known source, related to attack, specific to this target",
+ "ui-priority": 0,
+ "categories": ["External analysis"],
+ "multiple": true
+ }
+ }
+}
\ No newline at end of file
--
GitLab