diff --git a/idea/misp.py b/idea/misp.py
index d9ebd956c1f9f577b7ded7bf52cfedc12ba24b07..a744ed2f2c2a095986bd6e027e5b6ef35135cca0 100644
--- a/idea/misp.py
+++ b/idea/misp.py
@@ -3,307 +3,557 @@
# Copyright (c) 2018, CESNET, z. s. p. o.
# Use of this source is governed by an ISC license, see LICENSE file.
+from pymisp import MISPEvent, MISPObject, NewAttributeError
from uuid import uuid4
import time
import itertools
from _datetime import datetime
-
-misp_to_idea_dictionary = {}
-
-# Every list on current index corresponds to idea_category on the same index
-misp_tags = [['circl:incident-classification="spam"'],
- ['veris:action:malware:variety="Exploit vuln"', 'veris:action:malware:variety="Backdoor"',
- 'ecsirt:intrusions="backdoor"'],
- ['circl:incident-classification="phishing"', 'europol-incident:information-gathering="phishing"',
- 'enisa:nefarious-activity-abuse="phishing-attacks"'],
- ['circl:incident-classification="scan"'],
- ['veris:action:malware:variety="Rootkit"'],
- ['europol-incident:availability="dos-ddos"', 'ms-caro-malware:malware-type="DDoS"'],
- ['circl:incident-classification="denial-of-service"'],
- ['circl:incident-classification="malware"']]
-
-idea_categories = ["Abusive.Spam", "Attempt.Exploit", "Fraud.Phishing", "Recon.Scanning", "Malware.Rootkit",
- "Availability.DDoS", "Availability.DoS", "Malware"]
-
-# prepare category translation dictionary
-for index, tag_list in enumerate(misp_tags):
- for tag in tag_list:
- misp_to_idea_dictionary[tag] = idea_categories[index]
-
-useful_hashes = ["md5", "sha1", "sha224", "sha256", "sha384", "sha512", "sha512/224", "sha512/256", "filename|md5",
- "filename|sha1", "filename|sha224", "filename|sha256", "filename|sha384", "filename|sha512",
- "filename|sha512/224", "filename|sha512/256"]
-
-useful_ip_objects = ["ip-port", "domain-ip", "netflow", "network-connection", "network-socket"]
-
-
-def get_date_from_timestamp(timestamp):
- date_and_time = convert_epoch_to_utc(timestamp)
- # returns only date from datetime, date stops at index 9 (2018-05-16T06:20:35Z)
- return date_and_time[0:9]
-
-
-def convert_epoch_to_utc(timestamp):
+import re
+
+
+category_to_taxonomy = {
+ 'Abusive.Spam': ['ecsirt:abusive-content="spam"', 'rsit:abusive-content="spam"'],
+ 'Abusive.Harassment': ['ecsirt:abusive-content="harmful-speech"', 'rsit:abusive-content="harmful-speech"'],
+ 'Abusive.Child': ['ecsirt:abusive-content="violence"', 'rsit:abusive-content="violence"'],
+ 'Abusive.Sexual': ['ecsirt:abusive-content="violence"', 'rsit:abusive-content="violence"'],
+ 'Abusive.Violence': ['ecsirt:abusive-content="violence"', 'rsit:abusive-content="violence"'],
+ 'Malware.Virus': ['ecsirt:malicious-code="virus"', 'rsit:malicious-code="virus"'],
+ 'Malware.Worm': ['ecsirt:malicious-code="worm"', 'rsit:malicious-code="worm"'],
+ 'Malware.Trojan': ['ecsirt:malicious-code="trojan"', 'rsit:malicious-code="trojan"'],
+ 'Malware.Spyware': ['ecsirt:malicious-code="spyware"', 'rsit:malicious-code="spyware"'],
+ 'Malware.Dialer': ['ecsirt:malicious-code="dialer"', 'rsit:malicious-code="dialer"'],
+ 'Malware.Rootkit': ['ecsirt:malicious-code="rootkit"', 'rsit:malicious-code="rootkit"'],
+ 'Recon.Scanning': ['ecsirt:information-gathering="scanner"', 'rsit:information-gathering="scanner"'],
+ 'Recon.Sniffing': ['ecsirt:information-gathering="sniffing"', 'rsit:information-gathering="sniffing"'],
+ 'Recon.SocialEngineering': ['ecsirt:information-gathering="social-engineering"',
+ 'rsit:information-gathering="social-engineering"'],
+ 'Recon.Searching': ['ecsirt:information-gathering', 'rsit:information-gathering'],
+ 'Attempt.Exploit': ['ecsirt:intrusion-attempts="exploit"', 'rsit:intrusion-attempts="exploit"'],
+ 'Attempt.Login': ['ecsirt:intrusion-attempts="brute-force"', 'rsit:intrusion-attempts="brute-force"'],
+ 'Attempt.NewSignature': ['ecsirt:intrusion-attempts', 'rsit:intrusion-attempts'],
+ 'Intrusion.AdminCompromise': ['ecsirt:intrusions="unprivileged-account-compromise"',
+ 'rsit:intrusions="unprivileged-account-compromise"'],
+ 'Intrusion.UserCompromise': ['ecsirt:intrusions="unprivileged-account-compromise"',
+ 'rsit:intrusions="unprivileged-account-compromise"'],
+ 'Intrusion.AppCompromise': ['ecsirt:intrusions="application-compromise"',
+ 'rsit:intrusions="application-compromise"'],
+ 'Intrusion.Botnet': ['ecsirt:intrusions="bot"', 'rsit:intrusions="bot"'],
+ 'Availability.DoS': ['ecsirt:availability="dos"', 'rsit:availability="dos"'],
+ 'Availability.DDoS': ['ecsirt:availability="ddos"', 'rsit:availability="ddos"'],
+ 'Availability.Sabotage': ['ecsirt:availability="sabotage"', 'rsit:availability="sabotage"'],
+ 'Availability.Outage': ['ecsirt:availability="outage"', 'rsit:availability="outage"'],
+ 'Information.UnauthorizedAccess': ['ecsirt:information-content-security="Unauthorised-information-access"',
+ 'rsit:information-content-security="Unauthorised-information-access"'],
+ 'Information.UnauthorizedModification':
+ ['ecsirt:information-content-security="Unauthorised-information-modification"',
+ 'rsit:information-content-security="Unauthorised-information-modification"'],
+ 'Fraud.UnauthorizedUsage': ['ecsirt:fraud="unauthorized-use-of-resources"',
+ 'rsit:fraud="unauthorized-use-of-resources"'],
+ 'Fraud.Copyright': ['ecsirt:fraud="copyright"', 'rsit:fraud="copyright"'],
+ 'Fraud.Masquerade': ['ecsirt:fraud="masquerade"', 'rsit:fraud="masquerade"'],
+ 'Fraud.Phishing': ['ecsirt:fraud="phishing"', 'ecsirt:fraud="phishing"'],
+ 'Fraud.Scam': ['ecsirt:fraud', 'rsit:fraud'],
+ 'Vulnerable.Open': ['ecsirt:vulnerable="vulnerable-service"', 'rsit:vulnerable="vulnerable-service"'],
+ 'Vulnerable.Config': ['ecsirt:vulnerable="vulnerable-service"', 'rsit:vulnerable="vulnerable-service"'],
+ 'Anomaly.Traffic': ['ecsirt:other="other"', 'rsit:other="other"'],
+ 'Anomaly.Connection': ['ecsirt:other="other"', 'rsit:other="other"'],
+ 'Anomaly.Protocol': ['ecsirt:other="other"', 'rsit:other="other"'],
+ 'Anomaly.System': ['ecsirt:other="other"', 'rsit:other="other"'],
+ 'Anomaly.Application': ['ecsirt:other="other"', 'rsit:other="other"'],
+ 'Anomaly.Behaviour': ['ecsirt:other="other"', 'rsit:other="other"'],
+ 'Other': ['ecsirt:other="other"', 'rsit:other="other"'],
+ 'Test': ['ecsirt:test="test"', 'rsit:test="test"']
+}
+
+
+class MispToIdea(object):
"""
- Converts Unix timestamp to to UTC datetime
- :param timestamp: Unix timestamp
- :return: UTC datetime in string
+ Converts MISP event in MISP core format to IDEA event
"""
- return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(int(timestamp)))
-
-def get_ip_version(ip_addr):
- """
- Gets verstion of IP address
- :param ip_addr: the IP address
- :return: version of IP address
- """
- return "IP6" if ":" in ip_addr else "IP4"
-
-
-def get_ip_addr(attrib, ip_only=True):
- """
- Gets IP address value from event's attribute
- :param attrib: the attribute
- :param ip_only: gets only IP address
- :return: returns IP address, role of IP address (Source|Target), port, hostname or IP address only if ip_only == True
- """
- role = "Source" if "src" in attrib['type'] else "Target"
- if "ip-src" == attrib['type'] or "ip-dst" == attrib['type']:
- return attrib['value'] if ip_only is True else (attrib['value'], role, None, None)
- elif "ip-src|port" == attrib['type'] or "ip-dst|port" == attrib['type']:
- split_attrib = attrib['value'].split('|')
- if len(split_attrib) == 1:
- split_attrib = attrib['value'].split(':')
- return split_attrib[0] if ip_only else (split_attrib[0], role, split_attrib[1], None)
- else:
- # attrib['type'] == "domain|ip"
- split_attrib = attrib['value'].split('|')
- return split_attrib[1] if ip_only else (split_attrib[1], "Target", None, split_attrib[0])
-
-
-# https://stackoverflow.com/questions/9807634/find-all-occurrences-of-a-key-in-nested-python-dictionaries-and-lists
-def find(key, value):
- """
- find gradually all values by key
- :param key: searched key
- :param value: searched object (dict|list)
- :return: found value by key
- """
- if isinstance(value, dict):
- for k, v in value.items():
- if k == key:
- yield v
- elif isinstance(v, dict):
- for result in find(key, v):
- yield result
- elif isinstance(v, list):
- for d in v:
- for result in find(key, d):
+ useful_hashes = ["md5", "sha1", "sha224", "sha256", "sha384", "sha512", "sha512/224", "sha512/256", "filename|md5",
+ "filename|sha1", "filename|sha224", "filename|sha256", "filename|sha384", "filename|sha512",
+ "filename|sha512/224", "filename|sha512/256"]
+ useful_ip_objects = ["ip-port", "domain-ip", "netflow", "network-connection", "network-socket"]
+ _taxonomy_to_category_created = False
+ taxonomy_to_category = {}
+
+ def __init__(self):
+ if not self._taxonomy_to_category_created:
+ # create taxonomy_to_category conversion dictionary
+ for idea_category, misp_taxonomies in category_to_taxonomy.items():
+ # take ECSIRT category and assign corresponding IDEA category
+ self.taxonomy_to_category[misp_taxonomies[0]] = idea_category
+ # take RSIT category and assign corresponding IDEA category
+ self.taxonomy_to_category[misp_taxonomies[1]] = idea_category
+ self._taxonomy_to_category_created = True
+
+ self._attach_counter = 0
+ self._source = []
+ self._target = []
+ self._attach = []
+
+ @staticmethod
+ def convert_epoch_to_utc(timestamp):
+ """
+ Converts Unix timestamp to to UTC datetime
+ :param timestamp: Unix timestamp
+ :return: UTC datetime in string
+ """
+ return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(int(timestamp)))
+
+ @staticmethod
+ def get_date_from_timestamp(timestamp):
+ date_and_time = __class__.convert_epoch_to_utc(timestamp)
+ # returns only date from datetime, date stops at index 9 (2018-05-16T06:20:35Z)
+ return date_and_time[0:9]
+
+ @staticmethod
+ def get_ip_version(ip_addr):
+ """
+ Gets verstion of IP address
+ :param ip_addr: the IP address
+ :return: version of IP address
+ """
+ return "IP6" if ":" in ip_addr else "IP4"
+
+ @staticmethod
+ def get_ip_addr(attrib, ip_only=True):
+ """
+ Gets IP address value from event's attribute
+ :param attrib: the attribute
+ :param ip_only: gets only IP address
+ :return: returns IP address, role of IP address (Source|Target), port, hostname or IP address only if
+ ip_only == True
+ """
+ role = "Source" if "src" in attrib['type'] else "Target"
+ if "ip-src" == attrib['type'] or "ip-dst" == attrib['type']:
+ return attrib['value'] if ip_only is True else (attrib['value'], role, None, None)
+ elif "ip-src|port" == attrib['type'] or "ip-dst|port" == attrib['type']:
+ split_attrib = attrib['value'].split('|')
+ if len(split_attrib) == 1:
+ split_attrib = attrib['value'].split(':')
+ return split_attrib[0] if ip_only else (split_attrib[0], role, split_attrib[1], None)
+ else:
+ # attrib['type'] == "domain|ip"
+ split_attrib = attrib['value'].split('|')
+ return split_attrib[1] if ip_only else (split_attrib[1], "Target", None, split_attrib[0])
+
+ @classmethod
+ # https://stackoverflow.com/questions/9807634/find-all-occurrences-of-a-key-in-nested-python-dictionaries-and-lists
+ def find(cls, key, value):
+ """
+ find gradually all values by key
+ :param key: searched key
+ :param value: searched object (dict|list)
+ :return: found value by key
+ """
+ if isinstance(value, dict):
+ for k, v in value.items():
+ if k == key:
+ yield v
+ elif isinstance(v, dict):
+ for result in cls.find(key, v):
yield result
- elif isinstance(value, list):
- for v in value:
- for result in find(key, v):
- yield result
+ elif isinstance(v, list):
+ for d in v:
+ for result in cls.find(key, d):
+ yield result
+ elif isinstance(value, list):
+ for v in value:
+ for result in cls.find(key, v):
+ yield result
+ def process_misp_attribute(self, attrib):
+ """
+ Process MISP attribute and get all useful data from it and insert it into corresponding IDEA object list
+ :param attrib: the attribute, which will be processed
+ :return: None
+ """
+ # if attribute contains ip address, get it with other potential attributes(port, hostname) and append it to
+ # Source or Target based on IP address type
+ if "ip" in attrib['type']:
+ ip_addr, role, port, hostname = self.get_ip_addr(attrib, False)
+ new_description = {self.get_ip_version(ip_addr): [ip_addr]}
+ if port:
+ new_description['Port'] = [int(port)]
+ if hostname:
+ new_description['Hostname'] = [hostname]
+ if role == "Source":
+ self._source.append(new_description)
+ if role == "Target":
+ self._target.append(new_description)
+
+ # if attrib is hash, create new attach
+ elif attrib['type'] in __class__.useful_hashes:
+ new_attach = {'Handle': "attach" + str(self._attach_counter)}
+ self._attach_counter += 1
+ if "|" in attrib['type']:
+ new_attach['FileName'] = [attrib['value'].split("|")[0]]
+ new_attach['Hash'] = [attrib['type'].split("|")[1] + ":" + attrib['value'].split("|")[1]]
+ else:
+ new_attach['Hash'] = [attrib['type'] + ":" + attrib['value']]
+ self._attach.append(new_attach)
+
+ elif attrib['type'] == "url":
+ new_attach_name = "attach" + str(self._attach_counter)
+ self._attach_counter += 1
+ self._attach.append({'Handle': new_attach_name, 'Ref': [attrib['value']]})
+
+ elif attrib['type'] == "email-src":
+ self._source.append({'Email': [attrib['value']]})
+
+ elif attrib['type'] == "email-dst":
+ self._target.append({'Email': [attrib['value']]})
+
+ @staticmethod
+ def append_value_or_create_list(key, value, idea_object):
+ """
+ Append value to array retrieved by key from idea_object, if key does not exist, create key in idea_object
+ and assign array with inserted value to this key
+ :param key: search key
+ :param value: appended value
+ :param idea_object: updated object
+ :return: None
+ """
+ try:
+ idea_object[key].append(value)
+ except KeyError:
+ idea_object[key] = [value]
+
+ def process_misp_object(self, misp_object):
+ """
+ Process MISP object and get all useful data from it and insert it into corresponding IDEA object array
+ :param misp_object: the object, which will be processed
+ :return: None
+ """
+ # if object name is in useful_ip_objects, all its attributes will be filled into source or target
+ if misp_object['name'] in __class__.useful_ip_objects:
+ source = {}
+ target = {}
+ proto_dict = {}
+ for attrib in misp_object['Attribute']:
+ if attrib['type'] == "ip-dst":
+ ip_addr = self.get_ip_addr(attrib)
+ self.append_value_or_create_list(self.get_ip_version(ip_addr), ip_addr, target)
+ elif attrib['type'] == "ip-src":
+ ip_addr = self.get_ip_addr(attrib)
+ self.append_value_or_create_list(self.get_ip_version(ip_addr), ip_addr, source)
+ elif attrib['type'] == "src-port" or attrib['object_relation'] == "src-port":
+ source['Port'] = [int(attrib['value'])]
+ elif attrib['type'] == "dst-port" or attrib['object_relation'] == "dst-port":
+ target['Port'] = [int(attrib['value'])]
+ elif attrib['object_relation'] == "hostname-src":
+ source['Hostname'] = [attrib['value']]
+ elif attrib['object_relation'] == "hostname-dst" or attrib['type'] == "hostname":
+ target['Hostname'] = [attrib['value']]
+ elif attrib['object_relation'] == "protocol":
+ source['Proto'] = [attrib['value']]
+ target['Proto'] = [attrib['value']]
+ elif attrib['object_relation'].startswith("layer"):
+ # means layer3-protocol, layer4-protocol or layer7-protocol
+ # save its value to number of layer
+ proto_dict[attrib['object_relation'][5]] = attrib['value']
+ elif attrib['type'] == "src-as" or attrib['object_relation'] == "src-as":
+ source['ASN'] = [int(attrib['value'])]
+ elif attrib['type'] == "dst-as" or attrib['object_relation'] == "dst-as":
+ target['ASN'] = [int(attrib['value'])]
+ elif attrib['type'] == "domain":
+ # won't overwrite previous value of hostname-dst, hostname-dst can occur in object of type
+ # network-connection and network-socket, but domain occurs only in other object types
+ target['Hostname'] = [attrib['value']]
+
+ if proto_dict:
+ # sort protocols by layer
+ proto_list = [proto_dict.get("3", "")] + [proto_dict.get("4", "")] + [proto_dict.get("7", "")]
+ # remove empty strings from proto list
+ proto_list = [proto for proto in proto_list if proto != ""]
+ source['Proto'] = proto_list
+ target['Proto'] = proto_list
+
+ if source:
+ self._source.append(source)
+ if target:
+ self._target.append(target)
+
+ elif misp_object['name'] == "file":
+ new_attach = {'Handle': "attach" + str(self._attach_counter)}
+ self._attach_counter += 1
+ for attrib in misp_object['Attribute']:
+ if attrib['type'] in __class__.useful_hashes:
+ try:
+ new_attach['Hash'].append(attrib['type'] + ":" + attrib['value'])
+ except KeyError:
+ new_attach['Hash'] = [attrib['type'] + ":" + attrib['value']]
+ elif attrib['type'] == "filename":
+ try:
+ new_attach['FileName'].append(attrib['value'])
+ except KeyError:
+ new_attach['FileName'] = [attrib['value']]
+ if new_attach.get('Hash') or new_attach.get('FileName'):
+ self._attach.append(new_attach)
+
+ def process_source_or_target_object(self, misp_object):
+ """
+ Process source or target object, and retrieved data insert to IDEA source or target object and append it to
+ corresponding object array
+ :param misp_object: the object, which will be processed
+ :return: None
+ """
+ converted_object = {}
+ for misp_attrib in misp_object['Attribute']:
+ if misp_attrib['object_relation'] not in ("Note", "Vulnerability", "Reference"):
+ self.append_value_or_create_list(misp_attrib['object_relation'], misp_attrib['value'], converted_object)
+ else:
+ if misp_attrib['object_relation'] == "Note":
+ converted_object['Note'] = misp_attrib['value']
+ elif misp_attrib['object_relation'] == "Vulnerability":
+ self.append_value_or_create_list("Ref", "cve:" + misp_attrib['value'],
+ converted_object)
+ elif misp_attrib['object_relation'] == "Reference":
+ self.append_value_or_create_list("Ref", misp_attrib['value'], converted_object)
+ if misp_object['name'] == "source":
+ self._source.append(converted_object)
+ else:
+ self._target.append(converted_object)
+
+ def process_attach_object(self, misp_object):
+ """
+ Process attach object, and retrieved data insert to IDEA attach object and append it to attach array
+ :param misp_object: the object, which will be processed
+ :return: None
+ """
+ converted_object = {"Handle": "att" + str(self._attach_counter)}
+ self._attach_counter += 1
+ for attach_attrib in misp_object['Attribute']:
+ if attach_attrib['object_relation'] in ("Note", "ContentType", "ContentCharset", "ContentEncoding",
+ "Content"):
+ converted_object[attach_attrib['object_relation']] = attach_attrib['value']
+ elif attach_attrib['object_relation'] == "FileName":
+ self.append_value_or_create_list("FileName", attach_attrib['value'], converted_object)
+ elif attach_attrib['object_relation'] == "Reference":
+ self.append_value_or_create_list("Ref", attach_attrib['value'], converted_object)
+ elif attach_attrib['object_relation'] == "Vulnerability":
+ self.append_value_or_create_list("Ref", "cve" + attach_attrib['value'], converted_object)
+ elif attach_attrib['object_relation'] == "Size":
+ converted_object['Size'] = int(attach_attrib['value'])
+ else:
+ self.append_value_or_create_list("Hash", attach_attrib['type'] + ":" + attach_attrib['value'],
+ converted_object)
+ self._attach.append(converted_object)
+
+ def to_idea(self, misp_event, idea_id=None, test=False, origdata=False):
+ """
+ Creates whole IDEA message from MISP event
+ :param misp_event: the misp event
+ :param idea_id: uuid of IDEA message (when needs to be preset)
+ :param test: add Test into IDEA['Category']
+ :param origdata: add original data to attachment
+ :return: new converted IDEA message
+ """
+ idea_event = {
+ 'Format': "IDEA0",
+ 'ID': str(idea_id) if idea_id is not None else str(uuid4()),
+ 'Category': [],
+ 'Description': misp_event['info']
+ }
+
+ # fill in the IDEA event Category
+ for tag in misp_event.get('Tag', []):
+ try:
+ new_category = self.taxonomy_to_category[tag['name']]
+ if new_category not in idea_event['Category']:
+ idea_event['Category'].append(new_category)
+ except KeyError:
+ pass
+
+ # cannot determine IDEA Category, cannot convert
+ if not idea_event['Category']:
+ return None
+ if test and "Test" not in idea_event['Category']:
+ idea_event['Category'].append("Test")
+
+ if not int(misp_event['publish_timestamp']):
+ idea_event['CreateTime'] = datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ")
+ else:
+ idea_event['CreateTime'] = self.convert_epoch_to_utc(misp_event['publish_timestamp'])
-def process_misp_attribute(attrib, attach_counter):
+ timestamps = itertools.chain(self.find('timestamp', misp_event['Attribute']),
+ self.find('timestamp', misp_event['Object']))
+ try:
+ oldest_timestamp = min(map(int, timestamps))
+ idea_event['DetectTime'] = self.convert_epoch_to_utc(oldest_timestamp)
+ except ValueError:
+ # no object and attributes --> min() ValueError for empty sequence
+ idea_event['DetectTime'] = misp_event['date'] + "T00:00:00Z"
+
+ # fill in info about organizations
+ idea_event['Node'] = [
+ {
+ 'Name': misp_event['Orgc']['name'],
+ 'Note': "MISP organization id (created event): " + misp_event['Orgc']['id']
+ },
+ {
+ 'Name': misp_event['Org']['name'],
+ 'Note': "MISP organization id (reported event): " + misp_event['Org']['id']
+ }]
+
+ # check all attributes for all potentially useful data
+ for attrib in misp_event['Attribute']:
+ self.process_misp_attribute(attrib)
+
+ # check all objects for all potentially useful data
+ for misp_object in misp_event['Object']:
+ if misp_object['name'] in ("source", "target"):
+ # process the object
+ self.process_source_or_target_object(misp_object)
+ elif misp_object['name'] == "attach":
+ self.process_attach_object(misp_object)
+ else:
+ self.process_misp_object(misp_object)
+
+ if origdata:
+ idea_event['Attach'].append({
+ 'Handle': "att" + str(self._attach_counter),
+ 'Note': "original data",
+ 'Content': misp_event
+ })
+
+ if self._source:
+ idea_event['Source'] = self._source
+ if self._target:
+ idea_event['Target'] = self._target
+ if self._attach:
+ idea_event['Attach'] = self._attach
+
+ # prepare instance for another conversion
+ self.__init__()
+
+ return idea_event
+
+
+class IdeaToMisp(object):
"""
- Process MISP attribute and get all useful data from it
- :param attrib: the attribute
- :param attach_counter: IDEA attachment counter
- :return: IDEA key (Source| Target | Attach) and its data
+ Converts IDEA event to MISP event in MISP core format
"""
- # if attribute contains ip address, get it with other potential attributes(port, hostname) and append it to
- # Source or Target based on IP address type
- if "ip" in attrib['type']:
- ip_addr, role, port, hostname = get_ip_addr(attrib, False)
- new_description = {get_ip_version(ip_addr): ip_addr}
- if port:
- new_description['Port'] = [int(port)]
- if hostname:
- new_description['Hostname'] = [hostname]
- return role, new_description
-
- # if attrib is hash, create new attach
- elif attrib['type'] in useful_hashes:
- new_attach = {'Handle': "attach" + str(attach_counter)}
- if "|" in attrib['type']:
- new_attach['Filename'] = [attrib['value'].split("|")[0]]
- new_attach['Hash'] = [attrib['type'].split("|")[1] + ":" + attrib['value'].split("|")[1]]
- else:
- new_attach['Hash'] = [attrib['type'] + ":" + attrib['value']]
- return "Attach", new_attach
- elif attrib['type'] == "url":
- return "Attach", {'Handle': "attach" + str(attach_counter), 'Ref': [attrib['value']]}
+ re_cve = re.compile("cve:", re.IGNORECASE)
+ severity_conversion = ["", "low", "medium", "high"]
+
+ def __init__(self):
+ """
+ Initializes class and prepares instance variables
+ """
+ self._idea_event = None
+ self._new_event = None
+
+ def process_basic_info(self):
+ """
+ Process basic info of MISP event such distribution, threat_level_id, info, Tags, and standalone attributes
+ :return: None
+ """
+ self._new_event = MISPEvent()
+ # add basic info about event
+ self._new_event.distribution = 1
- elif attrib['type'] == "email-src":
- return "Source", [attrib['value']]
-
- elif attrib['type'] == "email-dst":
- return "Target", [attrib['value']]
-
- return None, None
-
-
-def process_misp_object(misp_object, attach_counter):
- """
- Process MISP object and get all useful data from it
- :param misp_object: the object
- :param attach_counter: IDEA attachment counter
- :return: IDEA key (Source| Target | Attach) and its data
- """
- # if object name is in useful_ip_objects, all its attributes will be filled into source or target
- if misp_object['name'] in useful_ip_objects:
- source = {}
- target = {}
- proto_dict = {}
- for attrib in misp_object['Attribute']:
- if attrib['type'] == "ip-dst":
- ip_addr = get_ip_addr(attrib)
- target[get_ip_version(ip_addr)] = ip_addr
- elif attrib['type'] == "ip-src":
- ip_addr = get_ip_addr(attrib)
- source[get_ip_version(ip_addr)] = ip_addr
- elif attrib['type'] == "src-port" or attrib['object_relation'] == "src-port":
- source['Port'] = [int(attrib['value'])]
- elif attrib['type'] == "dst-port" or attrib['object_relation'] == "dst-port":
- target['Port'] = [int(attrib['value'])]
- elif attrib['type'] == "hostname-src":
- source['Hostname'] = [attrib['value']]
- elif attrib['type'] == "hostname-dst":
- target['Hostname'] = [attrib['value']]
- elif attrib['type'] == "protocol":
- source['Proto'] = [attrib['value']]
- target['Proto'] = [attrib['value']]
- elif attrib['type'].startswith("layer"):
- # means layer3-protocol, layer4-protocol or layer7-protocol
- # save its value to number of layer
- proto_dict[attrib['type'][5]] = attrib['value']
- elif attrib['type'] == "src-as" or attrib['object_relation'] == "src-as":
- source['ASN'] = [int(attrib['value'])]
- elif attrib['type'] == "dst-as" or attrib['object_relation'] == "dst-as":
- target['ASN'] = [int(attrib['value'])]
- elif attrib['type'] == "domain":
- # won't be overwrite previous value of hostname-dst, hostname-dst can occur in object of type
- # network-connection and network-socket, but domain occurs only in other object types
- target['Hostname'] = [attrib['value']]
-
- if proto_dict:
- # sort protocols by layer
- proto_list = [proto_dict.get("3", "")] + [proto_dict.get("4", "")] + [proto_dict.get("7", "")]
- # remove empty strings from proto list
- proto_list = [proto for proto in proto_list if proto != ""]
- source['Proto'] = proto_list
- target['Proto'] = proto_list
-
- if source:
- yield "Source", source
- if target:
- yield "Target", target
-
- elif misp_object['name'] == "file":
- new_attach = {'Handle': "attach" + str(attach_counter)}
- for attrib in misp_object['Attribute']:
- if attrib['type'] in useful_hashes:
- try:
- new_attach['Hash'].append(attrib['type'] + ":" + attrib['value'])
- except KeyError:
- new_attach['Hash'] = [attrib['type'] + ":" + attrib['value']]
- elif attrib['type'] == "filename":
- try:
- new_attach['Filename'].append(attrib['value'])
- except KeyError:
- new_attach['Filename'] = [attrib['value']]
- if new_attach.get('Hash') or new_attach.get('Filename'):
- yield "Attach", new_attach
- else:
- return None, None
-
-
-def to_idea(misp_event, idea_id=None, test=False, origdata=False):
- """
- Creates whole IDEA message from MISP event
- :param misp_event: the misp event
- :param idea_id: uuid of IDEA message (when needs to be preset)
- :param test: add Test into IDEA['Category']
- :param origdata: add original data to attachment
- :return: new converted IDEA message
- """
- idea_event = {
- 'Format': "IDEA0",
- 'ID': str(idea_id) if idea_id is not None else str(uuid4()),
- 'Category': [],
- 'Description': misp_event['info'],
- 'Source': [],
- 'Target': [],
- 'Attach': []
- }
-
- # fill in the IDEA event Category
- for tag in misp_event.get('Tag', []):
try:
- new_category = misp_to_idea_dictionary[tag['name']]
- if new_category not in idea_event['Category']:
- idea_event['Category'].append(new_category)
+ self._new_event.threat_level_id = __class__.severity_conversion.index(
+ self._idea_event['_CESNET']['EventSeverity'])
except KeyError:
- pass
-
- # cannot determine IDEA Category, cannot convert
- if not idea_event['Category']:
- return None
- if test:
- idea_event['Category'].append("Test")
-
- if not int(misp_event['publish_timestamp']):
- idea_event['CreateTime'] = datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ")
- else:
- idea_event['CreateTime'] = convert_epoch_to_utc(misp_event['publish_timestamp'])
-
- attach_counter = 0
-
- timestamps = itertools.chain(find('timestamp', misp_event['Attribute']),
- find('timestamp', misp_event['Object']))
- try:
- oldest_timestamp = min(map(int, timestamps))
- idea_event['DetectTime'] = convert_epoch_to_utc(oldest_timestamp)
- except ValueError:
- # no object and attributes --> min() ValueError for empty sequence
- idea_event['DetectTime'] = misp_event['date'] + "T00:00:00Z"
-
- # fill in info about organizations
- idea_event['Node'] = [
- {
- 'Name': misp_event['Orgc']['name'],
- 'Note': "MISP organization id (created event): " + misp_event['Orgc']['id']
- },
- {
- 'Name': misp_event['Org']['name'],
- 'Note': "MISP organization id (reported event): " + misp_event['Org']['id']
- }]
-
- # check all attributes for all potentially useful data
- for attrib in misp_event['Attribute']:
- key, new_data = process_misp_attribute(attrib, attach_counter)
- if key and new_data:
- idea_event[key].append(new_data)
- if key == "Attach":
- attach_counter += 1
-
- # check all objects for all potentially useful data
- for misp_object in misp_event['Object']:
- for key, new_data in process_misp_object(misp_object, attach_counter):
- idea_event[key].append(new_data)
- if key == "Attach":
- attach_counter += 1
-
- if origdata:
- idea_event['Attach'].append({
- 'Handle': "att" + str(attach_counter),
- 'Note': "original data",
- 'Content': misp_event
- })
- attach_counter += 1
-
- return idea_event
+ # EventSeverity not in IDEA message --> 0 = Undefined
+ self._new_event.threat_level_id = 1
+
+ self._new_event.analysis = 2
+ # [0:10] is for taking date only from DetectTime (2019-01-26xxxxxxx)
+ self._new_event.date = datetime.strptime(self._idea_event['DetectTime'][0:10], "%Y-%m-%d")
+
+ if self._idea_event.get('Description'):
+ # add Description as event info and add Note as well if present
+ try:
+ self._new_event.info = self._idea_event['Description'] + ". " + self._idea_event['Note']
+ except KeyError:
+ self._new_event.info = self._idea_event['Description']
+ elif self._idea_event.get('Note'):
+ # or atleast Note if Description is not present
+ self._new_event.info = self._idea_event['Note']
+
+ # convert all categories in IDEA to taxonomies
+ for idea_category in self._idea_event['Category']:
+ # add both taxonomy ECSIRT and RSIT
+ self._new_event.add_tag(category_to_taxonomy[idea_category][0])
+ self._new_event.add_tag(category_to_taxonomy[idea_category][1])
+
+ for reference in self._idea_event.get('Ref', []):
+ # if reference is cve, add MISP attribute as vulnerability, otherwise add MISP attribute as link
+ if __class__.re_cve.search(reference):
+ self._new_event.add_attribute(category="External analysis", type="vulnerability",
+ value=__class__.re_cve.split(reference)[1])
+ else:
+ self._new_event.add_attribute(category="External analysis", type="link",
+ value=reference)
+
+ @classmethod
+ def add_attribute_to_object(cls, key, value, misp_object):
+ """
+ Add one value (new MISP attribute) to source or target or attach object
+ :param key: name of attribute relation
+ :param value: inserted value
+ :param misp_object: object, which will the attribute be inserted into
+ :return: the object with inserted attribute
+ """
+ try:
+ misp_object.add_attribute(key, value=value)
+ except NewAttributeError:
+ if key == "Ref":
+ if __class__.re_cve.search(value):
+ misp_object.add_attribute("Vulnerability", value=cls.re_cve.split(value)[1])
+ else:
+ misp_object.add_attribute("Reference", value=value)
+ if misp_object.name == "attach" and key == "Hash":
+ hash_method = value.split(":", 1)[0]
+ # FIXME may not be fixed properly
+ hash_value = value.split(":", 1)[1]
+ misp_object.add_attribute(hash_method.lower(), value=hash_value)
+ return misp_object
+
+ def process_one_idea_object(self, object_name):
+ """
+ Process IDEA's Source or Target or Attach and convert it into MISP object
+ :param object_name: defines processing of IDEA's 'Source' or 'Target' or 'Attach'
+ :return: None
+ """
+ for idea_obj in self._idea_event.get(object_name, []):
+ new_object = MISPObject(name=object_name.lower(), strict=True, standalone=False,
+ misp_objects_path_custom='object_templates')
+ for key, value in idea_obj.items():
+ if isinstance(value, list):
+ for list_value in value:
+ new_object = self.add_attribute_to_object(key, list_value, new_object)
+ else:
+ new_object = self.add_attribute_to_object(key, value, new_object)
+
+ self._new_event.add_object(new_object)
+
+ def process_all_idea_objects(self):
+ """
+ Calls processing of Source, Target and Attach objects
+ :return: None
+ """
+ self.process_one_idea_object('Source')
+ self.process_one_idea_object('Target')
+ self.process_one_idea_object('Attach')
+
+ def to_misp(self, idea_event):
+ """
+ Converts IDEA message into MISP core format message
+ :param idea_event: the idea event, which will be converted
+ :return: instance of MISPEvent (with converted data)
+ """
+ self._idea_event = idea_event
+ # fill new event with basic info and set event as published
+ self.process_basic_info()
+ self._new_event.publish()
+
+ self.process_all_idea_objects()
+
+ return self._new_event
diff --git a/misp_object_templates/attach/definition.json b/misp_object_templates/attach/definition.json
new file mode 100644
index 0000000000000000000000000000000000000000..b559e02995a05f711da11676d6a9d13c4ba0abb4
--- /dev/null
+++ b/misp_object_templates/attach/definition.json
@@ -0,0 +1,121 @@
+{
+ "name": "attach",
+ "meta-category": "misc",
+ "description": "Event attachment",
+ "uuid": "f5a964ac-5782-4c3e-8056-9b2783c987a8",
+ "version": 1,
+ "attributes": {
+ "FileName": {
+ "misp-attribute": "filename",
+ "description": "Name of file",
+ "ui-priority": 1,
+ "categories": ["Payload delivery"],
+ "multiple": true
+ },
+ "md5": {
+ "misp-attribute": "md5",
+ "description": "[Insecure] MD5 hash (128 bits)",
+ "ui-priority": 0,
+ "categories": ["Payload delivery"],
+ "multiple": true
+ },
+ "sha1": {
+ "misp-attribute": "sha1",
+ "description": "[Insecure] Secure Hash Algorithm 1 (160 bits)",
+ "ui-priority": 0,
+ "categories": ["Payload delivery"],
+ "multiple": true
+ },
+ "sha224": {
+ "misp-attribute": "sha224",
+ "description": "Secure Hash Algorithm 2 (224 bits)",
+ "ui-priority": 0,
+ "categories": ["Payload delivery"],
+ "multiple": true
+ },
+ "sha256": {
+ "misp-attribute": "sha256",
+ "description": "Secure Hash Algorithm 2 (256 bits)",
+ "ui-priority": 0,
+ "categories": ["Payload delivery"],
+ "multiple": true
+ },
+ "sha384": {
+ "misp-attribute": "sha384",
+ "description": "Secure Hash Algorithm 2 (384 bits)",
+ "ui-priority": 0,
+ "categories": ["Payload delivery"],
+ "multiple": true
+ },
+ "sha512": {
+ "misp-attribute": "sha512",
+ "description": "Secure Hash Algorithm 2 (512 bits)",
+ "ui-priority": 0,
+ "categories": ["Payload delivery"],
+ "multiple": true
+ },
+ "sha512/224": {
+ "misp-attribute": "sha512/224",
+ "description": "Secure Hash Algorithm 2 (224 bits)",
+ "ui-priority": 0,
+ "categories": ["Payload delivery"],
+ "multiple": true
+ },
+ "Size": {
+ "misp-attribute": "size-in-bytes",
+ "description": "Length of the content",
+ "ui-priority": 0,
+ "categories": ["Other"],
+ "multiple": false
+ },
+ "Vulnerability": {
+ "misp-attribute": "vulnerability",
+ "description": "Reference to known source, related to vulnerability, specific to this attachment",
+ "ui-priority": 0,
+ "categories": ["External analysis"],
+ "multiple": true
+ },
+ "Reference": {
+ "misp-attribute": "other",
+ "description": "Reference to known source, related to attack, specific to this attachment",
+ "ui-priority": 0,
+ "categories": ["External analysis"],
+ "multiple": true
+ },
+ "Note": {
+ "misp-attribute": "text",
+ "description": "Free text human readable additional note",
+ "ui-priority": 0,
+ "categories": ["Payload delivery"],
+ "multiple": false
+ },
+ "ContentType": {
+ "misp-attribute": "text",
+ "description": "Internet Media Type of the attachment",
+ "ui-priority": 1,
+ "categories": ["Payload delivery"],
+ "multiple": false
+ },
+ "ContentCharset": {
+ "misp-attribute": "text",
+ "description": "Name of the content character set",
+ "ui-priority": 0,
+ "categories": ["Payload delivery"],
+ "multiple": false
+ },
+ "ContentEncoding": {
+ "misp-attribute": "text",
+ "description": "Encoding of the content",
+ "ui-priority": 0,
+ "categories": ["Payload delivery"],
+ "multiple": false
+ },
+ "Content": {
+ "misp-attribute": "text",
+ "description": "Attachment content",
+ "ui-priority": 1,
+ "categories": ["Payload delivery"],
+ "multiple": false
+ }
+ }
+}
\ No newline at end of file
diff --git a/misp_object_templates/source/definition.json b/misp_object_templates/source/definition.json
new file mode 100644
index 0000000000000000000000000000000000000000..1df52b3d787e58446795d09ccc4c913063488be6
--- /dev/null
+++ b/misp_object_templates/source/definition.json
@@ -0,0 +1,104 @@
+{
+ "name": "source",
+ "meta-category": "network",
+ "description": "Description of the source of the event",
+ "uuid": "63cf1c78-4afe-49be-baff-2c101a942000",
+ "version": 1,
+ "attributes": {
+ "Hostname": {
+ "misp-attribute": "hostname",
+ "description": "Participating hostname",
+ "ui-priority": 0,
+ "categories": ["Network activity"],
+ "multiple": true
+ },
+ "IP4": {
+ "misp-attribute": "ip-src",
+ "description": "Source IPv4 address",
+ "ui-priority": 1,
+ "categories": ["Network activity"],
+ "multiple": true
+ },
+ "MAC": {
+ "misp-attribute": "mac-address",
+ "description": "Source MAC adress",
+ "ui-priority": 0,
+ "categories": ["Network activity"],
+ "multiple": true
+ },
+ "IP6": {
+ "misp-attribute": "ip-src",
+ "description": "Source IPv6 address",
+ "ui-priority": 1,
+ "categories": ["Network activity"],
+ "multiple": true
+ },
+ "Port": {
+ "misp-attribute": "port",
+ "description": "Source port",
+ "ui-priority": 1,
+ "categories": ["Other"],
+ "multiple": true
+ },
+ "Proto": {
+ "misp-attribute": "text",
+ "description": "Name of used protocol",
+ "ui-priority": 1,
+ "categories": ["Network activity"],
+ "multiple": true,
+ "values_list": [
+ "tcp",
+ "udp",
+ "icmp",
+ "ip",
+ "arp",
+ "http",
+ "https",
+ "ftp"
+ ],
+ "required": false
+ },
+ "URL": {
+ "misp-attribute": "url",
+ "description": "Source URL",
+ "ui-priority": 0,
+ "categories": ["Network activity"],
+ "multiple": true
+ },
+ "Email": {
+ "misp-attribute": "email-src",
+ "description": "Source email adress",
+ "ui-priority": 0,
+ "categories": ["Payload delivery"],
+ "multiple": true
+ },
+ "Note": {
+ "misp-attribute": "text",
+ "description": "Free text human readable additional note",
+ "ui-priority": 0,
+ "categories": ["Payload delivery"],
+ "multiple": false
+ },
+ "ASN": {
+ "misp-attribute": "AS",
+ "description": "Autonomous system number of this source",
+ "ui-priority": 0,
+ "categories": ["Network activity"],
+ "multiple": true
+ },
+ "Vulnerability": {
+ "misp-attribute": "vulnerability",
+ "description": "Reference to known source, related to vulnerability, specific to this source",
+ "ui-priority": 0,
+ "categories": ["External analysis"],
+ "multiple": true
+ },
+ "Reference": {
+ "misp-attribute": "other",
+ "description": "Reference to known source, related to attack, specific to this source",
+ "ui-priority": 0,
+ "categories": ["External analysis"],
+ "multiple": true
+ }
+ }
+}
\ No newline at end of file
diff --git a/misp_object_templates/target/definition.json b/misp_object_templates/target/definition.json
new file mode 100644
index 0000000000000000000000000000000000000000..7336da9829da9016af34f6c0e18167e927ca1224
--- /dev/null
+++ b/misp_object_templates/target/definition.json
@@ -0,0 +1,104 @@
+{
+ "name": "target",
+ "meta-category": "network",
+ "description": "Description of the target of the event",
+ "uuid": "b251672f-3463-47d6-a20f-19d04c383195",
+ "version": 1,
+ "attributes": {
+ "Hostname": {
+ "misp-attribute": "hostname",
+ "description": "Participating hostname",
+ "ui-priority": 0,
+ "categories": ["Network activity"],
+ "multiple": true
+ },
+ "IP4": {
+ "misp-attribute": "ip-dst",
+ "description": "Target IPv4 address",
+ "ui-priority": 1,
+ "categories": ["Network activity"],
+ "multiple": true
+ },
+ "MAC": {
+ "misp-attribute": "mac-address",
+ "description": "Target MAC adress",
+ "ui-priority": 0,
+ "categories": ["Network activity"],
+ "multiple": true
+ },
+ "IP6": {
+ "misp-attribute": "ip-dst",
+ "description": "Target IPv6 address",
+ "ui-priority": 1,
+ "categories": ["Network activity"],
+ "multiple": true
+ },
+ "Port": {
+ "misp-attribute": "port",
+ "description": "Target port",
+ "ui-priority": 1,
+ "categories": ["Other"],
+ "multiple": true
+ },
+ "Proto": {
+ "misp-attribute": "text",
+ "description": "Name of used protocol",
+ "ui-priority": 1,
+ "categories": ["Network activity"],
+ "multiple": true,
+ "values_list": [
+ "tcp",
+ "udp",
+ "icmp",
+ "ip",
+ "arp",
+ "http",
+ "https",
+ "ftp"
+ ],
+ "required": false
+ },
+ "URL": {
+ "misp-attribute": "url",
+ "description": "Target URL",
+ "ui-priority": 0,
+ "categories": ["Network activity"],
+ "multiple": true
+ },
+ "Email": {
+ "misp-attribute": "email-dst",
+ "description": "Target email adress",
+ "ui-priority": 0,
+ "categories": ["Payload delivery"],
+ "multiple": true
+ },
+ "Note": {
+ "misp-attribute": "text",
+ "description": "Free text human readable additional note",
+ "ui-priority": 0,
+ "categories": ["Payload delivery"],
+ "multiple": false
+ },
+ "ASN": {
+ "misp-attribute": "AS",
+ "description": "Autonomous system number of this target",
+ "ui-priority": 0,
+ "categories": ["Network activity"],
+ "multiple": true
+ },
+ "Vulnerability": {
+ "misp-attribute": "vulnerability",
+ "description": "Reference to known source, related to vulnerability, specific to this target",
+ "ui-priority": 0,
+ "categories": ["External analysis"],
+ "multiple": true
+ },
+ "Reference": {
+ "misp-attribute": "other",
+ "description": "Reference to known source, related to attack, specific to this target",
+ "ui-priority": 0,
+ "categories": ["External analysis"],
+ "multiple": true
+ }
+ }
+}
\ No newline at end of file