Skip to content
Snippets Groups Projects
Commit 1184eb55 authored by Pavel Eis's avatar Pavel Eis
Browse files

Update of misp.py converter 

MISP to IDEA converted moved to class MispToIdea, some semantic
conversion fixes (Email in Source is array etc.). Now using RSIT and
ECSIRT taxonomy, mapping table can be found in 'category_to_taxonomy'
variable. Added support of custom objects, which were defined for
IDEA to MISP conversion. Conversion can be made by calling method
to_idea on MispToIdea instance.

Also IDEA to MISP conversion finished and placed to IdeaToMisp class.
Conversion is using custom object templates, which can be found in
/misp_object_templates. Conversion can be made by calling method
to_misp on IdeaToMisp instance.

Added custom object templates to /misp_object_templates. Their name
will probably change in the future.
parent 8d8d4e51
No related branches found
No related tags found
No related merge requests found
Show whitespace changes
Inline Side-by-side
idea/misp.py
+ 538
− 288
View file @ 1184eb55
...@@ -3,46 +3,97 @@ ...@@ -3,46 +3,97 @@
# Copyright (c) 2018, CESNET, z. s. p. o. # Copyright (c) 2018, CESNET, z. s. p. o.
# Use of this source is governed by an ISC license, see LICENSE file. # Use of this source is governed by an ISC license, see LICENSE file.
from pymisp import MISPEvent, MISPObject, NewAttributeError
from uuid import uuid4 from uuid import uuid4
import time import time
import itertools import itertools
from _datetime import datetime from _datetime import datetime
import re
category_to_taxonomy = {
'Abusive.Spam': ['ecsirt:abusive-content="spam"', 'rsit:abusive-content="spam"'],
'Abusive.Harassment': ['ecsirt:abusive-content="harmful-speech"', 'rsit:abusive-content="harmful-speech"'],
'Abusive.Child': ['ecsirt:abusive-content="violence"', 'rsit:abusive-content="violence"'],
'Abusive.Sexual': ['ecsirt:abusive-content="violence"', 'rsit:abusive-content="violence"'],
'Abusive.Violence': ['ecsirt:abusive-content="violence"', 'rsit:abusive-content="violence"'],
'Malware.Virus': ['ecsirt:malicious-code="virus"', 'rsit:malicious-code="virus"'],
'Malware.Worm': ['ecsirt:malicious-code="worm"', 'rsit:malicious-code="worm"'],
'Malware.Trojan': ['ecsirt:malicious-code="trojan"', 'rsit:malicious-code="trojan"'],
'Malware.Spyware': ['ecsirt:malicious-code="spyware"', 'rsit:malicious-code="spyware"'],
'Malware.Dialer': ['ecsirt:malicious-code="dialer"', 'rsit:malicious-code="dialer"'],
'Malware.Rootkit': ['ecsirt:malicious-code="rootkit"', 'rsit:malicious-code="rootkit"'],
'Recon.Scanning': ['ecsirt:information-gathering="scanner"', 'rsit:information-gathering="scanner"'],
'Recon.Sniffing': ['ecsirt:information-gathering="sniffing"', 'rsit:information-gathering="sniffing"'],
'Recon.SocialEngineering': ['ecsirt:information-gathering="social-engineering"',
'rsit:information-gathering="social-engineering"'],
'Recon.Searching': ['ecsirt:information-gathering', 'rsit:information-gathering'],
'Attempt.Exploit': ['ecsirt:intrusion-attempts="exploit"', 'rsit:intrusion-attempts="exploit"'],
'Attempt.Login': ['ecsirt:intrusion-attempts="brute-force"', 'rsit:intrusion-attempts="brute-force"'],
'Attempt.NewSignature': ['ecsirt:intrusion-attempts', 'rsit:intrusion-attempts'],
'Intrusion.AdminCompromise': ['ecsirt:intrusions="unprivileged-account-compromise"',
'rsit:intrusions="unprivileged-account-compromise"'],
'Intrusion.UserCompromise': ['ecsirt:intrusions="unprivileged-account-compromise"',
'rsit:intrusions="unprivileged-account-compromise"'],
'Intrusion.AppCompromise': ['ecsirt:intrusions="application-compromise"',
'rsit:intrusions="application-compromise"'],
'Intrusion.Botnet': ['ecsirt:intrusions="bot"', 'rsit:intrusions="bot"'],
'Availability.DoS': ['ecsirt:availability="dos"', 'rsit:availability="dos"'],
'Availability.DDoS': ['ecsirt:availability="ddos"', 'rsit:availability="ddos"'],
'Availability.Sabotage': ['ecsirt:availability="sabotage"', 'rsit:availability="sabotage"'],
'Availability.Outage': ['ecsirt:availability="outage"', 'rsit:availability="outage"'],
'Information.UnauthorizedAccess': ['ecsirt:information-content-security="Unauthorised-information-access"',
'rsit:information-content-security="Unauthorised-information-access"'],
'Information.UnauthorizedModification':
['ecsirt:information-content-security="Unauthorised-information-modification"',
'rsit:information-content-security="Unauthorised-information-modification"'],
'Fraud.UnauthorizedUsage': ['ecsirt:fraud="unauthorized-use-of-resources"',
'rsit:fraud="unauthorized-use-of-resources"'],
'Fraud.Copyright': ['ecsirt:fraud="copyright"', 'rsit:fraud="copyright"'],
'Fraud.Masquerade': ['ecsirt:fraud="masquerade"', 'rsit:fraud="masquerade"'],
'Fraud.Phishing': ['ecsirt:fraud="phishing"', 'ecsirt:fraud="phishing"'],
'Fraud.Scam': ['ecsirt:fraud', 'rsit:fraud'],
'Vulnerable.Open': ['ecsirt:vulnerable="vulnerable-service"', 'rsit:vulnerable="vulnerable-service"'],
'Vulnerable.Config': ['ecsirt:vulnerable="vulnerable-service"', 'rsit:vulnerable="vulnerable-service"'],
'Anomaly.Traffic': ['ecsirt:other="other"', 'rsit:other="other"'],
'Anomaly.Connection': ['ecsirt:other="other"', 'rsit:other="other"'],
'Anomaly.Protocol': ['ecsirt:other="other"', 'rsit:other="other"'],
'Anomaly.System': ['ecsirt:other="other"', 'rsit:other="other"'],
'Anomaly.Application': ['ecsirt:other="other"', 'rsit:other="other"'],
'Anomaly.Behaviour': ['ecsirt:other="other"', 'rsit:other="other"'],
'Other': ['ecsirt:other="other"', 'rsit:other="other"'],
'Test': ['ecsirt:test="test"', 'rsit:test="test"']
}
misp_to_idea_dictionary = {}
# Every list on current index corresponds to idea_category on the same index
misp_tags = [['circl:incident-classification="spam"'],
['veris:action:malware:variety="Exploit vuln"', 'veris:action:malware:variety="Backdoor"',
'ecsirt:intrusions="backdoor"'],
['circl:incident-classification="phishing"', 'europol-incident:information-gathering="phishing"',
'enisa:nefarious-activity-abuse="phishing-attacks"'],
['circl:incident-classification="scan"'],
['veris:action:malware:variety="Rootkit"'],
['europol-incident:availability="dos-ddos"', 'ms-caro-malware:malware-type="DDoS"'],
['circl:incident-classification="denial-of-service"'],
['circl:incident-classification="malware"']]
idea_categories = ["Abusive.Spam", "Attempt.Exploit", "Fraud.Phishing", "Recon.Scanning", "Malware.Rootkit",
"Availability.DDoS", "Availability.DoS", "Malware"]
# prepare category translation dictionary class MispToIdea(object):
for index, tag_list in enumerate(misp_tags): """
for tag in tag_list: Converts MISP event in MISP core format to IDEA event
misp_to_idea_dictionary[tag] = idea_categories[index] """
useful_hashes = ["md5", "sha1", "sha224", "sha256", "sha384", "sha512", "sha512/224", "sha512/256", "filename|md5", useful_hashes = ["md5", "sha1", "sha224", "sha256", "sha384", "sha512", "sha512/224", "sha512/256", "filename|md5",
"filename|sha1", "filename|sha224", "filename|sha256", "filename|sha384", "filename|sha512", "filename|sha1", "filename|sha224", "filename|sha256", "filename|sha384", "filename|sha512",
"filename|sha512/224", "filename|sha512/256"] "filename|sha512/224", "filename|sha512/256"]
useful_ip_objects = ["ip-port", "domain-ip", "netflow", "network-connection", "network-socket"] useful_ip_objects = ["ip-port", "domain-ip", "netflow", "network-connection", "network-socket"]
_taxonomy_to_category_created = False
taxonomy_to_category = {}
def get_date_from_timestamp(timestamp):
date_and_time = convert_epoch_to_utc(timestamp) def __init__(self):
# returns only date from datetime, date stops at index 9 (2018-05-16T06:20:35Z) if not self._taxonomy_to_category_created:
return date_and_time[0:9] # create taxonomy_to_category conversion dictionary
for idea_category, misp_taxonomies in category_to_taxonomy.items():
# take ECSIRT category and assign corresponding IDEA category
self.taxonomy_to_category[misp_taxonomies[0]] = idea_category
# take RSIT category and assign corresponding IDEA category
self.taxonomy_to_category[misp_taxonomies[1]] = idea_category
self._taxonomy_to_category_created = True
self._attach_counter = 0
self._source = []
self._target = []
self._attach = []
@staticmethod
def convert_epoch_to_utc(timestamp): def convert_epoch_to_utc(timestamp):
""" """
Converts Unix timestamp to to UTC datetime Converts Unix timestamp to to UTC datetime
...@@ -51,7 +102,13 @@ def convert_epoch_to_utc(timestamp): ...@@ -51,7 +102,13 @@ def convert_epoch_to_utc(timestamp):
""" """
return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(int(timestamp))) return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(int(timestamp)))
@staticmethod
def get_date_from_timestamp(timestamp):
date_and_time = __class__.convert_epoch_to_utc(timestamp)
# returns only date from datetime, date stops at index 9 (2018-05-16T06:20:35Z)
return date_and_time[0:9]
@staticmethod
def get_ip_version(ip_addr): def get_ip_version(ip_addr):
""" """
Gets verstion of IP address Gets verstion of IP address
...@@ -60,13 +117,14 @@ def get_ip_version(ip_addr): ...@@ -60,13 +117,14 @@ def get_ip_version(ip_addr):
""" """
return "IP6" if ":" in ip_addr else "IP4" return "IP6" if ":" in ip_addr else "IP4"
@staticmethod
def get_ip_addr(attrib, ip_only=True): def get_ip_addr(attrib, ip_only=True):
""" """
Gets IP address value from event's attribute Gets IP address value from event's attribute
:param attrib: the attribute :param attrib: the attribute
:param ip_only: gets only IP address :param ip_only: gets only IP address
:return: returns IP address, role of IP address (Source|Target), port, hostname or IP address only if ip_only == True :return: returns IP address, role of IP address (Source|Target), port, hostname or IP address only if
ip_only == True
""" """
role = "Source" if "src" in attrib['type'] else "Target" role = "Source" if "src" in attrib['type'] else "Target"
if "ip-src" == attrib['type'] or "ip-dst" == attrib['type']: if "ip-src" == attrib['type'] or "ip-dst" == attrib['type']:
...@@ -81,9 +139,9 @@ def get_ip_addr(attrib, ip_only=True): ...@@ -81,9 +139,9 @@ def get_ip_addr(attrib, ip_only=True):
split_attrib = attrib['value'].split('|') split_attrib = attrib['value'].split('|')
return split_attrib[1] if ip_only else (split_attrib[1], "Target", None, split_attrib[0]) return split_attrib[1] if ip_only else (split_attrib[1], "Target", None, split_attrib[0])
@classmethod
# https://stackoverflow.com/questions/9807634/find-all-occurrences-of-a-key-in-nested-python-dictionaries-and-lists # https://stackoverflow.com/questions/9807634/find-all-occurrences-of-a-key-in-nested-python-dictionaries-and-lists
def find(key, value): def find(cls, key, value):
""" """
find gradually all values by key find gradually all values by key
:param key: searched key :param key: searched key
...@@ -95,98 +153,113 @@ def find(key, value): ...@@ -95,98 +153,113 @@ def find(key, value):
if k == key: if k == key:
yield v yield v
elif isinstance(v, dict): elif isinstance(v, dict):
for result in find(key, v): for result in cls.find(key, v):
yield result yield result
elif isinstance(v, list): elif isinstance(v, list):
for d in v: for d in v:
for result in find(key, d): for result in cls.find(key, d):
yield result yield result
elif isinstance(value, list): elif isinstance(value, list):
for v in value: for v in value:
for result in find(key, v): for result in cls.find(key, v):
yield result yield result
def process_misp_attribute(self, attrib):
def process_misp_attribute(attrib, attach_counter):
""" """
Process MISP attribute and get all useful data from it Process MISP attribute and get all useful data from it and insert it into corresponding IDEA object list
:param attrib: the attribute :param attrib: the attribute, which will be processed
:param attach_counter: IDEA attachment counter :return: None
:return: IDEA key (Source| Target | Attach) and its data
""" """
# if attribute contains ip address, get it with other potential attributes(port, hostname) and append it to # if attribute contains ip address, get it with other potential attributes(port, hostname) and append it to
# Source or Target based on IP address type # Source or Target based on IP address type
if "ip" in attrib['type']: if "ip" in attrib['type']:
ip_addr, role, port, hostname = get_ip_addr(attrib, False) ip_addr, role, port, hostname = self.get_ip_addr(attrib, False)
new_description = {get_ip_version(ip_addr): ip_addr} new_description = {self.get_ip_version(ip_addr): [ip_addr]}
if port: if port:
new_description['Port'] = [int(port)] new_description['Port'] = [int(port)]
if hostname: if hostname:
new_description['Hostname'] = [hostname] new_description['Hostname'] = [hostname]
return role, new_description if role == "Source":
self._source.append(new_description)
if role == "Target":
self._target.append(new_description)
# if attrib is hash, create new attach # if attrib is hash, create new attach
elif attrib['type'] in useful_hashes: elif attrib['type'] in __class__.useful_hashes:
new_attach = {'Handle': "attach" + str(attach_counter)} new_attach = {'Handle': "attach" + str(self._attach_counter)}
self._attach_counter += 1
if "|" in attrib['type']: if "|" in attrib['type']:
new_attach['Filename'] = [attrib['value'].split("|")[0]] new_attach['FileName'] = [attrib['value'].split("|")[0]]
new_attach['Hash'] = [attrib['type'].split("|")[1] + ":" + attrib['value'].split("|")[1]] new_attach['Hash'] = [attrib['type'].split("|")[1] + ":" + attrib['value'].split("|")[1]]
else: else:
new_attach['Hash'] = [attrib['type'] + ":" + attrib['value']] new_attach['Hash'] = [attrib['type'] + ":" + attrib['value']]
return "Attach", new_attach self._attach.append(new_attach)
elif attrib['type'] == "url": elif attrib['type'] == "url":
return "Attach", {'Handle': "attach" + str(attach_counter), 'Ref': [attrib['value']]} new_attach_name = "attach" + str(self._attach_counter)
self._attach_counter += 1
self._attach.append({'Handle': new_attach_name, 'Ref': [attrib['value']]})
elif attrib['type'] == "email-src": elif attrib['type'] == "email-src":
return "Source", [attrib['value']] self._source.append({'Email': [attrib['value']]})
elif attrib['type'] == "email-dst": elif attrib['type'] == "email-dst":
return "Target", [attrib['value']] self._target.append({'Email': [attrib['value']]})
return None, None
@staticmethod
def append_value_or_create_list(key, value, idea_object):
"""
Append value to array retrieved by key from idea_object, if key does not exist, create key in idea_object
and assign array with inserted value to this key
:param key: search key
:param value: appended value
:param idea_object: updated object
:return: None
"""
try:
idea_object[key].append(value)
except KeyError:
idea_object[key] = [value]
def process_misp_object(misp_object, attach_counter): def process_misp_object(self, misp_object):
""" """
Process MISP object and get all useful data from it Process MISP object and get all useful data from it and insert it into corresponding IDEA object array
:param misp_object: the object :param misp_object: the object, which will be processed
:param attach_counter: IDEA attachment counter :return: None
:return: IDEA key (Source| Target | Attach) and its data
""" """
# if object name is in useful_ip_objects, all its attributes will be filled into source or target # if object name is in useful_ip_objects, all its attributes will be filled into source or target
if misp_object['name'] in useful_ip_objects: if misp_object['name'] in __class__.useful_ip_objects:
source = {} source = {}
target = {} target = {}
proto_dict = {} proto_dict = {}
for attrib in misp_object['Attribute']: for attrib in misp_object['Attribute']:
if attrib['type'] == "ip-dst": if attrib['type'] == "ip-dst":
ip_addr = get_ip_addr(attrib) ip_addr = self.get_ip_addr(attrib)
target[get_ip_version(ip_addr)] = ip_addr self.append_value_or_create_list(self.get_ip_version(ip_addr), ip_addr, target)
elif attrib['type'] == "ip-src": elif attrib['type'] == "ip-src":
ip_addr = get_ip_addr(attrib) ip_addr = self.get_ip_addr(attrib)
source[get_ip_version(ip_addr)] = ip_addr self.append_value_or_create_list(self.get_ip_version(ip_addr), ip_addr, source)
elif attrib['type'] == "src-port" or attrib['object_relation'] == "src-port": elif attrib['type'] == "src-port" or attrib['object_relation'] == "src-port":
source['Port'] = [int(attrib['value'])] source['Port'] = [int(attrib['value'])]
elif attrib['type'] == "dst-port" or attrib['object_relation'] == "dst-port": elif attrib['type'] == "dst-port" or attrib['object_relation'] == "dst-port":
target['Port'] = [int(attrib['value'])] target['Port'] = [int(attrib['value'])]
elif attrib['type'] == "hostname-src": elif attrib['object_relation'] == "hostname-src":
source['Hostname'] = [attrib['value']] source['Hostname'] = [attrib['value']]
elif attrib['type'] == "hostname-dst": elif attrib['object_relation'] == "hostname-dst" or attrib['type'] == "hostname":
target['Hostname'] = [attrib['value']] target['Hostname'] = [attrib['value']]
elif attrib['type'] == "protocol": elif attrib['object_relation'] == "protocol":
source['Proto'] = [attrib['value']] source['Proto'] = [attrib['value']]
target['Proto'] = [attrib['value']] target['Proto'] = [attrib['value']]
elif attrib['type'].startswith("layer"): elif attrib['object_relation'].startswith("layer"):
# means layer3-protocol, layer4-protocol or layer7-protocol # means layer3-protocol, layer4-protocol or layer7-protocol
# save its value to number of layer # save its value to number of layer
proto_dict[attrib['type'][5]] = attrib['value'] proto_dict[attrib['object_relation'][5]] = attrib['value']
elif attrib['type'] == "src-as" or attrib['object_relation'] == "src-as": elif attrib['type'] == "src-as" or attrib['object_relation'] == "src-as":
source['ASN'] = [int(attrib['value'])] source['ASN'] = [int(attrib['value'])]
elif attrib['type'] == "dst-as" or attrib['object_relation'] == "dst-as": elif attrib['type'] == "dst-as" or attrib['object_relation'] == "dst-as":
target['ASN'] = [int(attrib['value'])] target['ASN'] = [int(attrib['value'])]
elif attrib['type'] == "domain": elif attrib['type'] == "domain":
# won't be overwrite previous value of hostname-dst, hostname-dst can occur in object of type # won't overwrite previous value of hostname-dst, hostname-dst can occur in object of type
# network-connection and network-socket, but domain occurs only in other object types # network-connection and network-socket, but domain occurs only in other object types
target['Hostname'] = [attrib['value']] target['Hostname'] = [attrib['value']]
...@@ -199,30 +272,77 @@ def process_misp_object(misp_object, attach_counter): ...@@ -199,30 +272,77 @@ def process_misp_object(misp_object, attach_counter):
target['Proto'] = proto_list target['Proto'] = proto_list
if source: if source:
yield "Source", source self._source.append(source)
if target: if target:
yield "Target", target self._target.append(target)
elif misp_object['name'] == "file": elif misp_object['name'] == "file":
new_attach = {'Handle': "attach" + str(attach_counter)} new_attach = {'Handle': "attach" + str(self._attach_counter)}
self._attach_counter += 1
for attrib in misp_object['Attribute']: for attrib in misp_object['Attribute']:
if attrib['type'] in useful_hashes: if attrib['type'] in __class__.useful_hashes:
try: try:
new_attach['Hash'].append(attrib['type'] + ":" + attrib['value']) new_attach['Hash'].append(attrib['type'] + ":" + attrib['value'])
except KeyError: except KeyError:
new_attach['Hash'] = [attrib['type'] + ":" + attrib['value']] new_attach['Hash'] = [attrib['type'] + ":" + attrib['value']]
elif attrib['type'] == "filename": elif attrib['type'] == "filename":
try: try:
new_attach['Filename'].append(attrib['value']) new_attach['FileName'].append(attrib['value'])
except KeyError: except KeyError:
new_attach['Filename'] = [attrib['value']] new_attach['FileName'] = [attrib['value']]
if new_attach.get('Hash') or new_attach.get('Filename'): if new_attach.get('Hash') or new_attach.get('FileName'):
yield "Attach", new_attach self._attach.append(new_attach)
def process_source_or_target_object(self, misp_object):
"""
Process source or target object, and retrieved data insert to IDEA source or target object and append it to
corresponding object array
:param misp_object: the object, which will be processed
:return: None
"""
converted_object = {}
for misp_attrib in misp_object['Attribute']:
if misp_attrib['object_relation'] not in ("Note", "Vulnerability", "Reference"):
self.append_value_or_create_list(misp_attrib['object_relation'], misp_attrib['value'], converted_object)
else:
if misp_attrib['object_relation'] == "Note":
converted_object['Note'] = misp_attrib['value']
elif misp_attrib['object_relation'] == "Vulnerability":
self.append_value_or_create_list("Ref", "cve:" + misp_attrib['value'],
converted_object)
elif misp_attrib['object_relation'] == "Reference":
self.append_value_or_create_list("Ref", misp_attrib['value'], converted_object)
if misp_object['name'] == "source":
self._source.append(converted_object)
else: else:
return None, None self._target.append(converted_object)
def process_attach_object(self, misp_object):
"""
Process attach object, and retrieved data insert to IDEA attach object and append it to attach array
:param misp_object: the object, which will be processed
:return: None
"""
converted_object = {"Handle": "att" + str(self._attach_counter)}
self._attach_counter += 1
for attach_attrib in misp_object['Attribute']:
if attach_attrib['object_relation'] in ("Note", "ContentType", "ContentCharset", "ContentEncoding",
"Content"):
converted_object[attach_attrib['object_relation']] = attach_attrib['value']
elif attach_attrib['object_relation'] == "FileName":
self.append_value_or_create_list("FileName", attach_attrib['value'], converted_object)
elif attach_attrib['object_relation'] == "Reference":
self.append_value_or_create_list("Ref", attach_attrib['value'], converted_object)
elif attach_attrib['object_relation'] == "Vulnerability":
self.append_value_or_create_list("Ref", "cve" + attach_attrib['value'], converted_object)
elif attach_attrib['object_relation'] == "Size":
converted_object['Size'] = int(attach_attrib['value'])
else:
self.append_value_or_create_list("Hash", attach_attrib['type'] + ":" + attach_attrib['value'],
converted_object)
self._attach.append(converted_object)
def to_idea(misp_event, idea_id=None, test=False, origdata=False): def to_idea(self, misp_event, idea_id=None, test=False, origdata=False):
""" """
Creates whole IDEA message from MISP event Creates whole IDEA message from MISP event
:param misp_event: the misp event :param misp_event: the misp event
...@@ -235,16 +355,13 @@ def to_idea(misp_event, idea_id=None, test=False, origdata=False): ...@@ -235,16 +355,13 @@ def to_idea(misp_event, idea_id=None, test=False, origdata=False):
'Format': "IDEA0", 'Format': "IDEA0",
'ID': str(idea_id) if idea_id is not None else str(uuid4()), 'ID': str(idea_id) if idea_id is not None else str(uuid4()),
'Category': [], 'Category': [],
'Description': misp_event['info'], 'Description': misp_event['info']
'Source': [],
'Target': [],
'Attach': []
} }
# fill in the IDEA event Category # fill in the IDEA event Category
for tag in misp_event.get('Tag', []): for tag in misp_event.get('Tag', []):
try: try:
new_category = misp_to_idea_dictionary[tag['name']] new_category = self.taxonomy_to_category[tag['name']]
if new_category not in idea_event['Category']: if new_category not in idea_event['Category']:
idea_event['Category'].append(new_category) idea_event['Category'].append(new_category)
except KeyError: except KeyError:
...@@ -253,21 +370,19 @@ def to_idea(misp_event, idea_id=None, test=False, origdata=False): ...@@ -253,21 +370,19 @@ def to_idea(misp_event, idea_id=None, test=False, origdata=False):
# cannot determine IDEA Category, cannot convert # cannot determine IDEA Category, cannot convert
if not idea_event['Category']: if not idea_event['Category']:
return None return None
if test: if test and "Test" not in idea_event['Category']:
idea_event['Category'].append("Test") idea_event['Category'].append("Test")
if not int(misp_event['publish_timestamp']): if not int(misp_event['publish_timestamp']):
idea_event['CreateTime'] = datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ") idea_event['CreateTime'] = datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ")
else: else:
idea_event['CreateTime'] = convert_epoch_to_utc(misp_event['publish_timestamp']) idea_event['CreateTime'] = self.convert_epoch_to_utc(misp_event['publish_timestamp'])
attach_counter = 0 timestamps = itertools.chain(self.find('timestamp', misp_event['Attribute']),
self.find('timestamp', misp_event['Object']))
timestamps = itertools.chain(find('timestamp', misp_event['Attribute']),
find('timestamp', misp_event['Object']))
try: try:
oldest_timestamp = min(map(int, timestamps)) oldest_timestamp = min(map(int, timestamps))
idea_event['DetectTime'] = convert_epoch_to_utc(oldest_timestamp) idea_event['DetectTime'] = self.convert_epoch_to_utc(oldest_timestamp)
except ValueError: except ValueError:
# no object and attributes --> min() ValueError for empty sequence # no object and attributes --> min() ValueError for empty sequence
idea_event['DetectTime'] = misp_event['date'] + "T00:00:00Z" idea_event['DetectTime'] = misp_event['date'] + "T00:00:00Z"
...@@ -285,25 +400,160 @@ def to_idea(misp_event, idea_id=None, test=False, origdata=False): ...@@ -285,25 +400,160 @@ def to_idea(misp_event, idea_id=None, test=False, origdata=False):
# check all attributes for all potentially useful data # check all attributes for all potentially useful data
for attrib in misp_event['Attribute']: for attrib in misp_event['Attribute']:
key, new_data = process_misp_attribute(attrib, attach_counter) self.process_misp_attribute(attrib)
if key and new_data:
idea_event[key].append(new_data)
if key == "Attach":
attach_counter += 1
# check all objects for all potentially useful data # check all objects for all potentially useful data
for misp_object in misp_event['Object']: for misp_object in misp_event['Object']:
for key, new_data in process_misp_object(misp_object, attach_counter): if misp_object['name'] in ("source", "target"):
idea_event[key].append(new_data) # process the object
if key == "Attach": self.process_source_or_target_object(misp_object)
attach_counter += 1 elif misp_object['name'] == "attach":
self.process_attach_object(misp_object)
else:
self.process_misp_object(misp_object)
if origdata: if origdata:
idea_event['Attach'].append({ idea_event['Attach'].append({
'Handle': "att" + str(attach_counter), 'Handle': "att" + str(self._attach_counter),
'Note': "original data", 'Note': "original data",
'Content': misp_event 'Content': misp_event
}) })
attach_counter += 1
if self._source:
idea_event['Source'] = self._source
if self._target:
idea_event['Target'] = self._target
if self._attach:
idea_event['Attach'] = self._attach
# prepare instance for another conversion
self.__init__()
return idea_event return idea_event
class IdeaToMisp(object):
"""
Converts IDEA event to MISP event in MISP core format
"""
re_cve = re.compile("cve:", re.IGNORECASE)
severity_conversion = ["", "low", "medium", "high"]
def __init__(self):
"""
Initializes class and prepares instance variables
"""
self._idea_event = None
self._new_event = None
def process_basic_info(self):
"""
Process basic info of MISP event such distribution, threat_level_id, info, Tags, and standalone attributes
:return: None
"""
self._new_event = MISPEvent()
# add basic info about event
self._new_event.distribution = 1
try:
self._new_event.threat_level_id = __class__.severity_conversion.index(
self._idea_event['_CESNET']['EventSeverity'])
except KeyError:
# EventSeverity not in IDEA message --> 0 = Undefined
self._new_event.threat_level_id = 1
self._new_event.analysis = 2
# [0:10] is for taking date only from DetectTime (2019-01-26xxxxxxx)
self._new_event.date = datetime.strptime(self._idea_event['DetectTime'][0:10], "%Y-%m-%d")
if self._idea_event.get('Description'):
# add Description as event info and add Note as well if present
try:
self._new_event.info = self._idea_event['Description'] + ". " + self._idea_event['Note']
except KeyError:
self._new_event.info = self._idea_event['Description']
elif self._idea_event.get('Note'):
# or atleast Note if Description is not present
self._new_event.info = self._idea_event['Note']
# convert all categories in IDEA to taxonomies
for idea_category in self._idea_event['Category']:
# add both taxonomy ECSIRT and RSIT
self._new_event.add_tag(category_to_taxonomy[idea_category][0])
self._new_event.add_tag(category_to_taxonomy[idea_category][1])
for reference in self._idea_event.get('Ref', []):
# if reference is cve, add MISP attribute as vulnerability, otherwise add MISP attribute as link
if __class__.re_cve.search(reference):
self._new_event.add_attribute(category="External analysis", type="vulnerability",
value=__class__.re_cve.split(reference)[1])
else:
self._new_event.add_attribute(category="External analysis", type="link",
value=reference)
@classmethod
def add_attribute_to_object(cls, key, value, misp_object):
"""
Add one value (new MISP attribute) to source or target or attach object
:param key: name of attribute relation
:param value: inserted value
:param misp_object: object, which will the attribute be inserted into
:return: the object with inserted attribute
"""
try:
misp_object.add_attribute(key, value=value)
except NewAttributeError:
if key == "Ref":
if __class__.re_cve.search(value):
misp_object.add_attribute("Vulnerability", value=cls.re_cve.split(value)[1])
else:
misp_object.add_attribute("Reference", value=value)
if misp_object.name == "attach" and key == "Hash":
hash_method = value.split(":", 1)[0]
# FIXME may not be fixed properly
hash_value = value.split(":", 1)[1]
misp_object.add_attribute(hash_method.lower(), value=hash_value)
return misp_object
def process_one_idea_object(self, object_name):
"""
Process IDEA's Source or Target or Attach and convert it into MISP object
:param object_name: defines processing of IDEA's 'Source' or 'Target' or 'Attach'
:return: None
"""
for idea_obj in self._idea_event.get(object_name, []):
new_object = MISPObject(name=object_name.lower(), strict=True, standalone=False,
misp_objects_path_custom='object_templates')
for key, value in idea_obj.items():
if isinstance(value, list):
for list_value in value:
new_object = self.add_attribute_to_object(key, list_value, new_object)
else:
new_object = self.add_attribute_to_object(key, value, new_object)
self._new_event.add_object(new_object)
def process_all_idea_objects(self):
"""
Calls processing of Source, Target and Attach objects
:return: None
"""
self.process_one_idea_object('Source')
self.process_one_idea_object('Target')
self.process_one_idea_object('Attach')
def to_misp(self, idea_event):
"""
Converts IDEA message into MISP core format message
:param idea_event: the idea event, which will be converted
:return: instance of MISPEvent (with converted data)
"""
self._idea_event = idea_event
# fill new event with basic info and set event as published
self.process_basic_info()
self._new_event.publish()
self.process_all_idea_objects()
return self._new_event
misp_object_templates/attach/definition.json 0 → 100644
+ 121
− 0
View file @ 1184eb55
{
"name": "attach",
"meta-category": "misc",
"description": "Event attachment",
"uuid": "f5a964ac-5782-4c3e-8056-9b2783c987a8",
"version": 1,
"attributes": {
"FileName": {
"misp-attribute": "filename",
"description": "Name of file",
"ui-priority": 1,
"categories": ["Payload delivery"],
"multiple": true
},
"md5": {
"misp-attribute": "md5",
"description": "[Insecure] MD5 hash (128 bits)",
"ui-priority": 0,
"categories": ["Payload delivery"],
"multiple": true
},
"sha1": {
"misp-attribute": "sha1",
"description": "[Insecure] Secure Hash Algorithm 1 (160 bits)",
"ui-priority": 0,
"categories": ["Payload delivery"],
"multiple": true
},
"sha224": {
"misp-attribute": "sha224",
"description": "Secure Hash Algorithm 2 (224 bits)",
"ui-priority": 0,
"categories": ["Payload delivery"],
"multiple": true
},
"sha256": {
"misp-attribute": "sha256",
"description": "Secure Hash Algorithm 2 (256 bits)",
"ui-priority": 0,
"categories": ["Payload delivery"],
"multiple": true
},
"sha384": {
"misp-attribute": "sha384",
"description": "Secure Hash Algorithm 2 (384 bits)",
"ui-priority": 0,
"categories": ["Payload delivery"],
"multiple": true
},
"sha512": {
"misp-attribute": "sha512",
"description": "Secure Hash Algorithm 2 (512 bits)",
"ui-priority": 0,
"categories": ["Payload delivery"],
"multiple": true
},
"sha512/224": {
"misp-attribute": "sha512/224",
"description": "Secure Hash Algorithm 2 (224 bits)",
"ui-priority": 0,
"categories": ["Payload delivery"],
"multiple": true
},
"Size": {
"misp-attribute": "size-in-bytes",
"description": "Length of the content",
"ui-priority": 0,
"categories": ["Other"],
"multiple": false
},
"Vulnerability": {
"misp-attribute": "vulnerability",
"description": "Reference to known source, related to vulnerability, specific to this attachment",
"ui-priority": 0,
"categories": ["External analysis"],
"multiple": true
},
"Reference": {
"misp-attribute": "other",
"description": "Reference to known source, related to attack, specific to this attachment",
"ui-priority": 0,
"categories": ["External analysis"],
"multiple": true
},
"Note": {
"misp-attribute": "text",
"description": "Free text human readable additional note",
"ui-priority": 0,
"categories": ["Payload delivery"],
"multiple": false
},
"ContentType": {
"misp-attribute": "text",
"description": "Internet Media Type of the attachment",
"ui-priority": 1,
"categories": ["Payload delivery"],
"multiple": false
},
"ContentCharset": {
"misp-attribute": "text",
"description": "Name of the content character set",
"ui-priority": 0,
"categories": ["Payload delivery"],
"multiple": false
},
"ContentEncoding": {
"misp-attribute": "text",
"description": "Encoding of the content",
"ui-priority": 0,
"categories": ["Payload delivery"],
"multiple": false
},
"Content": {
"misp-attribute": "text",
"description": "Attachment content",
"ui-priority": 1,
"categories": ["Payload delivery"],
"multiple": false
}
}
}
\ No newline at end of file
misp_object_templates/source/definition.json 0 → 100644
+ 104
− 0
View file @ 1184eb55
{
"name": "source",
"meta-category": "network",
"description": "Description of the source of the event",
"uuid": "63cf1c78-4afe-49be-baff-2c101a942000",
"version": 1,
"attributes": {
"Hostname": {
"misp-attribute": "hostname",
"description": "Participating hostname",
"ui-priority": 0,
"categories": ["Network activity"],
"multiple": true
},
"IP4": {
"misp-attribute": "ip-src",
"description": "Source IPv4 address",
"ui-priority": 1,
"categories": ["Network activity"],
"multiple": true
},
"MAC": {
"misp-attribute": "mac-address",
"description": "Source MAC adress",
"ui-priority": 0,
"categories": ["Network activity"],
"multiple": true
},
"IP6": {
"misp-attribute": "ip-src",
"description": "Source IPv6 address",
"ui-priority": 1,
"categories": ["Network activity"],
"multiple": true
},
"Port": {
"misp-attribute": "port",
"description": "Source port",
"ui-priority": 1,
"categories": ["Other"],
"multiple": true
},
"Proto": {
"misp-attribute": "text",
"description": "Name of used protocol",
"ui-priority": 1,
"categories": ["Network activity"],
"multiple": true,
"values_list": [
"tcp",
"udp",
"icmp",
"ip",
"arp",
"http",
"https",
"ftp"
],
"required": false
},
"URL": {
"misp-attribute": "url",
"description": "Source URL",
"ui-priority": 0,
"categories": ["Network activity"],
"multiple": true
},
"Email": {
"misp-attribute": "email-src",
"description": "Source email adress",
"ui-priority": 0,
"categories": ["Payload delivery"],
"multiple": true
},
"Note": {
"misp-attribute": "text",
"description": "Free text human readable additional note",
"ui-priority": 0,
"categories": ["Payload delivery"],
"multiple": false
},
"ASN": {
"misp-attribute": "AS",
"description": "Autonomous system number of this source",
"ui-priority": 0,
"categories": ["Network activity"],
"multiple": true
},
"Vulnerability": {
"misp-attribute": "vulnerability",
"description": "Reference to known source, related to vulnerability, specific to this source",
"ui-priority": 0,
"categories": ["External analysis"],
"multiple": true
},
"Reference": {
"misp-attribute": "other",
"description": "Reference to known source, related to attack, specific to this source",
"ui-priority": 0,
"categories": ["External analysis"],
"multiple": true
}
}
}
\ No newline at end of file
misp_object_templates/target/definition.json 0 → 100644
+ 104
− 0
View file @ 1184eb55
{
"name": "target",
"meta-category": "network",
"description": "Description of the target of the event",
"uuid": "b251672f-3463-47d6-a20f-19d04c383195",
"version": 1,
"attributes": {
"Hostname": {
"misp-attribute": "hostname",
"description": "Participating hostname",
"ui-priority": 0,
"categories": ["Network activity"],
"multiple": true
},
"IP4": {
"misp-attribute": "ip-dst",
"description": "Target IPv4 address",
"ui-priority": 1,
"categories": ["Network activity"],
"multiple": true
},
"MAC": {
"misp-attribute": "mac-address",
"description": "Target MAC adress",
"ui-priority": 0,
"categories": ["Network activity"],
"multiple": true
},
"IP6": {
"misp-attribute": "ip-dst",
"description": "Target IPv6 address",
"ui-priority": 1,
"categories": ["Network activity"],
"multiple": true
},
"Port": {
"misp-attribute": "port",
"description": "Target port",
"ui-priority": 1,
"categories": ["Other"],
"multiple": true
},
"Proto": {
"misp-attribute": "text",
"description": "Name of used protocol",
"ui-priority": 1,
"categories": ["Network activity"],
"multiple": true,
"values_list": [
"tcp",
"udp",
"icmp",
"ip",
"arp",
"http",
"https",
"ftp"
],
"required": false
},
"URL": {
"misp-attribute": "url",
"description": "Target URL",
"ui-priority": 0,
"categories": ["Network activity"],
"multiple": true
},
"Email": {
"misp-attribute": "email-dst",
"description": "Target email adress",
"ui-priority": 0,
"categories": ["Payload delivery"],
"multiple": true
},
"Note": {
"misp-attribute": "text",
"description": "Free text human readable additional note",
"ui-priority": 0,
"categories": ["Payload delivery"],
"multiple": false
},
"ASN": {
"misp-attribute": "AS",
"description": "Autonomous system number of this target",
"ui-priority": 0,
"categories": ["Network activity"],
"multiple": true
},
"Vulnerability": {
"misp-attribute": "vulnerability",
"description": "Reference to known source, related to vulnerability, specific to this target",
"ui-priority": 0,
"categories": ["External analysis"],
"multiple": true
},
"Reference": {
"misp-attribute": "other",
"description": "Reference to known source, related to attack, specific to this target",
"ui-priority": 0,
"categories": ["External analysis"],
"multiple": true
}
}
}
\ No newline at end of file
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment