diff --git a/misp_test_files/test_idea_event.json b/misp_test_files/test_idea_event.json new file mode 100644 index 0000000000000000000000000000000000000000..826f547199d613221112e9f595c2b96a26dbbd60 --- /dev/null +++ b/misp_test_files/test_idea_event.json @@ -0,0 +1,212 @@ +{ + "Format": "IDEA0", + "ID": "9be8f15e-cda5-4627-a7b8-422b9f166cd1", + "Category": [ + "Test", + "Availability.DoS" + ], + "Description": "Test event", + "CreateTime": "2019-01-30T11:29:39Z", + "DetectTime": "2019-01-28T09:28:32Z", + "Node": [ + { + "Name": "ORGNAME", + "Note": "MISP organization id (created event): 1" + }, + { + "Name": "ORGNAME", + "Note": "MISP organization id (reported event): 1" + } + ], + "Source": [ + { + "IP4": [ + "192.168.0.1" + ] + }, + { + "IP4": [ + "192.168.0.2" + ] + }, + { + "Email": [ + "test@example.test" + ] + }, + { + "IP4": [ + "192.168.0.3" + ], + "Port": [ + 333 + ] + }, + { + "Port": [ + 3333 + ], + "IP4": [ + "192.168.0.50" + ], + "Proto": [ + "UDP" + ] + }, + { + "IP4": [ + "192.168.0.100" + ], + "Port": [ + 222 + ], + "Proto": [ + "IP", + "TCP", + "HTTP" + ] + }, + { + "IP4": [ + "192.168.0.120" + ], + "Proto": [ + "TCP" + ] + }, + { + "IP4": [ + "190.90.90.90", + "180.80.80.80", + "120.30.30.30" + ], + "IP6": [ + "fdba:cf29:3b2b:bf4:ffff:ffff:ffff:ffff" + ], + "Port": [ + "222", + "222" + ], + "Proto": [ + "tcp" + ], + "MAC": [ + "2b:54:d6:0c:c2:f2", + "32:f0:9e:19:24:ec" + ], + "Email": [ + "admin@test.org" + ], + "Note": "Test source object", + "Ref": [ + "cve:CVE-2018-13280", + "cve:CVE-2017-7901" + ] + } + ], + "Target": [ + { + "IP4": [ + "192.90.0.1" + ] + }, + { + "IP4": [ + "192.90.0.2" + ], + "Port": [ + 80 + ] + }, + { + "Email": [ + "email@test.org" + ] + }, + { + "Hostname": [ + "example.com" + ], + "IP4": [ + "20.20.20.20" + ] + }, + { + "Port": [ + 480 + ], + "Hostname": [ + "example.com" + ], + "IP4": [ + "192.90.10.10" + ] + }, + { + "IP4": [ + "192.90.20.20" + ], + "Port": [ + 6666 + ], + "Proto": [ + "UDP" + ] + }, + { + "IP4": [ + "198.20.20.20" + ], + "Port": [ + 444 + ], + "Proto": [ + "IP", + "TCP", + "HTTP" + ] + }, + { + "IP4": [ + "180.200.200.100" + ], + "Port": [ + 200 + ], + "Proto": [ + "TCP" + ] + } + ], + "Attach": [ + { + "Handle": "attach0", + "FileName": [ + "definitelyNotVirus.exe" + ], + "Hash": [ + "md5:c760ee8d2c87a58b93c2df797123e09d" + ] + }, + { + "Handle": "attach1", + "Hash": [ + "sha256:cc9b1edd07ff57d5e6c8fe4749d69442662014a390005a2056226fef1e70a91d" + ] + }, + { + "Handle": "att2", + "ContentType": "text/plain", + "FileName": [ + "exampleName.exe" + ], + "Content": "This is test attachment", + "Hash": [ + "md5:2fd30a87e52f0189cb19d8c8ebb9761c", + "sha512:f5bf02f82bbc2db7603eba04ca90079e78504cf7198b49e5815683c258a155c0a735f230fd6d06c651ac70493c23fb0b7d413fc068b644c7650942fea06ac374" + ], + "Size": 222000, + "Note": "Test Attach object" + } + ] +} \ No newline at end of file diff --git a/misp_test_files/test_idea_event_02.json b/misp_test_files/test_idea_event_02.json new file mode 100644 index 0000000000000000000000000000000000000000..dbfb94d735053c0e076cfbcdc0ac947dbead1c49 --- /dev/null +++ b/misp_test_files/test_idea_event_02.json @@ -0,0 +1,140 @@ +{ + "Format": "IDEA0", + "ID": "3b06db03-f22b-4b68-864a-b89e9f141255", + "Category": [ + "Test", + "Availability.DoS" + ], + "Description": "Test event", + "Source": [ + { + "IP4": [ + "192.168.0.1", + "192.168.0.2", + "192.168.0.3", + "192.168.0.4" + ], + "IP6": [ + "fd8a:ff7e:9ba1:020c::/64", + "fd8a:ff7e:9ba1:20c:ffff:ffff:ffff:ffff" + ], + "MAC": [ + "2b:54:d6:0c:c2:f2", + "32:f0:9e:19:24:ec" + ], + "Port": [ + 90, 300, 443, 3333, 3334 + ], + "Proto": [ + "tcp", + "http" + ], + "Email": [ + "admin@test.org", + "test.admin@test.org" + ], + "Note": "Test source object", + "Ref": [ + "cve:CVE-2018-13280", + "cve:CVE-2017-7901" + ] + }, + { + "IP4": [ + "192.169.0.1", + "192.169.0.2", + "192.169.0.3", + "192.169.0.4" + ], + "IP6": [ + "fd8a:ff7e:9ba1:020d::/64", + "fd8a:ff7e:9ba1:20d:ffff:ffff:ffff:ffff" + ], + "MAC": [ + "2b:54:d6:09:42:ce", + "32:f0:9e:b2:45:6d" + ], + "Port": [ + 80 + ] + }, + { + "Email": [ + "test@example.test" + ] + }, + { + "IP4": [ + "192.170.0.120" + ], + "Proto": [ + "tcp" + ] + } + ], + "Target": [ + { + "IP4": [ + "120.90.0.1", + "120.90.0.2" + ], + "Port": [ + 480 + ], + "Hostname": [ + "example.com" + ], + "Note": "Test target object" + }, + { + "IP4": [ + "120.90.1.2" + ], + "Port": [ + 80, 90, 100 + ], + "Proto": [ + "tcp" + ], + "Email": [ + "email@test.org" + ] + }, + { + "Email": [ + "email02@test.org" + ] + }, + { + "Hostname": [ + "example.com" + ], + "IP4": [ + "120.90.5.1" + ] + } + ], + "Attach": [ + { + "Handle": "att1", + "FileName": ["killemall"], + "Type": ["Malware"], + "ContentType": "application/octet-stream", + "Hash": ["sha1:b43daa145cb39e74cc28fef4a2d7b027b75f97ff"], + "Size": 46, + "Ref": ["Trojan-Spy:W32/FinSpy.A"], + "ContentEncoding": "base64", + "Content": "TVpqdXN0a2lkZGluZwo=" + } + ], + "CreateTime": "2019-01-28T10:19:40Z", + "DetectTime": "2019-01-28T09:28:32Z", + "Node": [ + { + "Name": "cz.cesnet.kippo-honey", + "Type": ["Protocol", "Honeypot"], + "SW": ["Kippo"], + "AggrWin": "00:05:00" + } + ] +} \ No newline at end of file diff --git a/misp_test_files/test_misp_event.json b/misp_test_files/test_misp_event.json new file mode 100644 index 0000000000000000000000000000000000000000..9df704515eee64afe160099bcafe43eeea0d8e6c --- /dev/null +++ b/misp_test_files/test_misp_event.json @@ -0,0 +1,1228 @@ +{ + "id": "1077", + "orgc_id": "1", + "org_id": "1", + "date": "2019-01-28", + "threat_level_id": "4", + "info": "Test event", + "published": false, + "uuid": "5c4ecb08-1658-4033-8252-03ab0a00020f", + "attribute_count": "54", + "analysis": "2", + "timestamp": "1548678019", + "distribution": "1", + "proposal_email_lock": false, + "locked": false, + "publish_timestamp": "0", + "sharing_group_id": "0", + "disable_correlation": false, + "extends_uuid": "", + "event_creator_email": "admin@admin.test", + "Org": { + "id": "1", + "name": "ORGNAME", + "uuid": "5c1570d5-4ac4-42e4-bb9a-4582819ae5d7" + }, + "Orgc": { + "id": "1", + "name": "ORGNAME", + "uuid": "5c1570d5-4ac4-42e4-bb9a-4582819ae5d7" + }, + "Attribute": [ + { + "id": "216627", + "type": "ip-src", + "category": "Network activity", + "to_ids": false, + "uuid": "5c4ecb40-eb2c-4d5c-a582-08310a00020f", + "event_id": "1077", + "distribution": "5", + "timestamp": "1548667712", + "comment": "", + "sharing_group_id": "0", + "deleted": false, + "disable_correlation": false, + "object_id": "0", + "object_relation": null, + "value": "192.168.0.1", + "Galaxy": [], + "ShadowAttribute": [] + }, + { + "id": "216628", + "type": "ip-src", + "category": "Network activity", + "to_ids": false, + "uuid": "5c4ecbb7-600c-465a-8253-08320a00020f", + "event_id": "1077", + "distribution": "5", + "timestamp": "1548667831", + "comment": "", + "sharing_group_id": "0", + "deleted": false, + "disable_correlation": false, + "object_id": "0", + "object_relation": null, + "value": "192.168.0.2", + "Galaxy": [], + "ShadowAttribute": [] + }, + { + "id": "216629", + "type": "ip-dst", + "category": "Network activity", + "to_ids": false, + "uuid": "5c4ecbcf-456c-4f32-86cf-08320a00020f", + "event_id": "1077", + "distribution": "5", + "timestamp": "1548667855", + "comment": "", + "sharing_group_id": "0", + "deleted": false, + "disable_correlation": false, + "object_id": "0", + "object_relation": null, + "value": "192.90.0.1", + "Galaxy": [], + "ShadowAttribute": [] + }, + { + "id": "216630", + "type": "ip-dst|port", + "category": "Network activity", + "to_ids": false, + "uuid": "5c4ecbf2-76dc-4dd7-9c11-08320a00020f", + "event_id": "1077", + "distribution": "5", + "timestamp": "1548667890", + "comment": "", + "sharing_group_id": "0", + "deleted": false, + "disable_correlation": false, + "object_id": "0", + "object_relation": null, + "value": "192.90.0.2|80", + "Galaxy": [], + "ShadowAttribute": [] + }, + { + "id": "216631", + "type": "email-src", + "category": "Payload delivery", + "to_ids": false, + "uuid": "5c4ecc78-e4a0-4b1d-87de-088f0a00020f", + "event_id": "1077", + "distribution": "5", + "timestamp": "1548668024", + "comment": "", + "sharing_group_id": "0", + "deleted": false, + "disable_correlation": false, + "object_id": "0", + "object_relation": null, + "value": "test@example.test", + "Galaxy": [], + "ShadowAttribute": [] + }, + { + "id": "216632", + "type": "email-dst", + "category": "Payload delivery", + "to_ids": false, + "uuid": "5c4ecc90-f634-4dd2-a620-088f0a00020f", + "event_id": "1077", + "distribution": "5", + "timestamp": "1548668048", + "comment": "", + "sharing_group_id": "0", + "deleted": false, + "disable_correlation": false, + "object_id": "0", + "object_relation": null, + "value": "email@test.org", + "Galaxy": [], + "ShadowAttribute": [] + }, + { + "id": "216633", + "type": "filename|md5", + "category": "Payload delivery", + "to_ids": false, + "uuid": "5c4eccfc-9970-4184-a998-03aa0a00020f", + "event_id": "1077", + "distribution": "5", + "timestamp": "1548668156", + "comment": "", + "sharing_group_id": "0", + "deleted": false, + "disable_correlation": false, + "object_id": "0", + "object_relation": null, + "value": "definitelyNotVirus.exe|c760ee8d2c87a58b93c2df797123e09d", + "Galaxy": [], + "ShadowAttribute": [] + }, + { + "id": "216634", + "type": "sha256", + "category": "Payload delivery", + "to_ids": false, + "uuid": "5c4ecd2e-0a40-4786-b499-03a90a00020f", + "event_id": "1077", + "distribution": "5", + "timestamp": "1548668206", + "comment": "", + "sharing_group_id": "0", + "deleted": false, + "disable_correlation": false, + "object_id": "0", + "object_relation": null, + "value": "cc9b1edd07ff57d5e6c8fe4749d69442662014a390005a2056226fef1e70a91d", + "Galaxy": [], + "ShadowAttribute": [] + }, + { + "id": "216635", + "type": "ip-src|port", + "category": "Network activity", + "to_ids": false, + "uuid": "5c4ecd5d-6fa8-4c65-98c5-03a90a00020f", + "event_id": "1077", + "distribution": "5", + "timestamp": "1548668253", + "comment": "", + "sharing_group_id": "0", + "deleted": false, + "disable_correlation": false, + "object_id": "0", + "object_relation": null, + "value": "192.168.0.3|333", + "Galaxy": [], + "ShadowAttribute": [] + } + ], + "ShadowAttribute": [], + "RelatedEvent": [], + "Galaxy": [], + "Object": [ + { + "id": "10109", + "name": "domain-ip", + "meta-category": "network", + "description": "A domain and IP address seen as a tuple in a specific time frame.", + "template_uuid": "43b3b146-77eb-4931-b4cc-b66c60f28734", + "template_version": "6", + "event_id": "1077", + "uuid": "5c4ece00-f92c-49f6-8e5d-03ab0a00020f", + "timestamp": "1548668416", + "distribution": "5", + "sharing_group_id": "0", + "comment": "", + "deleted": false, + "ObjectReference": [], + "Attribute": [ + { + "id": "216636", + "type": "domain", + "category": "Network activity", + "to_ids": true, + "uuid": "5c4ece00-09c4-4913-b2a2-03ab0a00020f", + "event_id": "1077", + "distribution": "5", + "timestamp": "1548668416", + "comment": "", + "sharing_group_id": "0", + "deleted": false, + "disable_correlation": false, + "object_id": "10109", + "object_relation": "domain", + "value": "example.com", + "Galaxy": [], + "ShadowAttribute": [] + }, + { + "id": "216637", + "type": "ip-dst", + "category": "Network activity", + "to_ids": true, + "uuid": "5c4ece00-c344-4d30-b25e-03ab0a00020f", + "event_id": "1077", + "distribution": "5", + "timestamp": "1548668416", + "comment": "", + "sharing_group_id": "0", + "deleted": false, + "disable_correlation": false, + "object_id": "10109", + "object_relation": "ip", + "value": "20.20.20.20", + "Galaxy": [], + "ShadowAttribute": [] + } + ] + }, + { + "id": "10110", + "name": "ip-port", + "meta-category": "network", + "description": "An IP address (or domain or hostname) and a port seen as a tuple (or as a triple) in a specific time frame.", + "template_uuid": "9f8cea74-16fe-4968-a2b4-026676949ac6", + "template_version": "7", + "event_id": "1077", + "uuid": "5c4ece48-f5c4-45d6-96a4-03ab0a00020f", + "timestamp": "1548668488", + "distribution": "5", + "sharing_group_id": "0", + "comment": "", + "deleted": false, + "ObjectReference": [], + "Attribute": [ + { + "id": "216638", + "type": "port", + "category": "Network activity", + "to_ids": false, + "uuid": "5c4ece48-8d54-4a6d-b5d3-03ab0a00020f", + "event_id": "1077", + "distribution": "5", + "timestamp": "1548668488", + "comment": "", + "sharing_group_id": "0", + "deleted": false, + "disable_correlation": true, + "object_id": "10110", + "object_relation": "dst-port", + "value": "480", + "Galaxy": [], + "ShadowAttribute": [] + }, + { + "id": "216639", + "type": "domain", + "category": "Network activity", + "to_ids": true, + "uuid": "5c4ece48-2ce4-47cd-a7b5-03ab0a00020f", + "event_id": "1077", + "distribution": "5", + "timestamp": "1548668488", + "comment": "", + "sharing_group_id": "0", + "deleted": false, + "disable_correlation": false, + "object_id": "10110", + "object_relation": "domain", + "value": "example.com", + "Galaxy": [], + "ShadowAttribute": [] + }, + { + "id": "216640", + "type": "ip-dst", + "category": "Network activity", + "to_ids": true, + "uuid": "5c4ece48-1b74-4f72-9a44-03ab0a00020f", + "event_id": "1077", + "distribution": "5", + "timestamp": "1548668488", + "comment": "", + "sharing_group_id": "0", + "deleted": false, + "disable_correlation": false, + "object_id": "10110", + "object_relation": "ip", + "value": "192.90.10.10", + "Galaxy": [], + "ShadowAttribute": [] + } + ] + }, + { + "id": "10111", + "name": "netflow", + "meta-category": "network", + "description": "Netflow object describes an network object based on the Netflowv5/v9 minimal definition", + "template_uuid": "bf148c58-3e7e-414e-8de8-5d96379ca77e", + "template_version": "1", + "event_id": "1077", + "uuid": "5c4ecea8-9874-41d9-96bf-03ab0a00020f", + "timestamp": "1548668584", + "distribution": "5", + "sharing_group_id": "0", + "comment": "", + "deleted": false, + "ObjectReference": [], + "Attribute": [ + { + "id": "216641", + "type": "ip-dst", + "category": "Network activity", + "to_ids": true, + "uuid": "5c4ecea8-9fac-40c3-94f5-03ab0a00020f", + "event_id": "1077", + "distribution": "5", + "timestamp": "1548668584", + "comment": "", + "sharing_group_id": "0", + "deleted": false, + "disable_correlation": false, + "object_id": "10111", + "object_relation": "ip-dst", + "value": "192.90.20.20", + "Galaxy": [], + "ShadowAttribute": [] + }, + { + "id": "216642", + "type": "port", + "category": "Network activity", + "to_ids": false, + "uuid": "5c4ecea8-2074-4baf-8523-03ab0a00020f", + "event_id": "1077", + "distribution": "5", + "timestamp": "1548668584", + "comment": "", + "sharing_group_id": "0", + "deleted": false, + "disable_correlation": false, + "object_id": "10111", + "object_relation": "dst-port", + "value": "6666", + "Galaxy": [], + "ShadowAttribute": [] + }, + { + "id": "216643", + "type": "port", + "category": "Network activity", + "to_ids": false, + "uuid": "5c4ecea8-b6f0-4efe-b7af-03ab0a00020f", + "event_id": "1077", + "distribution": "5", + "timestamp": "1548668584", + "comment": "", + "sharing_group_id": "0", + "deleted": false, + "disable_correlation": false, + "object_id": "10111", + "object_relation": "src-port", + "value": "3333", + "Galaxy": [], + "ShadowAttribute": [] + }, + { + "id": "216644", + "type": "ip-src", + "category": "Network activity", + "to_ids": true, + "uuid": "5c4ecea8-11d0-4b4a-aab2-03ab0a00020f", + "event_id": "1077", + "distribution": "5", + "timestamp": "1548668584", + "comment": "", + "sharing_group_id": "0", + "deleted": false, + "disable_correlation": false, + "object_id": "10111", + "object_relation": "ip-src", + "value": "192.168.0.50", + "Galaxy": [], + "ShadowAttribute": [] + }, + { + "id": "216645", + "type": "text", + "category": "Other", + "to_ids": false, + "uuid": "5c4ecea8-0584-4341-bb94-03ab0a00020f", + "event_id": "1077", + "distribution": "5", + "timestamp": "1548668584", + "comment": "", + "sharing_group_id": "0", + "deleted": false, + "disable_correlation": true, + "object_id": "10111", + "object_relation": "direction", + "value": "Ingress", + "Galaxy": [], + "ShadowAttribute": [] + }, + { + "id": "216646", + "type": "text", + "category": "Other", + "to_ids": false, + "uuid": "5c4ecea8-db3c-447c-8530-03ab0a00020f", + "event_id": "1077", + "distribution": "5", + "timestamp": "1548668584", + "comment": "", + "sharing_group_id": "0", + "deleted": false, + "disable_correlation": false, + "object_id": "10111", + "object_relation": "protocol", + "value": "UDP", + "Galaxy": [], + "ShadowAttribute": [] + } + ] + }, + { + "id": "10112", + "name": "network-connection", + "meta-category": "network", + "description": "A local or remote network connection.", + "template_uuid": "af16764b-f8e5-4603-9de1-de34d272f80b", + "template_version": "2", + "event_id": "1077", + "uuid": "5c4eced9-c77c-4cb9-b1b2-03a60a00020f", + "timestamp": "1548668633", + "distribution": "5", + "sharing_group_id": "0", + "comment": "", + "deleted": false, + "ObjectReference": [], + "Attribute": [ + { + "id": "216647", + "type": "ip-src", + "category": "Network activity", + "to_ids": true, + "uuid": "5c4eced9-c6ac-4a3a-a8e3-03a60a00020f", + "event_id": "1077", + "distribution": "5", + "timestamp": "1548668633", + "comment": "", + "sharing_group_id": "0", + "deleted": false, + "disable_correlation": false, + "object_id": "10112", + "object_relation": "ip-src", + "value": "192.168.0.100", + "Galaxy": [], + "ShadowAttribute": [] + }, + { + "id": "216648", + "type": "ip-dst", + "category": "Network activity", + "to_ids": true, + "uuid": "5c4eced9-fd94-4aaa-adca-03a60a00020f", + "event_id": "1077", + "distribution": "5", + "timestamp": "1548668633", + "comment": "", + "sharing_group_id": "0", + "deleted": false, + "disable_correlation": false, + "object_id": "10112", + "object_relation": "ip-dst", + "value": "198.20.20.20", + "Galaxy": [], + "ShadowAttribute": [] + }, + { + "id": "216649", + "type": "port", + "category": "Network activity", + "to_ids": false, + "uuid": "5c4eced9-e514-4527-83cf-03a60a00020f", + "event_id": "1077", + "distribution": "5", + "timestamp": "1548668633", + "comment": "", + "sharing_group_id": "0", + "deleted": false, + "disable_correlation": false, + "object_id": "10112", + "object_relation": "src-port", + "value": "222", + "Galaxy": [], + "ShadowAttribute": [] + }, + { + "id": "216650", + "type": "port", + "category": "Network activity", + "to_ids": false, + "uuid": "5c4eced9-fbcc-4f24-9435-03a60a00020f", + "event_id": "1077", + "distribution": "5", + "timestamp": "1548668633", + "comment": "", + "sharing_group_id": "0", + "deleted": false, + "disable_correlation": false, + "object_id": "10112", + "object_relation": "dst-port", + "value": "444", + "Galaxy": [], + "ShadowAttribute": [] + }, + { + "id": "216651", + "type": "text", + "category": "Other", + "to_ids": false, + "uuid": "5c4eced9-679c-4924-af78-03a60a00020f", + "event_id": "1077", + "distribution": "5", + "timestamp": "1548668633", + "comment": "", + "sharing_group_id": "0", + "deleted": false, + "disable_correlation": true, + "object_id": "10112", + "object_relation": "layer3-protocol", + "value": "IP", + "Galaxy": [], + "ShadowAttribute": [] + }, + { + "id": "216652", + "type": "text", + "category": "Other", + "to_ids": false, + "uuid": "5c4eced9-5884-493e-8e54-03a60a00020f", + "event_id": "1077", + "distribution": "5", + "timestamp": "1548668633", + "comment": "", + "sharing_group_id": "0", + "deleted": false, + "disable_correlation": true, + "object_id": "10112", + "object_relation": "layer4-protocol", + "value": "TCP", + "Galaxy": [], + "ShadowAttribute": [] + }, + { + "id": "216653", + "type": "text", + "category": "Other", + "to_ids": false, + "uuid": "5c4eced9-e8fc-4184-a667-03a60a00020f", + "event_id": "1077", + "distribution": "5", + "timestamp": "1548668633", + "comment": "", + "sharing_group_id": "0", + "deleted": false, + "disable_correlation": true, + "object_id": "10112", + "object_relation": "layer7-protocol", + "value": "HTTP", + "Galaxy": [], + "ShadowAttribute": [] + } + ] + }, + { + "id": "10113", + "name": "network-socket", + "meta-category": "network", + "description": "Network socket object describes a local or remote network connections based on the socket data structure.", + "template_uuid": "48bbfd72-ef8e-4649-b14d-41b4b5a0eba2", + "template_version": "1", + "event_id": "1077", + "uuid": "5c4ecf25-17bc-418d-92d6-03a60a00020f", + "timestamp": "1548668709", + "distribution": "5", + "sharing_group_id": "0", + "comment": "", + "deleted": false, + "ObjectReference": [], + "Attribute": [ + { + "id": "216654", + "type": "ip-src", + "category": "Network activity", + "to_ids": true, + "uuid": "5c4ecf26-11e0-4cfb-845d-03a60a00020f", + "event_id": "1077", + "distribution": "5", + "timestamp": "1548668710", + "comment": "", + "sharing_group_id": "0", + "deleted": false, + "disable_correlation": false, + "object_id": "10113", + "object_relation": "ip-src", + "value": "192.168.0.120", + "Galaxy": [], + "ShadowAttribute": [] + }, + { + "id": "216655", + "type": "ip-dst", + "category": "Network activity", + "to_ids": true, + "uuid": "5c4ecf26-bc9c-4643-8218-03a60a00020f", + "event_id": "1077", + "distribution": "5", + "timestamp": "1548668710", + "comment": "", + "sharing_group_id": "0", + "deleted": false, + "disable_correlation": false, + "object_id": "10113", + "object_relation": "ip-dst", + "value": "180.200.200.100", + "Galaxy": [], + "ShadowAttribute": [] + }, + { + "id": "216656", + "type": "port", + "category": "Network activity", + "to_ids": false, + "uuid": "5c4ecf26-6f58-4371-8ea7-03a60a00020f", + "event_id": "1077", + "distribution": "5", + "timestamp": "1548668710", + "comment": "", + "sharing_group_id": "0", + "deleted": false, + "disable_correlation": false, + "object_id": "10113", + "object_relation": "dst-port", + "value": "200", + "Galaxy": [], + "ShadowAttribute": [] + }, + { + "id": "216657", + "type": "text", + "category": "Other", + "to_ids": false, + "uuid": "5c4ecf26-f7d0-4e14-bbeb-03a60a00020f", + "event_id": "1077", + "distribution": "5", + "timestamp": "1548668710", + "comment": "", + "sharing_group_id": "0", + "deleted": false, + "disable_correlation": false, + "object_id": "10113", + "object_relation": "address-family", + "value": "AF_INET", + "Galaxy": [], + "ShadowAttribute": [] + }, + { + "id": "216658", + "type": "text", + "category": "Other", + "to_ids": false, + "uuid": "5c4ecf26-47f8-4a41-bcc8-03a60a00020f", + "event_id": "1077", + "distribution": "5", + "timestamp": "1548668710", + "comment": "", + "sharing_group_id": "0", + "deleted": false, + "disable_correlation": false, + "object_id": "10113", + "object_relation": "domain-family", + "value": "PF_INET", + "Galaxy": [], + "ShadowAttribute": [] + }, + { + "id": "216659", + "type": "text", + "category": "Other", + "to_ids": false, + "uuid": "5c4ecf26-c8b4-4105-8b39-03a60a00020f", + "event_id": "1077", + "distribution": "5", + "timestamp": "1548668710", + "comment": "", + "sharing_group_id": "0", + "deleted": false, + "disable_correlation": false, + "object_id": "10113", + "object_relation": "state", + "value": "blocking", + "Galaxy": [], + "ShadowAttribute": [] + }, + { + "id": "216660", + "type": "text", + "category": "Other", + "to_ids": false, + "uuid": "5c4ecf26-e37c-4702-8666-03a60a00020f", + "event_id": "1077", + "distribution": "5", + "timestamp": "1548668710", + "comment": "", + "sharing_group_id": "0", + "deleted": false, + "disable_correlation": false, + "object_id": "10113", + "object_relation": "protocol", + "value": "TCP", + "Galaxy": [], + "ShadowAttribute": [] + } + ] + }, + { + "id": "10114", + "name": "source", + "meta-category": "network", + "description": "Description of the source of the event", + "template_uuid": "63cf1c78-4afe-49be-baff-2c101a942000", + "template_version": "1", + "event_id": "1077", + "uuid": "5c4ef2ad-ee24-4cc4-b74c-088f0a00020f", + "timestamp": "1548677805", + "distribution": "5", + "sharing_group_id": "0", + "comment": "", + "deleted": false, + "ObjectReference": [], + "Attribute": [ + { + "id": "216661", + "type": "ip-src", + "category": "Network activity", + "to_ids": true, + "uuid": "5c4ef2ad-0e64-4306-9b69-088f0a00020f", + "event_id": "1077", + "distribution": "5", + "timestamp": "1548677805", + "comment": "", + "sharing_group_id": "0", + "deleted": false, + "disable_correlation": false, + "object_id": "10114", + "object_relation": "IP4", + "value": "190.90.90.90", + "Galaxy": [], + "ShadowAttribute": [] + }, + { + "id": "216662", + "type": "ip-src", + "category": "Network activity", + "to_ids": true, + "uuid": "5c4ef2ad-d9c8-4228-b780-088f0a00020f", + "event_id": "1077", + "distribution": "5", + "timestamp": "1548677805", + "comment": "", + "sharing_group_id": "0", + "deleted": false, + "disable_correlation": false, + "object_id": "10114", + "object_relation": "IP4", + "value": "180.80.80.80", + "Galaxy": [], + "ShadowAttribute": [] + }, + { + "id": "216663", + "type": "ip-src", + "category": "Network activity", + "to_ids": true, + "uuid": "5c4ef2ad-9ac8-4b08-bebf-088f0a00020f", + "event_id": "1077", + "distribution": "5", + "timestamp": "1548677805", + "comment": "", + "sharing_group_id": "0", + "deleted": false, + "disable_correlation": false, + "object_id": "10114", + "object_relation": "IP4", + "value": "120.30.30.30", + "Galaxy": [], + "ShadowAttribute": [] + }, + { + "id": "216664", + "type": "ip-src", + "category": "Network activity", + "to_ids": true, + "uuid": "5c4ef2ad-e920-4d6a-8669-088f0a00020f", + "event_id": "1077", + "distribution": "5", + "timestamp": "1548677805", + "comment": "", + "sharing_group_id": "0", + "deleted": false, + "disable_correlation": false, + "object_id": "10114", + "object_relation": "IP6", + "value": "fdba:cf29:3b2b:bf4:ffff:ffff:ffff:ffff", + "Galaxy": [], + "ShadowAttribute": [] + }, + { + "id": "216665", + "type": "port", + "category": "Other", + "to_ids": false, + "uuid": "5c4ef2ad-1ae4-482e-835f-088f0a00020f", + "event_id": "1077", + "distribution": "5", + "timestamp": "1548677805", + "comment": "", + "sharing_group_id": "0", + "deleted": false, + "disable_correlation": false, + "object_id": "10114", + "object_relation": "Port", + "value": "222", + "Galaxy": [], + "ShadowAttribute": [] + }, + { + "id": "216666", + "type": "port", + "category": "Other", + "to_ids": false, + "uuid": "5c4ef2ad-4ae0-46fd-a3e5-088f0a00020f", + "event_id": "1077", + "distribution": "5", + "timestamp": "1548677805", + "comment": "", + "sharing_group_id": "0", + "deleted": false, + "disable_correlation": false, + "object_id": "10114", + "object_relation": "Port", + "value": "222", + "Galaxy": [], + "ShadowAttribute": [] + }, + { + "id": "216667", + "type": "text", + "category": "Network activity", + "to_ids": false, + "uuid": "5c4ef2ad-289c-4a9f-a14d-088f0a00020f", + "event_id": "1077", + "distribution": "5", + "timestamp": "1548677805", + "comment": "", + "sharing_group_id": "0", + "deleted": false, + "disable_correlation": false, + "object_id": "10114", + "object_relation": "Proto", + "value": "tcp", + "Galaxy": [], + "ShadowAttribute": [] + }, + { + "id": "216668", + "type": "mac-address", + "category": "Network activity", + "to_ids": false, + "uuid": "5c4ef2ad-e5a0-4c55-b35f-088f0a00020f", + "event_id": "1077", + "distribution": "5", + "timestamp": "1548677805", + "comment": "", + "sharing_group_id": "0", + "deleted": false, + "disable_correlation": false, + "object_id": "10114", + "object_relation": "MAC", + "value": "2b:54:d6:0c:c2:f2", + "Galaxy": [], + "ShadowAttribute": [] + }, + { + "id": "216669", + "type": "mac-address", + "category": "Network activity", + "to_ids": false, + "uuid": "5c4ef2ad-9284-433d-8169-088f0a00020f", + "event_id": "1077", + "distribution": "5", + "timestamp": "1548677805", + "comment": "", + "sharing_group_id": "0", + "deleted": false, + "disable_correlation": false, + "object_id": "10114", + "object_relation": "MAC", + "value": "32:f0:9e:19:24:ec", + "Galaxy": [], + "ShadowAttribute": [] + }, + { + "id": "216670", + "type": "email-src", + "category": "Payload delivery", + "to_ids": true, + "uuid": "5c4ef2ad-eb68-4d78-9c0e-088f0a00020f", + "event_id": "1077", + "distribution": "5", + "timestamp": "1548677805", + "comment": "", + "sharing_group_id": "0", + "deleted": false, + "disable_correlation": false, + "object_id": "10114", + "object_relation": "Email", + "value": "admin@test.org", + "Galaxy": [], + "ShadowAttribute": [] + }, + { + "id": "216671", + "type": "text", + "category": "Payload delivery", + "to_ids": false, + "uuid": "5c4ef2ad-227c-412c-8b00-088f0a00020f", + "event_id": "1077", + "distribution": "5", + "timestamp": "1548677805", + "comment": "", + "sharing_group_id": "0", + "deleted": false, + "disable_correlation": false, + "object_id": "10114", + "object_relation": "Note", + "value": "Test source object", + "Galaxy": [], + "ShadowAttribute": [] + }, + { + "id": "216672", + "type": "vulnerability", + "category": "External analysis", + "to_ids": false, + "uuid": "5c4ef2ad-a190-45b6-a220-088f0a00020f", + "event_id": "1077", + "distribution": "5", + "timestamp": "1548677805", + "comment": "", + "sharing_group_id": "0", + "deleted": false, + "disable_correlation": false, + "object_id": "10114", + "object_relation": "Vulnerability", + "value": "CVE-2018-13280", + "Galaxy": [], + "ShadowAttribute": [] + }, + { + "id": "216673", + "type": "vulnerability", + "category": "External analysis", + "to_ids": false, + "uuid": "5c4ef2ad-0f78-4c65-bdb5-088f0a00020f", + "event_id": "1077", + "distribution": "5", + "timestamp": "1548677805", + "comment": "", + "sharing_group_id": "0", + "deleted": false, + "disable_correlation": false, + "object_id": "10114", + "object_relation": "Vulnerability", + "value": "CVE-2017-7901", + "Galaxy": [], + "ShadowAttribute": [] + } + ] + }, + { + "id": "10115", + "name": "attach", + "meta-category": "misc", + "description": "Event attachment", + "template_uuid": "f5a964ac-5782-4c3e-8056-9b2783c987a8", + "template_version": "1", + "event_id": "1077", + "uuid": "5c4ef383-0390-4f23-9fd8-03a90a00020f", + "timestamp": "1548678019", + "distribution": "5", + "sharing_group_id": "0", + "comment": "", + "deleted": false, + "ObjectReference": [], + "Attribute": [ + { + "id": "216674", + "type": "text", + "category": "Payload delivery", + "to_ids": false, + "uuid": "5c4ef383-c1fc-44d3-bb05-03a90a00020f", + "event_id": "1077", + "distribution": "5", + "timestamp": "1548678019", + "comment": "", + "sharing_group_id": "0", + "deleted": false, + "disable_correlation": false, + "object_id": "10115", + "object_relation": "ContentType", + "value": "text/plain", + "Galaxy": [], + "ShadowAttribute": [] + }, + { + "id": "216675", + "type": "filename", + "category": "Payload delivery", + "to_ids": true, + "uuid": "5c4ef383-63f0-4ae8-9d2f-03a90a00020f", + "event_id": "1077", + "distribution": "5", + "timestamp": "1548678019", + "comment": "", + "sharing_group_id": "0", + "deleted": false, + "disable_correlation": false, + "object_id": "10115", + "object_relation": "FileName", + "value": "exampleName.exe", + "Galaxy": [], + "ShadowAttribute": [] + }, + { + "id": "216676", + "type": "text", + "category": "Payload delivery", + "to_ids": false, + "uuid": "5c4ef383-9b8c-4cda-b950-03a90a00020f", + "event_id": "1077", + "distribution": "5", + "timestamp": "1548678019", + "comment": "", + "sharing_group_id": "0", + "deleted": false, + "disable_correlation": false, + "object_id": "10115", + "object_relation": "Content", + "value": "This is test attachment", + "Galaxy": [], + "ShadowAttribute": [] + }, + { + "id": "216677", + "type": "md5", + "category": "Payload delivery", + "to_ids": true, + "uuid": "5c4ef383-0474-42de-878d-03a90a00020f", + "event_id": "1077", + "distribution": "5", + "timestamp": "1548678019", + "comment": "", + "sharing_group_id": "0", + "deleted": false, + "disable_correlation": false, + "object_id": "10115", + "object_relation": "md5", + "value": "2fd30a87e52f0189cb19d8c8ebb9761c", + "Galaxy": [], + "ShadowAttribute": [] + }, + { + "id": "216678", + "type": "sha512", + "category": "Payload delivery", + "to_ids": true, + "uuid": "5c4ef383-87b8-458d-a7cb-03a90a00020f", + "event_id": "1077", + "distribution": "5", + "timestamp": "1548678019", + "comment": "", + "sharing_group_id": "0", + "deleted": false, + "disable_correlation": false, + "object_id": "10115", + "object_relation": "sha512", + "value": "f5bf02f82bbc2db7603eba04ca90079e78504cf7198b49e5815683c258a155c0a735f230fd6d06c651ac70493c23fb0b7d413fc068b644c7650942fea06ac374", + "Galaxy": [], + "ShadowAttribute": [] + }, + { + "id": "216679", + "type": "size-in-bytes", + "category": "Other", + "to_ids": false, + "uuid": "5c4ef383-6564-4fa5-968d-03a90a00020f", + "event_id": "1077", + "distribution": "5", + "timestamp": "1548678019", + "comment": "", + "sharing_group_id": "0", + "deleted": false, + "disable_correlation": false, + "object_id": "10115", + "object_relation": "Size", + "value": "222000", + "Galaxy": [], + "ShadowAttribute": [] + }, + { + "id": "216680", + "type": "text", + "category": "Payload delivery", + "to_ids": false, + "uuid": "5c4ef383-ef64-4084-b4e4-03a90a00020f", + "event_id": "1077", + "distribution": "5", + "timestamp": "1548678019", + "comment": "", + "sharing_group_id": "0", + "deleted": false, + "disable_correlation": false, + "object_id": "10115", + "object_relation": "Note", + "value": "Test Attach object", + "Galaxy": [], + "ShadowAttribute": [] + } + ] + } + ], + "Tag": [ + { + "id": "550", + "name": "rsit:test=\"test\"", + "colour": "#7375f7", + "exportable": true, + "user_id": "0", + "hide_tag": false, + "numerical_value": null + }, + { + "id": "549", + "name": "ecsirt:test=\"test\"", + "colour": "#c01874", + "exportable": true, + "user_id": "0", + "hide_tag": false, + "numerical_value": null + }, + { + "id": "552", + "name": "rsit:availability=\"dos\"", + "colour": "#bb5e4b", + "exportable": true, + "user_id": "0", + "hide_tag": false, + "numerical_value": null + }, + { + "id": "551", + "name": "ecsirt:availability=\"dos\"", + "colour": "#6787fe", + "exportable": true, + "user_id": "0", + "hide_tag": false, + "numerical_value": null + } + ] +} diff --git a/test_misp.py b/test_misp.py new file mode 100644 index 0000000000000000000000000000000000000000..0c7a03fe0ea7bec1e11ed187a76219ea2d2b983b --- /dev/null +++ b/test_misp.py @@ -0,0 +1,177 @@ +#!/usr/bin/python3.6 +# -*- coding: utf-8 -*- +# +# Copyright (c) 2016, CESNET, z. s. p. o. +# Use of this source is governed by an ISC license, see LICENSE file. + +import unittest +from idea.misp import MispToIdea, IdeaToMisp +import os +import json +import re + + +class TestMispToIdeaConverter(unittest.TestCase): + """ + Basic unittest of MISP to IDEA conversion. generates IDEA event from MISP event saved in file and compares + output to saved IDEA event, which should be the same as the generated output + """ + + def setUp(self): + # load saved files, which contain designed test events + self.misp_test_file = open(os.path.join(os.getcwd(), "test_misp_files", "test_misp_event.json"), "r") + self.misp_event = json.load(self.misp_test_file) + self.idea_test_file = open(os.path.join(os.getcwd(), "test_misp_files", "test_idea_event.json"), "r") + self.idea_event = json.load(self.idea_test_file) + + def tearDown(self): + self.misp_test_file.close() + self.idea_test_file.close() + + def test_raw_ouput(self): + # convert MISP event to IDEA event + idea_converter = MispToIdea() + idea_converted_event = idea_converter.to_idea(self.misp_event) + + # CreateTime and ID can and probably will differ + idea_converted_event.pop('CreateTime') + idea_converted_event.pop('ID') + self.idea_event.pop('CreateTime') + self.idea_event.pop('ID') + + # compare correctly converted IDEA message (loaded from file) with currently generated version + self.assertTrue(json.dumps(self.idea_event, sort_keys=True) == json.dumps(idea_converted_event, sort_keys=True)) + + +class TestIdeaToMispConverter(unittest.TestCase): + """ + Basic unittest, which tests only, if all attributes, which should be converted, were converted + """ + re_cve = re.compile("cve:", re.IGNORECASE) + + def setUp(self): + # load test idea event + self.idea_test_file = open(os.path.join(os.getcwd(), "test_misp_files", "test_idea_event_02.json"), "r") + self.idea_event = json.load(self.idea_test_file) + + def tearDown(self): + self.idea_test_file.close() + + @staticmethod + def append_value_or_create_list(object_key, value_key, value, updated_object): + """ + Append value to list placed on key or, if key does not exist yet, create key with new list with the value + :param object_key: (source|target|attach) + :param value_key: key of value, which will be inserted (IP4|MAC|IP6|...) + :param value: the inserted value + :param updated_object: dictionary, which will be updated + :return: None (objects gets updated) + """ + try: + updated_object[object_key][value_key].append(str(value)) + except KeyError: + updated_object[object_key][value_key] = [str(value)] + + def process_source_or_target_object(self, name_of_object, updated_dict): + """ + Loads Source or Target object and get all data from it and save it into updated_dict + :param name_of_object: ("Source", "Target") + :param updated_dict: the dictionary, which will be updated + :return: None (updated_dict gets updated) + """ + for data_object in self.idea_event[name_of_object]: + for data_attrib, key_value in data_object.items(): + if data_attrib in ("Type", "AttachHand", "Spoofed", "Imprecise", "Anonymised", "Router", "Netname"): + # these keys are not being converted + continue + if data_attrib == "Note": + # Note is just string, not list as all other values + self.append_value_or_create_list(name_of_object.lower(), "Note", key_value, updated_dict) + else: + # all other keys conatin list + for value in key_value: + # If 'Ref', insert into Vulnerability or Reference + if data_attrib == "Ref" and __class__.re_cve.search(value): + self.append_value_or_create_list(name_of_object.lower(), "Vulnerability", + __class__.re_cve.split(value)[1], updated_dict) + elif data_attrib == "Ref": + self.append_value_or_create_list(name_of_object.lower(), "Reference", key_value, + updated_dict) + else: + # otherwise just insert value under the key + self.append_value_or_create_list(name_of_object.lower(), data_attrib, value, updated_dict) + + def test_whole_conversion(self): + """ + Load all attributes, which should be converted, from IDEA event into dictionary and then go through all + attributes of all MISP objects and try to pop these attributes from the dictionary. If dictionary will contain + no data at the end, all attributes from IDEA should be converted correctly + :return: + """ + # convert IDEA event to MISP event + misp_converter = IdeaToMisp() + misp_converted_event = json.loads(misp_converter.to_misp(self.idea_event).to_json())['Event'] + + attrib_dict = { + "source": {}, + "target": {}, + "attach": {} + } + + # process Source and Target objects + self.process_source_or_target_object("Source", attrib_dict) + self.process_source_or_target_object("Target", attrib_dict) + + # process Attach objects + for attach_object in self.idea_event['Attach']: + for attach_attrib, key_value in attach_object.items(): + if attach_attrib in ("Handle", "Type", "ContentID", "ExternalURI"): + # these keys are not being converted + continue + if attach_attrib in ("Size", "Note", "ContentType", "ContentCharset", "ContentEncoding", "Content"): + # these keys are just strings, not lists + self.append_value_or_create_list("attach", attach_attrib, key_value, attrib_dict) + else: + # all other values are lists + for value in key_value: + # If 'Ref', insert into Vulnerability or Reference + if attach_attrib == "Ref" and __class__.re_cve.search(value): + self.append_value_or_create_list("attach", "Vulnerability", + __class__.re_cve.split(value)[1], attrib_dict) + elif attach_attrib == "Ref": + self.append_value_or_create_list("attach", "Reference", value, attrib_dict) + elif attach_attrib == "Hash": + # Hash needs to be split and inserted under correct key (hash name) + hash_name = value.split(":", 1)[0].lower() + hash_value = value.split(":", 1)[1] + self.append_value_or_create_list("attach", hash_name, hash_value, attrib_dict) + else: + # otherwise just insert value under the key + self.append_value_or_create_list("attach", attach_attrib, value, attrib_dict) + + attribute_count = 0 + + # Now go through all attributes in all MISP objects and try to remove them from prepared dict before + for misp_object in misp_converted_event['Object']: + for object_attrib in misp_object['Attribute']: + try: + object_name = misp_object['name'] + object_key = object_attrib['object_relation'] + del_value = object_attrib['value'] + attrib_dict[object_name][object_key].remove(del_value) + attribute_count += 1 + except ValueError: + # Some value was inserted, while it should't have + self.fail("Value is in MISP object, but not in IDEA message.") + + # check attribute count + self.assertEqual(attribute_count, 53) + + # Now check all lists under the key, all lists should be empty --> all values were inserted correctly + for object_key in ["source", "target", "attach"]: + for key in attrib_dict[object_key]: + self.assertFalse(bool(attrib_dict[object_key][key])) + + +if __name__ == "__main__": + unittest.main()