Initial commit

schema/idea-warn.schema

+{
+    "$schema": "http://json-schema.org/draft-04/schema#",
+    "description": "IDEA0 basic objects - warning schema",
+    "type": "object",
+    "required": ["Format", "ID", "DetectTime", "Category"],
+    "additionalProperties": false,
+    "definitions": {
+        "UUID": {
+            "description": "ID should be UUID version 4 (random) or 5 (SHA-1).",
+            "type": "string",
+            "pattern": "^[a-f0-9]{8}-[a-f0-9]{4}-[45][a-f0-9]{3}-[89aAbB][a-f0-9]{3}-[a-f0-9]{12}$"
+        },
+        "Count": {
+            "description": "Negative count or size makes no sense.",
+            "type": "integer",
+            "minimum": 0
+        },
+        "EventTagValues": {
+            "description": "Event tag does not bear taxonomy value.",
+            "type": "string",
+            "enum": [
+                "Abusive",
+                "Abusive.Spam",
+                "Abusive.Harassment",
+                "Abusive.Child",
+                "Abusive.Sexual",
+                "Abusive.Violence",
+                "Malware",
+                "Malware.Virus",
+                "Malware.Worm",
+                "Malware.Trojan",
+                "Malware.Spyware",
+                "Malware.Dialer",
+                "Malware.Rootkit",
+                "Recon",
+                "Recon.Scanning",
+                "Recon.Sniffing",
+                "Recon.SocialEngineering",
+                "Recon.Searching",
+                "Attempt",
+                "Attempt.Exploit",
+                "Attempt.Login",
+                "Attempt.NewSignature",
+                "Intrusion",
+                "Intrusion.AdminCompromise",
+                "Intrusion.UserCompromise",
+                "Intrusion.AppCompromise",
+                "Intrusion.Botnet",
+                "Availability",
+                "Availability.DoS",
+                "Availability.DDoS",
+                "Availability.Sabotage",
+                "Availability.Outage",
+                "Information",
+                "Information.UnauthorizedAccess",
+                "Information.UnauthorizedModification",
+                "Fraud",
+                "Fraud.UnauthorizedUsage",
+                "Fraud.Copyright",
+                "Fraud.Masquerade",
+                "Fraud.Phishing",
+                "Fraud.Scam",
+                "Vulnerable",
+                "Vulnerable.Open",
+                "Anomaly",
+                "Anomaly.Traffic",
+                "Anomaly.Connection",
+                "Anomaly.Protocol",
+                "Anomaly.System",
+                "Anomaly.Application",
+                "Anomaly.Behaviour",
+                "Other",
+                "Test"
+            ]
+        },
+        "SourceTargetTagValues": {
+            "description": "Source/Target classification does not bear taxonomy value.",
+            "type": "string",
+            "enum": [
+                "Proxy",
+                "OriginMalware",
+                "OriginSandbox",
+                "OriginSpam",
+                "Phishing",
+                "Malware",
+                "MITM",
+                "Spam",
+                "Backscatter",
+                "Open",
+                "Poisoned",
+                "FastFlux",
+                "Botnet",
+                "CC"
+            ]
+        },
+        "PortNumber": {
+            "description": "Port number out of range.",
+            "type": "integer",
+            "minimum": 1,
+            "maximum": 65535
+        },
+        "ASN": {
+            "description": "ASN invalid.",
+            "type": "integer",
+            "anyOf": [
+                {
+                    "minimum": 1,
+                    "maximum": 65534
+                },
+                {
+                    "minimum": 65536,
+                    "maximum": 4294967294
+                }
+            ]
+        },
+        "RIRNetname": {
+            "description": "Netname contains unknown RIR identifier.",
+            "type": "string",
+            "pattern": "^(?:ripe|arin|apnic|lacnic|afrinic):"
+        },
+        "AttachmentTagValues": {
+            "description": "Attachment description does not bear taxonomy value.",
+            "type": "string",
+            "enum": [
+                "WinLog",
+                "Syslog",
+                "Malware",
+                "ShellCode",
+                "Exploit"
+            ]
+        },
+        "HashHex": {
+            "description": "Unusual hash or hash value not hexadecimal.",
+            "type": "string",
+            "pattern": "^(?:sha1|sha224|sha256|sha384|sha512|md2|md4|md5|crc32|adler32):[0-9a-fA-F]+$"
+        },
+        "NodeTagValues": {
+            "description": "Node description does not bear taxonomy value.",
+            "type": "string",
+            "enum": [
+                "Connection",
+                "Datagram",
+                "Content",
+                "Data",
+                "File",
+                "Flow",
+                "Log",
+                "Protocol",
+                "Host",
+                "Network",
+                "Correlation",
+                "External",
+                "Reporting",
+                "Blackhole",
+                "Signature",
+                "Statistical",
+                "Heuristic",
+                "Integrity",
+                "Policy",
+                "Honeypot",
+                "Tarpit",
+                "Recon",
+                "Monitor"
+            ]
+        }
+    },
+    "properties": {
+        "Format": {},
+        "ID": {
+            "$ref": "#/definitions/UUID"
+        },
+        "AltNames": {
+            "type": "array"
+        },
+        "CorrelID": {
+            "type": "array",
+            "items": {
+                "$ref": "#/definitions/UUID"
+            }
+        },
+        "AggrID": {
+            "type": "array",
+            "items": {
+                "$ref": "#/definitions/UUID"
+            }
+        },
+        "PredID": {
+            "type": "array",
+            "items": {
+                "$ref": "#/definitions/UUID"
+            }
+        },
+        "RelID": {
+            "type": "array",
+            "items": {
+                "$ref": "#/definitions/UUID"
+            }
+        },
+        "CreateTime": {},
+        "DetectTime": {},
+        "EventTime": {},
+        "CeaseTime": {},
+        "WinStartTime": {},
+        "WinEndTime": {},
+        "ConnCount": {
+            "$ref": "#/definitions/Count"
+        },
+        "FlowCount": {
+            "$ref": "#/definitions/Count"
+        },
+        "PacketCount": {
+            "$ref": "#/definitions/Count"
+        },
+        "ByteCount": {
+            "$ref": "#/definitions/Count"
+        },
+        "Category": {
+            "type": "array",
+            "items": {
+                "$ref": "#/definitions/EventTagValues"
+            }
+        },
+        "Ref": {},
+        "Confidence": {},
+        "Description": {},
+        "Note": {},
+        "Source": {
+            "description": "Array of source or target descriptions.",
+            "type": "array",
+            "items": {
+                "type": "object",
+                "description": "Information concerning particular source or target.",
+                "additionalProperties": false,
+                "properties": {
+                    "Type": {
+                        "type": "array",
+                        "items": {
+                            "$ref": "#/definitions/SourceTargetTagValues"
+                        }
+                    },
+                    "Hostname": {},
+                    "IP4": {},
+                    "MAC": {},
+                    "IP6": {},
+                    "Port": {
+                        "type": "array",
+                        "items": {
+                            "$ref": "#/definitions/PortNumber"
+                        }
+                    },
+                    "Proto": {},
+                    "URL": {},
+                    "Email": {},
+                    "AttachHand": {},
+                    "Note": {},
+                    "Spoofed": {},
+                    "Imprecise": {},
+                    "Anonymised": {},
+                    "ASN": {
+                        "type": "array",
+                        "items": {
+                            "$ref": "#/definitions/ASN"
+                        }
+                    },
+                    "Router": {},
+                    "Netname": {
+                        "type": "array",
+                        "items": {
+                            "$ref": "#/definitions/RIRNetname"
+                        }
+                    },
+                    "Ref": {}
+                }
+            }
+        },
+        "Target": {
+            "$ref": "#/properties/Source"
+        },
+        "Attach": {
+            "description": "Array of attachment descriptions.",
+            "type": "array",
+            "items": {
+                "description": "Additional attachment information and data.",
+                "type": "object",
+                "additionalProperties": false,
+                "properties": {
+                    "Handle": {},
+                    "FileName": {},
+                    "Type": {
+                        "type": "array",
+                        "items": {
+                            "$ref": "#/definitions/AttachmentTagValues"
+                        }
+                    },
+                    "Hash": {
+                        "type": "array",
+                        "items": {
+                            "$ref": "#/definitions/HashHex"
+                        }
+                    },
+                    "Size": {
+                        "$ref": "#/definitions/Count"
+                    },
+                    "Ref": {},
+                    "Note": {},
+                    "ContentType": {},
+                    "ContentCharset": {},
+                    "ContentEncoding": {},
+                    "Content": {},
+                    "ContentID": {},
+                    "ExternalURI": {}
+                }
+            }
+        },
+        "Node": {
+            "description": "Array of detector descriptions.",
+            "type": "array",
+            "items": {
+                "description": "Detector or possible intermediary (event aggregator, correlator, etc.) description.",
+                "type": "object",
+                "additionalProperties": false,
+                "properties": {
+                    "Name": {},
+                    "Realm": {},
+                    "Tags": {
+                        "type": "array",
+                        "items": {
+                            "$ref": "#/definitions/NodeTagValues"
+                        }
+                    },
+                    "SW": {},
+                    "AggrWin": {},
+                    "Note": {}
+                }
+            }
+        }
+    }
+}

tools/json-schema-validate.py

+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+from sys import argv, stderr
+from os.path import split
+from json import load, dumps
+from jsonschema import Draft4Validator, FormatChecker
+
+
+def bailout(s):
+    print >>stderr, s
+    exit(1)
+
+
+def loadJSON(p):
+    try:
+        with open(str(p), "r") as f:
+            try:
+                json = load(f)
+            except ValueError, err:
+                bailout ("%s: %s" % (p, err))
+    except IOError, err:
+        bailout(err)
+
+    return json
+
+
+def sortkey(k):
+    """
+    Treat keys as lowercase, prefer keys with less path segments
+    """
+    return (len(k.path), "/".join(str(k.path)).lower())
+
+
+def main():
+    if len(argv)==3:
+        schp, jsonp = argv[1:3]
+    else:
+        bailout("Usage: %s <schemafile> <jsonfile>" % split(argv[0])[-1])
+
+    schema = loadJSON(schp)
+    json = loadJSON(jsonp)
+
+    validator = Draft4Validator(schema, format_checker=FormatChecker())
+    for error in sorted(validator.iter_errors(json), key=sortkey):
+        print (
+            "Validation error: key \"%s\", value \"%s\", expected - %s, error message - %s\n" % (
+                u"/".join(str(v) for v in error.path),
+                error.instance,
+                error.schema.get('description', 'no additional info'),
+                error.message
+            )
+        ).encode("UTF-8")
+
+
+main()

tools/jsonschema2mentat.py

+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+from sys import argv, stderr
+from json import load, dumps
+from collections import Sequence
+from urllib import unquote
+from pprint import pprint
+
+
+def flee(s):
+    print >>stderr, s
+    exit(1)
+
+
+def loadJSON(p):
+    try:
+        with open(str(p), "r") as f:
+            try:
+                json = load(f)
+            except ValueError, err:
+                flee ("%s: %s" % (p, err))
+    except IOError, err:
+        flee(err)
+
+    return json
+
+
+def resolve_ref(document, fragment):
+    """
+    Resolve $ref. Only local relative URIs are supported.
+    """
+    fragment = unquote(fragment).lstrip("#").lstrip("/")
+    parts = fragment.split("/") if fragment else []
+
+    for part in parts:
+        part = part.replace("~1", "/").replace("~0", "~")
+
+        if isinstance(document, Sequence):
+            # Array indexes should be turned into integers
+            try:
+                part = int(part)
+            except ValueError:
+                pass
+        try:
+            document = document[part]
+        except (TypeError, LookupError):
+            flee("Unresolvable JSON pointer: %r" % fragment)
+
+    return document
+
+
+xlattype = {
+    "duration": "timedelta",
+    "object": "document",
+    "number": "real",
+}
+
+
+def process_simple(sch, name, parent, ordlist, card, whole, res):
+    """
+    Generate Mentat schema entry
+    """
+
+    # If IDEA type referenced, use that as type name, otherwise
+    # use basic type, lowercased, possibly renamed.
+    if "$ref" in sch and sch["$ref"].startswith("#/definitions/"):
+        mytype = sch["$ref"].split("/")[-1]
+    else:
+        mytype = sch["type"]
+    mytype = mytype.lower()
+    if mytype in xlattype:
+        mytype = xlattype[mytype]
+
+    # Populate basic Mentat schema data
+    new = {
+        "type": mytype,
+        "ordinality": "mandatory" if name is not None and name in ordlist else "optional",
+        "cardinality": "multi" if card else "single",
+        "description": sch["description"],
+    }
+    if "enum" in sch:
+        new["values"] = sch["enum"]
+    if parent:
+        new["parents"] = [parent]
+
+    # If same name entries exist, create new
+    # Should create only if data differ, but that would need some nitpicky
+    # comparison dancing on all candidates, so we just create duplicates
+    # in any case of clash
+    newname = name
+    while newname in res:
+        new["name"] = name
+        newname = newname + "A"
+
+    res[newname] = new
+
+
+def recurse(sch, name, parent, ordlist, card, whole, res):
+    """
+    Recurse into subtrees based on type.
+    Also crudely resolves references (by merging into current tree).
+    """
+    # Crudely resolve references (by merging into current tree)
+    if "$ref" in sch:
+        ref = resolve_ref(whole, sch["$ref"])
+        sch = dict(ref.items() + sch.items())
+
+    # Jumptable recursion
+    if "type" in sch:
+        fork[sch["type"]](sch, name, parent, ordlist, card, whole, res)
+    else:
+        flee("Basic type not defined: %s", pprint(schema))
+
+
+def process_object(sch, name, parent, ordlist, card, whole, res):
+    """
+    Process object type and recurse for children
+    """
+    # Generate data for non root objects only
+    if name:
+        process_simple(sch, name, parent, ordlist, card, whole, res)
+
+    # Set ordinality list for direct subobject of this object
+    neword = sch["required"] if "required" in sch else []
+
+    # Process each child, with False cardinality
+    for n, subsch in sch["properties"].iteritems():
+        recurse(subsch, n, name, neword, False, whole, res)
+
+
+def process_array(sch, name, parent, ordlist, card, whole, res):
+    """
+    Process array type (recurse just for one "items" children, but indicate
+    cardinality
+    """
+    recurse(sch["items"], name, parent, ordlist, True, whole, res)
+
+
+fork = {
+    "object": process_object,
+    "array": process_array,
+    "string": process_simple,
+    "number": process_simple,
+    "integer": process_simple,
+    "boolean": process_simple
+}
+
+
+def main():
+    if len(argv)==2:
+        schp = argv[1]
+    else:
+        flee("Usage: %s <jsonschemafile>" % split(argv[0])[-1])
+
+    schema = loadJSON(schp)
+
+    res = {}
+    # First structure of JSON schema must be object
+    process_object(schema, None, "", [""], False, schema, res)
+
+    print dumps(res, indent=8, sort_keys=True)
+
+
+main()