From 68fd3194dade090e9e79a2e8206b28ad5eb89a1a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Pavel=20K=C3=A1cha?= <ph@cesnet.cz>
Date: Fri, 16 Jan 2015 16:38:43 +0100
Subject: [PATCH] Moved category and tag mappings into configurable json files,
wrong cat/tag now raises Error
---
warden3/warden_server/catmap_mysql.json | 55 ++++++++++
warden3/warden_server/tagmap_mysql.json | 26 +++++
warden3/warden_server/warden_server.py | 133 +++++-------------------
3 files changed, 107 insertions(+), 107 deletions(-)
create mode 100644 warden3/warden_server/catmap_mysql.json
create mode 100644 warden3/warden_server/tagmap_mysql.json
diff --git a/warden3/warden_server/catmap_mysql.json b/warden3/warden_server/catmap_mysql.json
new file mode 100644
index 0000000..d995202
--- /dev/null
+++ b/warden3/warden_server/catmap_mysql.json
@@ -0,0 +1,55 @@
+{
+ "Abusive" : 100,
+ "Abusive.Spam" : 101,
+ "Abusive.Harassment" : 102,
+ "Abusive.Child" : 103,
+ "Abusive.Sexual" : 104,
+ "Abusive.Violence" : 105,
+ "Malware" : 200,
+ "Malware.Virus" : 201,
+ "Malware.Worm" : 202,
+ "Malware.Trojan" : 203,
+ "Malware.Spyware" : 204,
+ "Malware.Dialer" : 205,
+ "Malware.Rootkit" : 206,
+ "Recon.Scanning" : 3,
+ "Recon.Scanning" : 301,
+ "Recon.Sniffing" : 302,
+ "Recon.SocialEngineering" : 303,
+ "Recon.Searching" : 304,
+ "Attempt" : 400,
+ "Attempt.Exploit" : 401,
+ "Attempt.Login" : 402,
+ "Attempt.NewSignature" : 403,
+ "Intrusion" : 500,
+ "Intrusion.AdminCompromise" : 501,
+ "Intrusion.UserCompromise" : 502,
+ "Intrusion.AppCompromise" : 503,
+ "Intrusion.Botnet" : 504,
+ "Availability" : 600,
+ "Availability.DoS" : 601,
+ "Availability.DDoS" : 602,
+ "Availability.Sabotage" : 603,
+ "Availability.Outage" : 604,
+ "Information" : 700,
+ "Information.UnauthorizedAccess" : 701,
+ "Information.UnauthorizedModification" : 702,
+ "Fraud" : 800,
+ "Fraud.UnauthorizedUsage" : 801,
+ "Fraud.Copyright" : 802,
+ "Fraud.Masquerade" : 803,
+ "Fraud.Phishing" : 804,
+ "Fraud.Scam" : 805,
+ "Vulnerable" : 900,
+ "Vulnerable.Open" : 901,
+ "Anomaly" : 1000,
+ "Anomaly.Traffic" : 1001,
+ "Anomaly.Connection" : 1002,
+ "Anomaly.Protocol" : 1003,
+ "Anomaly.System" : 1004,
+ "Anomaly.Application" : 1005,
+ "Anomaly.Behaviour" : 1006,
+ "Other" : 9998,
+ "Test" : 9999
+}
+
diff --git a/warden3/warden_server/tagmap_mysql.json b/warden3/warden_server/tagmap_mysql.json
new file mode 100644
index 0000000..98e3b5c
--- /dev/null
+++ b/warden3/warden_server/tagmap_mysql.json
@@ -0,0 +1,26 @@
+{
+ "Connection": 1,
+ "Datagram": 2,
+ "Content": 3,
+ "Data": 4,
+ "File": 5,
+ "Flow": 6,
+ "Log": 7,
+ "Protocol": 8,
+ "Host": 9,
+ "Network": 10,
+ "Correlation": 11,
+ "External": 12,
+ "Reporting": 13,
+ "Blackhole": 30,
+ "Signature": 31,
+ "Statistical": 32,
+ "Heuristic": 33,
+ "Integrity": 34,
+ "Policy": 35,
+ "Honeypot": 36,
+ "Tarpit": 37,
+ "Recon": 38,
+ "Monitor": 39,
+ "Other" : 99
+}
diff --git a/warden3/warden_server/warden_server.py b/warden3/warden_server/warden_server.py
index ac06c2c..9f02135 100755
--- a/warden3/warden_server/warden_server.py
+++ b/warden3/warden_server/warden_server.py
@@ -271,12 +271,22 @@ class JSONSchemaValidator(NoValidator):
class MySQL(Object):
- def __init__(self, host, user, password, dbname, port):
+ def __init__(self, host, user, password, dbname, port, catmap_filename, tagmap_filename):
self.host = host
self.user = user
self.password = password
self.dbname = dbname
self.port = port
+ self.catmap_filename = catmap_filename
+ self.tagmap_filename = tagmap_filename
+
+ with open(catmap_filename, "r") as catmap_fd:
+ self.catmap = json.load(catmap_fd)
+ self.catmap_other = self.catmap["Other"] # Catch error soon, avoid lookup later
+
+ with open(tagmap_filename, "r") as tagmap_fd:
+ self.tagmap = json.load(tagmap_fd)
+ self.tagmap_other = self.catmap["Other"] # Catch error soon, avoid lookup later
self.con = my.connect(host=self.host, user=self.user, passwd=self.password,
db=self.dbname, port=self.port, cursorclass=mycursors.DictCursor)
@@ -284,8 +294,8 @@ class MySQL(Object):
def __str__(self):
- return "%s(host='%s', user='%s', dbname='%s', port=%d)" % (
- type(self).__name__, self.host, self.user, self.dbname, self.port)
+ return "%s(host='%s', user='%s', dbname='%s', port=%d, catmap_filename=\"%s\", tagmap_filename=\"%s\")" % (
+ type(self).__name__, self.host, self.user, self.dbname, self.port, self.catmap_filename, self.tagmap_filename)
def get_client_by_name(self, name):
@@ -325,28 +335,16 @@ class MySQL(Object):
return {}
- def gen_random_idea(self):
-
- def get_precise_timestamp():
- t = time()
- us = trunc((t-trunc(t))*1000000)
- g = gmtime(t)
- iso = '%04d-%02d-%02dT%02d:%02d:%02d.%0dZ' % (g[0:6]+(us,))
- return iso
-
- return {
- "Format": "IDEA0",
- "ID": str(uuid4()),
- "DetectTime": get_precise_timestamp(),
- "Category": ["Test"],
- }
-
-
def generateDynamicQuery(self, section, query_string, variables, parent_cats = []):
variables_id = []
for v in variables:
- mapped_id = self.map_id(section, v)
+ try:
+ mapped_id = section[v]
+ except KeyError:
+ raise Error("Wrong tag or category used in query.", 422, method='getEvents',
+ exc=sys.exc_info(), detail={"key": v})
+
if mapped_id % 100:
variables_id.append(mapped_id)
else:
@@ -384,7 +382,7 @@ class MySQL(Object):
if cat or nocat:
not_op = "" if cat else "NOT"
parent_cats = []
- sqltemp, sqlpar = self.generateDynamicQuery("Category", "category_id %s IN (%%s)" % not_op, (cat or nocat), parent_cats)
+ sqltemp, sqlpar = self.generateDynamicQuery(self.catmap, "category_id %s IN (%%s)" % not_op, (cat or nocat), parent_cats)
for pcats in parent_cats:
sqltemp += " %s category_id DIV %s = 1 " % (("OR" if sqltemp else ""), pcats)
@@ -393,7 +391,7 @@ class MySQL(Object):
if tag or notag:
not_op = "" if tag else "NOT"
- sqltemp, sqlpar = self.generateDynamicQuery("Tag", "tag_id %s IN (%%s)" % not_op, (tag or notag))
+ sqltemp, sqlpar = self.generateDynamicQuery(self.tagmap, "tag_id %s IN (%%s)" % not_op, (tag or notag))
sqlwhere.append(" AND e.id IN (SELECT event_id FROM event_tag_mapping WHERE %s)" % sqltemp)
sqlparams.extend(sqlpar)
@@ -435,7 +433,7 @@ class MySQL(Object):
logging.debug("store_event: Last ID in events - %i" % lastid)
for cat in event.get('Category', ["Other"]):
- cat_id = self.map_id('Category', cat) or self.map_id('Category', 'Other')
+ cat_id = self.catmap.get(cat, self.catmap_other)
logging.debug("store_event: Category \"%s\" translated to %i" % (cat, cat_id))
self.crs.execute("INSERT INTO event_category_mapping (event_id,category_id) VALUES (%s, %s)", (lastid, cat_id))
@@ -445,7 +443,7 @@ class MySQL(Object):
tags = []
for tag in tags:
- tag_id = self.map_id('Tag', tag) or self.map_id('Tag', 'Other')
+ tag_id = self.tagmap.get(tag, self.tagmap_other)
logging.debug("store_event: Tag \"%s\" translated to %i" % (tag, tag_id))
self.crs.execute("INSERT INTO event_tag_mapping (event_id,tag_id) VALUES (%s, %s)", (lastid, tag_id))
@@ -476,87 +474,6 @@ class MySQL(Object):
return id
- def map_id (self, section, key):
- # Should by placed in config file
- data = {}
- data['Tag'] = {
- "Connection" : 1,
- "Datagram" : 2,
- "Content" : 3,
- "Data" : 4,
- "File" : 5,
- "Flow" : 6,
- "Log": 7,
- "Protocol" : 8,
- "Host" : 9,
- "Network" : 10,
- "Correlation" : 11,
- "External" : 12,
- "Reporting" : 13,
- "Other" : 99
- }
-
- data['Category'] = {
- "Abusive" : 100,
- "Abusive.Spam" : 101,
- "Abusive.Harassment" : 102,
- "Abusive.Child" : 103,
- "Abusive.Sexual" : 104,
- "Abusive.Violence" : 105,
- "Malware" : 200,
- "Malware.Virus" : 201,
- "Malware.Worm" : 202,
- "Malware.Trojan" : 203,
- "Malware.Spyware" : 204,
- "Malware.Dialer" : 205,
- "Malware.Rootkit" : 206,
- "Recon.Scanning" : 3,
- "Recon.Scanning" : 301,
- "Recon.Sniffing" : 302,
- "Recon.SocialEngineering" : 303,
- "Recon.Searching" : 304,
- "Attempt" : 400,
- "Attempt.Exploit" : 401,
- "Attempt.Login" : 402,
- "Attempt.NewSignature" : 403,
- "Intrusion" : 500,
- "Intrusion.AdminCompromise" : 501,
- "Intrusion.UserCompromise" : 502,
- "Intrusion.AppCompromise" : 503,
- "Intrusion.Botnet" : 504,
- "Availability" : 600,
- "Availability.DoS" : 601,
- "Availability.DDoS" : 602,
- "Availability.Sabotage" : 603,
- "Availability.Outage" : 604,
- "Information" : 700,
- "Information.UnauthorizedAccess" : 701,
- "Information.UnauthorizedModification" : 702,
- "Fraud" : 800,
- "Fraud.UnauthorizedUsage" : 801,
- "Fraud.Copyright" : 802,
- "Fraud.Masquerade" : 803,
- "Fraud.Phishing" : 804,
- "Fraud.Scam" : 805,
- "Vulnerable" : 900,
- "Vulnerable.Open" : 901,
- "Anomaly" : 1000,
- "Anomaly.Traffic" : 1001,
- "Anomaly.Connection" : 1002,
- "Anomaly.Protocol" : 1003,
- "Anomaly.System" : 1004,
- "Anomaly.Application" : 1005,
- "Anomaly.Behaviour" : 1006,
- "Other" : 9998,
- "Test" : 9999,
- }
-
- try:
- return data[section][key]
- except KeyError:
- return 0
-
-
def expose(meth):
meth.exposed = True
@@ -914,7 +831,9 @@ def build_server(conf):
"user": {"type": str, "default": "warden"},
"password": {"type": str, "default": ""},
"dbname": {"type": str, "default": "warden3"},
- "port": {"type": natural, "default": 3306}
+ "port": {"type": natural, "default": 3306},
+ "catmap_filename": {"type": filepath, "default": path.join(path.dirname(__file__), "catmap_mysql.json")},
+ "tagmap_filename": {"type": filepath, "default": path.join(path.dirname(__file__), "tagmap_mysql.json")}
},
"WardenHandler": {
"validator": {"type": obj, "default": "validator"},
--
GitLab