Skip to content
Snippets Groups Projects
Commit e8e12ce0 authored by Pavel Kácha's avatar Pavel Kácha
Browse files

Fixed multiple HTTP args encoding/parsing, store_event now uses SQL... 

Fixed multiple HTTP args encoding/parsing, store_event now uses SQL placeholders, more robust fetching of cat/tag in store_event, simplified map_id usage
parent 1eea2bfc
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
warden3/warden_client/warden_client.py
+ 2
− 5
View file @ e8e12ce0
......@@ -253,7 +253,7 @@ class Client(object):
for k in kwargs.keys():
if kwargs[k] is None:
del kwargs[k]
argurl = "?" + urlencode(kwargs)
argurl = "?" + urlencode(kwargs, doseq=True)
else:
argurl = ""
......@@ -362,10 +362,7 @@ class Client(object):
def sendEvents(self, events=[]):
res = self.sendRequest(
"sendEvents", payload=events)
if not res:
return res # Should be Error instance
return res.get("saved", 0)
return res
def getEvents(self, id=None, idstore=None, count=1,
......
warden3/warden_client/warden_client_test.py
+ 15
− 15
View file @ e8e12ce0
......@@ -31,7 +31,7 @@ def gen_min_idea():
"Category": ["Test"],
}
def gen_random_idea():
def gen_random_idea(client_name="cz.example.warden.test"):
def geniprange(gen):
......@@ -115,8 +115,7 @@ def gen_random_idea():
],
"Node": [
{
"Name": "com.example.test-node",
# "Name": "Test",
"Name": client_name,
"Tags": ["Protocol", "Honeypot"],
"SW": ["Kippo"],
"AggrWin": "00:05:00"
......@@ -139,32 +138,33 @@ def main():
# errlog={"level": "debug"},
# filelog={"level": "debug"},
# idstore="MyClient.id",
# name="MyClient")
# name="cz.example.warden.test")
print "=== Getting 10 events ==="
start = time()
cat = json.dumps(['Availability', 'Abusive.Spam','Attempt.Login'])
nocat = json.dumps(['Fraud.Scam','Malware.Virus'])
cat = ['Availability', 'Abusive.Spam','Attempt.Login']
nocat = ['Fraud.Scam','Malware.Virus']
tag = json.dumps(['Log', 'Data'])
notag = json.dumps(['Flow', 'Datagram'])
tag = ['Log', 'Data']
notag = ['Flow', 'Datagram']
group = json.dumps(['cz.tul.ward.kippo','cz.vsb.buldog.kippo'])
nogroup = json.dumps(['cz.zcu.civ.afrodita.%','cz.vutbr.net.bee.hpscan'])
group = ['cz.tul.ward.kippo','cz.vsb.buldog.kippo']
nogroup = ['cz.zcu.civ.afrodita','cz.vutbr.net.bee.hpscan']
ret = wclient.getEvents(count=10, cat=cat, nocat=None, tag=tag, notag=None, group=None, nogroup=nogroup)
#ret = wclient.getEvents(count=10)
print "Time: %f" % (time()-start)
print "Got %i events" % len(ret)
for e in ret:
print e
if ret:
print len(ret)
if isinstance(ret, Error):
print ret
print "=== Sending 1 event(s) ==="
start = time()
ret = wclient.sendEvents([gen_random_idea() for i in range(1)])
if ret:
print ret
ret = wclient.sendEvents([gen_random_idea(client_name=wclient.name) for i in range(1)])
print ret
print "Time: %f" % (time()-start)
print "=== Server info ==="
......
warden3/warden_server/warden_server.py
+ 46
− 41
View file @ e8e12ce0
......@@ -291,7 +291,6 @@ class MySQL(Object):
format_strings = ','.join(['%s'] * len(name))
self.crs.execute("SELECT cl.`id`, cl.`hostname`, s.`service`, s.`service_id`, s.`identity`, cl.`read`, s.`write`, s.`test`, cl.`debug` FROM `clients` cl LEFT JOIN `services` s ON cl.`id` = s.`client_id` WHERE cl.`valid` = 1 AND s.`valid` = 1 AND `hostname` IN (%s)" % format_strings, tuple(name))
rows = self.crs.fetchall()
logging.debug("Client/service info: " + str(rows))
if not rows:
return None
......@@ -308,7 +307,7 @@ class MySQL(Object):
client["services"] = services
logging.debug("Client/service formatted info: " + str(client))
logging.debug("Client/services: " + str(client))
return client
......@@ -350,6 +349,8 @@ class MySQL(Object):
sqlwhere = []
sqltemp = {}
logging.debug("fetch_events: id=%i, count=%i, cat=%s, nocat=%s, tag=%s, notag=%s, group=%s, nogroup=%s" % (id, count, str(cat), str(nocat), str(tag), str(notag), str(group), str(nogroup)))
if cat is not None and nocat is not None:
raise Error("Unrealizable conditions. Choose cat or nocat option.", 500, method='getEvents',
exc=sys.exc_info(), detail={'cat': cat, 'nocat' : nocat})
......@@ -357,12 +358,12 @@ class MySQL(Object):
if cat is not None or nocat is not None:
if cat is not None:
parent_cats = []
sqltemp['cat'] = self.generateDynamicQuery("Category", "category_id IN (%s)", json.loads(cat), parent_cats)
sqltemp['cat'] = self.generateDynamicQuery("Category", "category_id IN (%s)", cat, parent_cats)
for pcats in parent_cats:
sqltemp['cat'] += " %s category_id DIV %s = 1 " % (("OR" if sqltemp['cat'] else ""), pcats)
if nocat is not None:
parent_cats = []
sqltemp['cat'] = self.generateDynamicQuery("Category", "category_id NOT IN (%s)", json.loads(nocat), parent_cats)
sqltemp['cat'] = self.generateDynamicQuery("Category", "category_id NOT IN (%s)", nocat, parent_cats)
for pcats in parent_cats:
sqltemp['cat'] += " %s category_id DIV %s = 1 " % (("OR" if sqltemp['cat'] else ""), pcats)
......@@ -374,9 +375,9 @@ class MySQL(Object):
if tag is not None or notag is not None:
if tag is not None:
sqltemp['tag'] = self.generateDynamicQuery("Tag", "tag_id IN (%s)", json.loads(tag))
sqltemp['tag'] = self.generateDynamicQuery("Tag", "tag_id IN (%s)", tag)
if notag is not None:
sqltemp['tag'] = self.generateDynamicQuery("Tag", "tag_id NOT IN (%s)", json.loads(notag))
sqltemp['tag'] = self.generateDynamicQuery("Tag", "tag_id NOT IN (%s)", notag)
sqlwhere.append("e.id IN (SELECT event_id FROM event_tag_mapping WHERE %s)" % sqltemp['tag'])
......@@ -389,10 +390,10 @@ class MySQL(Object):
sqltemp['group'] = ""
if group is not None:
for identity in json.loads(group):
for identity in group:
sqltemp['group'] += ("s.identity LIKE '%s' AND " % (identity))
if nogroup is not None:
for identity in json.loads(nogroup):
for identity in nogroup:
sqltemp['group'] += ("s.identity NOT LIKE '%s' AND " % (identity))
# logging.debug(sqltemp['group'][:-4])
......@@ -415,38 +416,42 @@ class MySQL(Object):
maxid = self.getLastEventId()
# logging.debug("MAX ID = %s", str(maxid))
#for r in row:
# logging.debug(json.loads(r["data"]))
return {
"lastid": maxid,
# "lastid": row[-1]['id'] if row else str(id),
"events": [row[i]['data'] for i in range(len(row))]
"events": [json.loads(r["data"]) for r in row]
}
def store_event(self, client, event):
try:
# logging.debug("INSERT INTO events (received,service_id,data) VALUES (NOW(), '%s', '%s')" % (event['DetectTime'], client["service"]["service_id"], self.con.escape_string(str(event))))
self.crs.execute("INSERT INTO events (received,service_id,data) VALUES (NOW(), '%s', '%s')" % (client["service"]["service_id"], self.con.escape_string(str(event))))
self.crs.execute("INSERT INTO events (received,service_id,data) VALUES (NOW(), %s, %s)", (client["service"]["service_id"], json.dumps(event)))
lastid = self.crs.lastrowid
# logging.debug(str(lastid))
for cat in event['Category']:
# logging.debug({'cat': cat})
cat_id = self.map_id('Category', cat) if self.map_id('Category', cat) else self.map_id('Category', 'Other.Other')
# logging.debug({'cat_id': cat_id})
# logging.debug("INSERT INTO event_category_mapping (event_id,category_id) VALUES ('%s', '%s')" % (str(lastid), str(cat_id)))
self.crs.execute("INSERT INTO event_category_mapping (event_id,category_id) VALUES ('%s', '%s')" % (str(lastid), str(cat_id)))
logging.debug("store_event: Last ID in events - %i" % lastid)
for cat in event.get('Category', ["Other"]):
cat_id = self.map_id('Category', cat) or self.map_id('Category', 'Other')
logging.debug("store_event: Category \"%s\" translated to %i" % (cat, cat_id))
self.crs.execute("INSERT INTO event_category_mapping (event_id,category_id) VALUES (%s, %s)", (lastid, cat_id))
for tag in event['Node'][0]['Tags']:
tag_id = self.map_id('Tag', tag) if self.map_id('Tag', tag) else self.map_id('Tag', 'Other')
# logging.debug({'tag_id': tag_id})
# logging.debug("INSERT INTO event_tag_mapping (event_id,tag_id) VALUES ('%s', '%s')" % (str(lastid), tag_id))
self.crs.execute("INSERT INTO event_tag_mapping (event_id,tag_id) VALUES ('%s', '%s')" % (str(lastid), str(tag_id)))
try:
tags = event['Node'][0]['Tags']
except (KeyError, IndexError):
tags = []
for tag in tags:
tag_id = self.map_id('Tag', tag) or self.map_id('Tag', 'Other')
logging.debug("store_event: Tag \"%s\" translated to %i" % (tag, tag_id))
self.crs.execute("INSERT INTO event_tag_mapping (event_id,tag_id) VALUES (%s, %s)", (lastid, tag_id))
self.con.commit()
return []
except Exception as e:
self.con.rollback()
return [{"event": event, "error": type(e).__name__ + ": " + str(e)}]
return [type(e).__name__ + ": " + str(e)]
def insertLastReceivedId(self, client, id):
......@@ -472,7 +477,7 @@ class MySQL(Object):
return row['id'] if row is not None else 0
def map_id (self, section, key, strict = False):
def map_id (self, section, key):
# Should by placed in config file
data = {}
data['Tag'] = {
......@@ -549,9 +554,8 @@ class MySQL(Object):
try:
return data[section][key]
except:
#Return 0 for strict mode (searching), otherwise map everything else to 'Other'
return 0 if strict else data[section]['Other']
except KeyError:
return 0
def generateDynamicQuery(self, section, query_string, variables, parent_cats = []):
......@@ -559,7 +563,7 @@ class MySQL(Object):
# parent_cats = []
for v in variables:
mapped_id = self.map_id(section, v, True)
mapped_id = self.map_id(section, v)
if mapped_id % 100 != 0:
variables_id.append(mapped_id)
else:
......@@ -639,8 +643,8 @@ class Server(Object):
exc=sys.exc_info(), detail={"args": injson})
args = parse_qs(environ.get('QUERY_STRING', ""))
for k, v in args.iteritems():
args[k] = v[0]
#for k, v in args.iteritems():
# args[k] = v[0]
logging.debug("%s called with %s" % (path, str(args)))
if events:
args["events"] = events
......@@ -742,24 +746,25 @@ class WardenHandler(Object):
group=None, nogroup=None):
try:
id = int(id)
except (ValueError, TypeError):
id=0
id = int(id[0])
except (ValueError, TypeError, IndexError):
id = 0
if id == 0:
if not id:
try:
id = self.db.getLastReceivedId(_client)
# logging.debug("Last received ID for %s is %s" % (_client['hostname'], str(id)))
except Exception, e:
logging.error(e)
logging.error("getEvents: cannot getLastReceivedId - " + type(e).__name__ + ": " + e)
id = 0
if id == 0:
if not id:
try:
id = self.db.getLastEventId()
except Exception as e:
raise Error("Last event id receiving error", 500, detail={"client": _client})
# First access, remember the guy
self.db.insertLastReceivedId(_client, id)
return {
......@@ -768,17 +773,17 @@ class WardenHandler(Object):
}
try:
count = int(count)
except (ValueError, TypeError):
count = int(count[0])
except (ValueError, TypeError, IndexError):
count = 1
if self.get_events_limit:
count = min(count, self.get_events_limit)
logging.debug("getEvents - count: %s" % count)
res = self.db.fetch_events(_client, id or 0, count, cat, nocat, tag, notag, group, nogroup)
res = self.db.fetch_events(_client, id, count, cat, nocat, tag, notag, group, nogroup)
logging.info("getEvents(%d, %d, %s, %s, %s, %s, %s, %s): sending %d events" % (
id or 0, count, cat, nocat, tag, notag, group, nogroup, len(res["events"])))
id, count, cat, nocat, tag, notag, group, nogroup, len(res["events"])))
self.db.insertLastReceivedId(_client, res['lastid'])
logging.debug("lastid inserting: %s" % {'lastid': res['lastid'], 'client' : _client})
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment