Skip to content
Snippets Groups Projects
Commit e8e12ce0 authored by Pavel Kácha's avatar Pavel Kácha
Browse files

Fixed multiple HTTP args encoding/parsing, store_event now uses SQL... 

Fixed multiple HTTP args encoding/parsing, store_event now uses SQL placeholders, more robust fetching of cat/tag in store_event, simplified map_id usage
parent 1eea2bfc
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
warden3/warden_client/warden_client.py
+ 2
− 5
View file @ e8e12ce0
...@@ -253,7 +253,7 @@ class Client(object): ...@@ -253,7 +253,7 @@ class Client(object):
for k in kwargs.keys(): for k in kwargs.keys():
if kwargs[k] is None: if kwargs[k] is None:
del kwargs[k] del kwargs[k]
argurl = "?" + urlencode(kwargs) argurl = "?" + urlencode(kwargs, doseq=True)
else: else:
argurl = "" argurl = ""
...@@ -362,10 +362,7 @@ class Client(object): ...@@ -362,10 +362,7 @@ class Client(object):
def sendEvents(self, events=[]): def sendEvents(self, events=[]):
res = self.sendRequest( res = self.sendRequest(
"sendEvents", payload=events) "sendEvents", payload=events)
if not res: return res
return res # Should be Error instance
return res.get("saved", 0)
def getEvents(self, id=None, idstore=None, count=1, def getEvents(self, id=None, idstore=None, count=1,
......
warden3/warden_client/warden_client_test.py
+ 15
− 15
View file @ e8e12ce0
...@@ -31,7 +31,7 @@ def gen_min_idea(): ...@@ -31,7 +31,7 @@ def gen_min_idea():
"Category": ["Test"], "Category": ["Test"],
} }
def gen_random_idea(): def gen_random_idea(client_name="cz.example.warden.test"):
def geniprange(gen): def geniprange(gen):
...@@ -115,8 +115,7 @@ def gen_random_idea(): ...@@ -115,8 +115,7 @@ def gen_random_idea():
], ],
"Node": [ "Node": [
{ {
"Name": "com.example.test-node", "Name": client_name,
# "Name": "Test",
"Tags": ["Protocol", "Honeypot"], "Tags": ["Protocol", "Honeypot"],
"SW": ["Kippo"], "SW": ["Kippo"],
"AggrWin": "00:05:00" "AggrWin": "00:05:00"
...@@ -139,32 +138,33 @@ def main(): ...@@ -139,32 +138,33 @@ def main():
# errlog={"level": "debug"}, # errlog={"level": "debug"},
# filelog={"level": "debug"}, # filelog={"level": "debug"},
# idstore="MyClient.id", # idstore="MyClient.id",
# name="MyClient") # name="cz.example.warden.test")
print "=== Getting 10 events ===" print "=== Getting 10 events ==="
start = time() start = time()
cat = json.dumps(['Availability', 'Abusive.Spam','Attempt.Login']) cat = ['Availability', 'Abusive.Spam','Attempt.Login']
nocat = json.dumps(['Fraud.Scam','Malware.Virus']) nocat = ['Fraud.Scam','Malware.Virus']
tag = json.dumps(['Log', 'Data']) tag = ['Log', 'Data']
notag = json.dumps(['Flow', 'Datagram']) notag = ['Flow', 'Datagram']
group = json.dumps(['cz.tul.ward.kippo','cz.vsb.buldog.kippo']) group = ['cz.tul.ward.kippo','cz.vsb.buldog.kippo']
nogroup = json.dumps(['cz.zcu.civ.afrodita.%','cz.vutbr.net.bee.hpscan']) nogroup = ['cz.zcu.civ.afrodita','cz.vutbr.net.bee.hpscan']
ret = wclient.getEvents(count=10, cat=cat, nocat=None, tag=tag, notag=None, group=None, nogroup=nogroup) ret = wclient.getEvents(count=10, cat=cat, nocat=None, tag=tag, notag=None, group=None, nogroup=nogroup)
#ret = wclient.getEvents(count=10)
print "Time: %f" % (time()-start) print "Time: %f" % (time()-start)
print "Got %i events" % len(ret)
for e in ret: for e in ret:
print e print e
if ret: if isinstance(ret, Error):
print len(ret) print ret
print "=== Sending 1 event(s) ===" print "=== Sending 1 event(s) ==="
start = time() start = time()
ret = wclient.sendEvents([gen_random_idea() for i in range(1)]) ret = wclient.sendEvents([gen_random_idea(client_name=wclient.name) for i in range(1)])
if ret: print ret
print ret
print "Time: %f" % (time()-start) print "Time: %f" % (time()-start)
print "=== Server info ===" print "=== Server info ==="
......
warden3/warden_server/warden_server.py
+ 46
− 41
View file @ e8e12ce0
...@@ -291,7 +291,6 @@ class MySQL(Object): ...@@ -291,7 +291,6 @@ class MySQL(Object):
format_strings = ','.join(['%s'] * len(name)) format_strings = ','.join(['%s'] * len(name))
self.crs.execute("SELECT cl.`id`, cl.`hostname`, s.`service`, s.`service_id`, s.`identity`, cl.`read`, s.`write`, s.`test`, cl.`debug` FROM `clients` cl LEFT JOIN `services` s ON cl.`id` = s.`client_id` WHERE cl.`valid` = 1 AND s.`valid` = 1 AND `hostname` IN (%s)" % format_strings, tuple(name)) self.crs.execute("SELECT cl.`id`, cl.`hostname`, s.`service`, s.`service_id`, s.`identity`, cl.`read`, s.`write`, s.`test`, cl.`debug` FROM `clients` cl LEFT JOIN `services` s ON cl.`id` = s.`client_id` WHERE cl.`valid` = 1 AND s.`valid` = 1 AND `hostname` IN (%s)" % format_strings, tuple(name))
rows = self.crs.fetchall() rows = self.crs.fetchall()
logging.debug("Client/service info: " + str(rows))
if not rows: if not rows:
return None return None
...@@ -308,7 +307,7 @@ class MySQL(Object): ...@@ -308,7 +307,7 @@ class MySQL(Object):
client["services"] = services client["services"] = services
logging.debug("Client/service formatted info: " + str(client)) logging.debug("Client/services: " + str(client))
return client return client
...@@ -350,6 +349,8 @@ class MySQL(Object): ...@@ -350,6 +349,8 @@ class MySQL(Object):
sqlwhere = [] sqlwhere = []
sqltemp = {} sqltemp = {}
logging.debug("fetch_events: id=%i, count=%i, cat=%s, nocat=%s, tag=%s, notag=%s, group=%s, nogroup=%s" % (id, count, str(cat), str(nocat), str(tag), str(notag), str(group), str(nogroup)))
if cat is not None and nocat is not None: if cat is not None and nocat is not None:
raise Error("Unrealizable conditions. Choose cat or nocat option.", 500, method='getEvents', raise Error("Unrealizable conditions. Choose cat or nocat option.", 500, method='getEvents',
exc=sys.exc_info(), detail={'cat': cat, 'nocat' : nocat}) exc=sys.exc_info(), detail={'cat': cat, 'nocat' : nocat})
...@@ -357,12 +358,12 @@ class MySQL(Object): ...@@ -357,12 +358,12 @@ class MySQL(Object):
if cat is not None or nocat is not None: if cat is not None or nocat is not None:
if cat is not None: if cat is not None:
parent_cats = [] parent_cats = []
sqltemp['cat'] = self.generateDynamicQuery("Category", "category_id IN (%s)", json.loads(cat), parent_cats) sqltemp['cat'] = self.generateDynamicQuery("Category", "category_id IN (%s)", cat, parent_cats)
for pcats in parent_cats: for pcats in parent_cats:
sqltemp['cat'] += " %s category_id DIV %s = 1 " % (("OR" if sqltemp['cat'] else ""), pcats) sqltemp['cat'] += " %s category_id DIV %s = 1 " % (("OR" if sqltemp['cat'] else ""), pcats)
if nocat is not None: if nocat is not None:
parent_cats = [] parent_cats = []
sqltemp['cat'] = self.generateDynamicQuery("Category", "category_id NOT IN (%s)", json.loads(nocat), parent_cats) sqltemp['cat'] = self.generateDynamicQuery("Category", "category_id NOT IN (%s)", nocat, parent_cats)
for pcats in parent_cats: for pcats in parent_cats:
sqltemp['cat'] += " %s category_id DIV %s = 1 " % (("OR" if sqltemp['cat'] else ""), pcats) sqltemp['cat'] += " %s category_id DIV %s = 1 " % (("OR" if sqltemp['cat'] else ""), pcats)
...@@ -374,9 +375,9 @@ class MySQL(Object): ...@@ -374,9 +375,9 @@ class MySQL(Object):
if tag is not None or notag is not None: if tag is not None or notag is not None:
if tag is not None: if tag is not None:
sqltemp['tag'] = self.generateDynamicQuery("Tag", "tag_id IN (%s)", json.loads(tag)) sqltemp['tag'] = self.generateDynamicQuery("Tag", "tag_id IN (%s)", tag)
if notag is not None: if notag is not None:
sqltemp['tag'] = self.generateDynamicQuery("Tag", "tag_id NOT IN (%s)", json.loads(notag)) sqltemp['tag'] = self.generateDynamicQuery("Tag", "tag_id NOT IN (%s)", notag)
sqlwhere.append("e.id IN (SELECT event_id FROM event_tag_mapping WHERE %s)" % sqltemp['tag']) sqlwhere.append("e.id IN (SELECT event_id FROM event_tag_mapping WHERE %s)" % sqltemp['tag'])
...@@ -389,10 +390,10 @@ class MySQL(Object): ...@@ -389,10 +390,10 @@ class MySQL(Object):
sqltemp['group'] = "" sqltemp['group'] = ""
if group is not None: if group is not None:
for identity in json.loads(group): for identity in group:
sqltemp['group'] += ("s.identity LIKE '%s' AND " % (identity)) sqltemp['group'] += ("s.identity LIKE '%s' AND " % (identity))
if nogroup is not None: if nogroup is not None:
for identity in json.loads(nogroup): for identity in nogroup:
sqltemp['group'] += ("s.identity NOT LIKE '%s' AND " % (identity)) sqltemp['group'] += ("s.identity NOT LIKE '%s' AND " % (identity))
# logging.debug(sqltemp['group'][:-4]) # logging.debug(sqltemp['group'][:-4])
...@@ -415,38 +416,42 @@ class MySQL(Object): ...@@ -415,38 +416,42 @@ class MySQL(Object):
maxid = self.getLastEventId() maxid = self.getLastEventId()
# logging.debug("MAX ID = %s", str(maxid)) # logging.debug("MAX ID = %s", str(maxid))
#for r in row:
# logging.debug(json.loads(r["data"]))
return { return {
"lastid": maxid, "lastid": maxid,
# "lastid": row[-1]['id'] if row else str(id), # "lastid": row[-1]['id'] if row else str(id),
"events": [row[i]['data'] for i in range(len(row))] "events": [json.loads(r["data"]) for r in row]
} }
def store_event(self, client, event): def store_event(self, client, event):
try: try:
# logging.debug("INSERT INTO events (received,service_id,data) VALUES (NOW(), '%s', '%s')" % (event['DetectTime'], client["service"]["service_id"], self.con.escape_string(str(event)))) self.crs.execute("INSERT INTO events (received,service_id,data) VALUES (NOW(), %s, %s)", (client["service"]["service_id"], json.dumps(event)))
self.crs.execute("INSERT INTO events (received,service_id,data) VALUES (NOW(), '%s', '%s')" % (client["service"]["service_id"], self.con.escape_string(str(event))))
lastid = self.crs.lastrowid lastid = self.crs.lastrowid
# logging.debug(str(lastid)) logging.debug("store_event: Last ID in events - %i" % lastid)
for cat in event['Category']:
# logging.debug({'cat': cat}) for cat in event.get('Category', ["Other"]):
cat_id = self.map_id('Category', cat) if self.map_id('Category', cat) else self.map_id('Category', 'Other.Other') cat_id = self.map_id('Category', cat) or self.map_id('Category', 'Other')
# logging.debug({'cat_id': cat_id}) logging.debug("store_event: Category \"%s\" translated to %i" % (cat, cat_id))
# logging.debug("INSERT INTO event_category_mapping (event_id,category_id) VALUES ('%s', '%s')" % (str(lastid), str(cat_id))) self.crs.execute("INSERT INTO event_category_mapping (event_id,category_id) VALUES (%s, %s)", (lastid, cat_id))
self.crs.execute("INSERT INTO event_category_mapping (event_id,category_id) VALUES ('%s', '%s')" % (str(lastid), str(cat_id)))
for tag in event['Node'][0]['Tags']: try:
tag_id = self.map_id('Tag', tag) if self.map_id('Tag', tag) else self.map_id('Tag', 'Other') tags = event['Node'][0]['Tags']
# logging.debug({'tag_id': tag_id}) except (KeyError, IndexError):
# logging.debug("INSERT INTO event_tag_mapping (event_id,tag_id) VALUES ('%s', '%s')" % (str(lastid), tag_id)) tags = []
self.crs.execute("INSERT INTO event_tag_mapping (event_id,tag_id) VALUES ('%s', '%s')" % (str(lastid), str(tag_id)))
for tag in tags:
tag_id = self.map_id('Tag', tag) or self.map_id('Tag', 'Other')
logging.debug("store_event: Tag \"%s\" translated to %i" % (tag, tag_id))
self.crs.execute("INSERT INTO event_tag_mapping (event_id,tag_id) VALUES (%s, %s)", (lastid, tag_id))
self.con.commit() self.con.commit()
return [] return []
except Exception as e: except Exception as e:
self.con.rollback() self.con.rollback()
return [{"event": event, "error": type(e).__name__ + ": " + str(e)}] return [type(e).__name__ + ": " + str(e)]
def insertLastReceivedId(self, client, id): def insertLastReceivedId(self, client, id):
...@@ -472,7 +477,7 @@ class MySQL(Object): ...@@ -472,7 +477,7 @@ class MySQL(Object):
return row['id'] if row is not None else 0 return row['id'] if row is not None else 0
def map_id (self, section, key, strict = False): def map_id (self, section, key):
# Should by placed in config file # Should by placed in config file
data = {} data = {}
data['Tag'] = { data['Tag'] = {
...@@ -549,9 +554,8 @@ class MySQL(Object): ...@@ -549,9 +554,8 @@ class MySQL(Object):
try: try:
return data[section][key] return data[section][key]
except: except KeyError:
#Return 0 for strict mode (searching), otherwise map everything else to 'Other' return 0
return 0 if strict else data[section]['Other']
def generateDynamicQuery(self, section, query_string, variables, parent_cats = []): def generateDynamicQuery(self, section, query_string, variables, parent_cats = []):
...@@ -559,7 +563,7 @@ class MySQL(Object): ...@@ -559,7 +563,7 @@ class MySQL(Object):
# parent_cats = [] # parent_cats = []
for v in variables: for v in variables:
mapped_id = self.map_id(section, v, True) mapped_id = self.map_id(section, v)
if mapped_id % 100 != 0: if mapped_id % 100 != 0:
variables_id.append(mapped_id) variables_id.append(mapped_id)
else: else:
...@@ -639,8 +643,8 @@ class Server(Object): ...@@ -639,8 +643,8 @@ class Server(Object):
exc=sys.exc_info(), detail={"args": injson}) exc=sys.exc_info(), detail={"args": injson})
args = parse_qs(environ.get('QUERY_STRING', "")) args = parse_qs(environ.get('QUERY_STRING', ""))
for k, v in args.iteritems(): #for k, v in args.iteritems():
args[k] = v[0] # args[k] = v[0]
logging.debug("%s called with %s" % (path, str(args))) logging.debug("%s called with %s" % (path, str(args)))
if events: if events:
args["events"] = events args["events"] = events
...@@ -742,24 +746,25 @@ class WardenHandler(Object): ...@@ -742,24 +746,25 @@ class WardenHandler(Object):
group=None, nogroup=None): group=None, nogroup=None):
try: try:
id = int(id) id = int(id[0])
except (ValueError, TypeError): except (ValueError, TypeError, IndexError):
id=0 id = 0
if id == 0: if not id:
try: try:
id = self.db.getLastReceivedId(_client) id = self.db.getLastReceivedId(_client)
# logging.debug("Last received ID for %s is %s" % (_client['hostname'], str(id))) # logging.debug("Last received ID for %s is %s" % (_client['hostname'], str(id)))
except Exception, e: except Exception, e:
logging.error(e) logging.error("getEvents: cannot getLastReceivedId - " + type(e).__name__ + ": " + e)
id = 0 id = 0
if id == 0: if not id:
try: try:
id = self.db.getLastEventId() id = self.db.getLastEventId()
except Exception as e: except Exception as e:
raise Error("Last event id receiving error", 500, detail={"client": _client}) raise Error("Last event id receiving error", 500, detail={"client": _client})
# First access, remember the guy
self.db.insertLastReceivedId(_client, id) self.db.insertLastReceivedId(_client, id)
return { return {
...@@ -768,17 +773,17 @@ class WardenHandler(Object): ...@@ -768,17 +773,17 @@ class WardenHandler(Object):
} }
try: try:
count = int(count) count = int(count[0])
except (ValueError, TypeError): except (ValueError, TypeError, IndexError):
count = 1 count = 1
if self.get_events_limit: if self.get_events_limit:
count = min(count, self.get_events_limit) count = min(count, self.get_events_limit)
logging.debug("getEvents - count: %s" % count) logging.debug("getEvents - count: %s" % count)
res = self.db.fetch_events(_client, id or 0, count, cat, nocat, tag, notag, group, nogroup) res = self.db.fetch_events(_client, id, count, cat, nocat, tag, notag, group, nogroup)
logging.info("getEvents(%d, %d, %s, %s, %s, %s, %s, %s): sending %d events" % ( logging.info("getEvents(%d, %d, %s, %s, %s, %s, %s, %s): sending %d events" % (
id or 0, count, cat, nocat, tag, notag, group, nogroup, len(res["events"]))) id, count, cat, nocat, tag, notag, group, nogroup, len(res["events"])))
self.db.insertLastReceivedId(_client, res['lastid']) self.db.insertLastReceivedId(_client, res['lastid'])
logging.debug("lastid inserting: %s" % {'lastid': res['lastid'], 'client' : _client}) logging.debug("lastid inserting: %s" % {'lastid': res['lastid'], 'client' : _client})
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment