diff --git a/warden3/warden_client/warden_client.py b/warden3/warden_client/warden_client.py
index 302215d70e3fdf50ba147c511abe4fe0e005aa29..39e52ce11b32571ab3d7ecc76faba9e7c1d03950 100644
--- a/warden3/warden_client/warden_client.py
+++ b/warden3/warden_client/warden_client.py
@@ -253,7 +253,7 @@ class Client(object):
for k in kwargs.keys():
if kwargs[k] is None:
del kwargs[k]
- argurl = "?" + urlencode(kwargs)
+ argurl = "?" + urlencode(kwargs, doseq=True)
else:
argurl = ""
@@ -362,10 +362,7 @@ class Client(object):
def sendEvents(self, events=[]):
res = self.sendRequest(
"sendEvents", payload=events)
- if not res:
- return res # Should be Error instance
-
- return res.get("saved", 0)
+ return res
def getEvents(self, id=None, idstore=None, count=1,
diff --git a/warden3/warden_client/warden_client_test.py b/warden3/warden_client/warden_client_test.py
index a9cb9990429dbe3bace04d6024a5c27fd982439d..96985d466c5f565ca74103e79e00a440e134c802 100755
--- a/warden3/warden_client/warden_client_test.py
+++ b/warden3/warden_client/warden_client_test.py
@@ -31,7 +31,7 @@ def gen_min_idea():
"Category": ["Test"],
}
-def gen_random_idea():
+def gen_random_idea(client_name="cz.example.warden.test"):
def geniprange(gen):
@@ -115,8 +115,7 @@ def gen_random_idea():
],
"Node": [
{
- "Name": "com.example.test-node",
- # "Name": "Test",
+ "Name": client_name,
"Tags": ["Protocol", "Honeypot"],
"SW": ["Kippo"],
"AggrWin": "00:05:00"
@@ -139,32 +138,33 @@ def main():
# errlog={"level": "debug"},
# filelog={"level": "debug"},
# idstore="MyClient.id",
- # name="MyClient")
+ # name="cz.example.warden.test")
print "=== Getting 10 events ==="
start = time()
- cat = json.dumps(['Availability', 'Abusive.Spam','Attempt.Login'])
- nocat = json.dumps(['Fraud.Scam','Malware.Virus'])
+ cat = ['Availability', 'Abusive.Spam','Attempt.Login']
+ nocat = ['Fraud.Scam','Malware.Virus']
- tag = json.dumps(['Log', 'Data'])
- notag = json.dumps(['Flow', 'Datagram'])
+ tag = ['Log', 'Data']
+ notag = ['Flow', 'Datagram']
- group = json.dumps(['cz.tul.ward.kippo','cz.vsb.buldog.kippo'])
- nogroup = json.dumps(['cz.zcu.civ.afrodita.%','cz.vutbr.net.bee.hpscan'])
+ group = ['cz.tul.ward.kippo','cz.vsb.buldog.kippo']
+ nogroup = ['cz.zcu.civ.afrodita','cz.vutbr.net.bee.hpscan']
ret = wclient.getEvents(count=10, cat=cat, nocat=None, tag=tag, notag=None, group=None, nogroup=nogroup)
+ #ret = wclient.getEvents(count=10)
print "Time: %f" % (time()-start)
+ print "Got %i events" % len(ret)
for e in ret:
print e
- if ret:
- print len(ret)
+ if isinstance(ret, Error):
+ print ret
print "=== Sending 1 event(s) ==="
start = time()
- ret = wclient.sendEvents([gen_random_idea() for i in range(1)])
- if ret:
- print ret
+ ret = wclient.sendEvents([gen_random_idea(client_name=wclient.name) for i in range(1)])
+ print ret
print "Time: %f" % (time()-start)
print "=== Server info ==="
diff --git a/warden3/warden_server/warden_server.py b/warden3/warden_server/warden_server.py
index 774c70079b662d2920f627d18db8e2d69d3cbd64..f9b0a0431afa3ae779404d6ca39fda59d56ef371 100755
--- a/warden3/warden_server/warden_server.py
+++ b/warden3/warden_server/warden_server.py
@@ -291,7 +291,6 @@ class MySQL(Object):
format_strings = ','.join(['%s'] * len(name))
self.crs.execute("SELECT cl.`id`, cl.`hostname`, s.`service`, s.`service_id`, s.`identity`, cl.`read`, s.`write`, s.`test`, cl.`debug` FROM `clients` cl LEFT JOIN `services` s ON cl.`id` = s.`client_id` WHERE cl.`valid` = 1 AND s.`valid` = 1 AND `hostname` IN (%s)" % format_strings, tuple(name))
rows = self.crs.fetchall()
- logging.debug("Client/service info: " + str(rows))
if not rows:
return None
@@ -308,7 +307,7 @@ class MySQL(Object):
client["services"] = services
- logging.debug("Client/service formatted info: " + str(client))
+ logging.debug("Client/services: " + str(client))
return client
@@ -350,6 +349,8 @@ class MySQL(Object):
sqlwhere = []
sqltemp = {}
+ logging.debug("fetch_events: id=%i, count=%i, cat=%s, nocat=%s, tag=%s, notag=%s, group=%s, nogroup=%s" % (id, count, str(cat), str(nocat), str(tag), str(notag), str(group), str(nogroup)))
+
if cat is not None and nocat is not None:
raise Error("Unrealizable conditions. Choose cat or nocat option.", 500, method='getEvents',
exc=sys.exc_info(), detail={'cat': cat, 'nocat' : nocat})
@@ -357,12 +358,12 @@ class MySQL(Object):
if cat is not None or nocat is not None:
if cat is not None:
parent_cats = []
- sqltemp['cat'] = self.generateDynamicQuery("Category", "category_id IN (%s)", json.loads(cat), parent_cats)
+ sqltemp['cat'] = self.generateDynamicQuery("Category", "category_id IN (%s)", cat, parent_cats)
for pcats in parent_cats:
sqltemp['cat'] += " %s category_id DIV %s = 1 " % (("OR" if sqltemp['cat'] else ""), pcats)
if nocat is not None:
parent_cats = []
- sqltemp['cat'] = self.generateDynamicQuery("Category", "category_id NOT IN (%s)", json.loads(nocat), parent_cats)
+ sqltemp['cat'] = self.generateDynamicQuery("Category", "category_id NOT IN (%s)", nocat, parent_cats)
for pcats in parent_cats:
sqltemp['cat'] += " %s category_id DIV %s = 1 " % (("OR" if sqltemp['cat'] else ""), pcats)
@@ -374,9 +375,9 @@ class MySQL(Object):
if tag is not None or notag is not None:
if tag is not None:
- sqltemp['tag'] = self.generateDynamicQuery("Tag", "tag_id IN (%s)", json.loads(tag))
+ sqltemp['tag'] = self.generateDynamicQuery("Tag", "tag_id IN (%s)", tag)
if notag is not None:
- sqltemp['tag'] = self.generateDynamicQuery("Tag", "tag_id NOT IN (%s)", json.loads(notag))
+ sqltemp['tag'] = self.generateDynamicQuery("Tag", "tag_id NOT IN (%s)", notag)
sqlwhere.append("e.id IN (SELECT event_id FROM event_tag_mapping WHERE %s)" % sqltemp['tag'])
@@ -389,10 +390,10 @@ class MySQL(Object):
sqltemp['group'] = ""
if group is not None:
- for identity in json.loads(group):
+ for identity in group:
sqltemp['group'] += ("s.identity LIKE '%s' AND " % (identity))
if nogroup is not None:
- for identity in json.loads(nogroup):
+ for identity in nogroup:
sqltemp['group'] += ("s.identity NOT LIKE '%s' AND " % (identity))
# logging.debug(sqltemp['group'][:-4])
@@ -415,38 +416,42 @@ class MySQL(Object):
maxid = self.getLastEventId()
# logging.debug("MAX ID = %s", str(maxid))
+ #for r in row:
+ # logging.debug(json.loads(r["data"]))
return {
"lastid": maxid,
# "lastid": row[-1]['id'] if row else str(id),
- "events": [row[i]['data'] for i in range(len(row))]
+ "events": [json.loads(r["data"]) for r in row]
}
def store_event(self, client, event):
try:
- # logging.debug("INSERT INTO events (received,service_id,data) VALUES (NOW(), '%s', '%s')" % (event['DetectTime'], client["service"]["service_id"], self.con.escape_string(str(event))))
- self.crs.execute("INSERT INTO events (received,service_id,data) VALUES (NOW(), '%s', '%s')" % (client["service"]["service_id"], self.con.escape_string(str(event))))
+ self.crs.execute("INSERT INTO events (received,service_id,data) VALUES (NOW(), %s, %s)", (client["service"]["service_id"], json.dumps(event)))
lastid = self.crs.lastrowid
- # logging.debug(str(lastid))
- for cat in event['Category']:
- # logging.debug({'cat': cat})
- cat_id = self.map_id('Category', cat) if self.map_id('Category', cat) else self.map_id('Category', 'Other.Other')
- # logging.debug({'cat_id': cat_id})
- # logging.debug("INSERT INTO event_category_mapping (event_id,category_id) VALUES ('%s', '%s')" % (str(lastid), str(cat_id)))
- self.crs.execute("INSERT INTO event_category_mapping (event_id,category_id) VALUES ('%s', '%s')" % (str(lastid), str(cat_id)))
+ logging.debug("store_event: Last ID in events - %i" % lastid)
+
+ for cat in event.get('Category', ["Other"]):
+ cat_id = self.map_id('Category', cat) or self.map_id('Category', 'Other')
+ logging.debug("store_event: Category \"%s\" translated to %i" % (cat, cat_id))
+ self.crs.execute("INSERT INTO event_category_mapping (event_id,category_id) VALUES (%s, %s)", (lastid, cat_id))
- for tag in event['Node'][0]['Tags']:
- tag_id = self.map_id('Tag', tag) if self.map_id('Tag', tag) else self.map_id('Tag', 'Other')
- # logging.debug({'tag_id': tag_id})
- # logging.debug("INSERT INTO event_tag_mapping (event_id,tag_id) VALUES ('%s', '%s')" % (str(lastid), tag_id))
- self.crs.execute("INSERT INTO event_tag_mapping (event_id,tag_id) VALUES ('%s', '%s')" % (str(lastid), str(tag_id)))
+ try:
+ tags = event['Node'][0]['Tags']
+ except (KeyError, IndexError):
+ tags = []
+
+ for tag in tags:
+ tag_id = self.map_id('Tag', tag) or self.map_id('Tag', 'Other')
+ logging.debug("store_event: Tag \"%s\" translated to %i" % (tag, tag_id))
+ self.crs.execute("INSERT INTO event_tag_mapping (event_id,tag_id) VALUES (%s, %s)", (lastid, tag_id))
self.con.commit()
return []
except Exception as e:
self.con.rollback()
- return [{"event": event, "error": type(e).__name__ + ": " + str(e)}]
+ return [type(e).__name__ + ": " + str(e)]
def insertLastReceivedId(self, client, id):
@@ -472,7 +477,7 @@ class MySQL(Object):
return row['id'] if row is not None else 0
- def map_id (self, section, key, strict = False):
+ def map_id (self, section, key):
# Should by placed in config file
data = {}
data['Tag'] = {
@@ -549,9 +554,8 @@ class MySQL(Object):
try:
return data[section][key]
- except:
- #Return 0 for strict mode (searching), otherwise map everything else to 'Other'
- return 0 if strict else data[section]['Other']
+ except KeyError:
+ return 0
def generateDynamicQuery(self, section, query_string, variables, parent_cats = []):
@@ -559,7 +563,7 @@ class MySQL(Object):
# parent_cats = []
for v in variables:
- mapped_id = self.map_id(section, v, True)
+ mapped_id = self.map_id(section, v)
if mapped_id % 100 != 0:
variables_id.append(mapped_id)
else:
@@ -639,8 +643,8 @@ class Server(Object):
exc=sys.exc_info(), detail={"args": injson})
args = parse_qs(environ.get('QUERY_STRING', ""))
- for k, v in args.iteritems():
- args[k] = v[0]
+ #for k, v in args.iteritems():
+ # args[k] = v[0]
logging.debug("%s called with %s" % (path, str(args)))
if events:
args["events"] = events
@@ -742,24 +746,25 @@ class WardenHandler(Object):
group=None, nogroup=None):
try:
- id = int(id)
- except (ValueError, TypeError):
- id=0
+ id = int(id[0])
+ except (ValueError, TypeError, IndexError):
+ id = 0
- if id == 0:
+ if not id:
try:
id = self.db.getLastReceivedId(_client)
# logging.debug("Last received ID for %s is %s" % (_client['hostname'], str(id)))
except Exception, e:
- logging.error(e)
+ logging.error("getEvents: cannot getLastReceivedId - " + type(e).__name__ + ": " + e)
id = 0
- if id == 0:
+ if not id:
try:
id = self.db.getLastEventId()
except Exception as e:
raise Error("Last event id receiving error", 500, detail={"client": _client})
+ # First access, remember the guy
self.db.insertLastReceivedId(_client, id)
return {
@@ -768,17 +773,17 @@ class WardenHandler(Object):
}
try:
- count = int(count)
- except (ValueError, TypeError):
+ count = int(count[0])
+ except (ValueError, TypeError, IndexError):
count = 1
if self.get_events_limit:
count = min(count, self.get_events_limit)
logging.debug("getEvents - count: %s" % count)
- res = self.db.fetch_events(_client, id or 0, count, cat, nocat, tag, notag, group, nogroup)
+ res = self.db.fetch_events(_client, id, count, cat, nocat, tag, notag, group, nogroup)
logging.info("getEvents(%d, %d, %s, %s, %s, %s, %s, %s): sending %d events" % (
- id or 0, count, cat, nocat, tag, notag, group, nogroup, len(res["events"])))
+ id, count, cat, nocat, tag, notag, group, nogroup, len(res["events"])))
self.db.insertLastReceivedId(_client, res['lastid'])
logging.debug("lastid inserting: %s" % {'lastid': res['lastid'], 'client' : _client})