Skip to content
Snippets Groups Projects
Commit 9436580f authored by Pavel Kácha's avatar Pavel Kácha
Browse files

Parent categories are now also saved in event_category_mapping, which...

Parent categories are now also saved in event_category_mapping, which simplifies fetch_events. Fixed AND->OR in group checking. Group now also matches only at "." boundary.
parent 63ad23ac
No related branches found
No related tags found
No related merge requests found
...@@ -75,8 +75,9 @@ def gen_random_idea(client_name="cz.example.warden.test"): ...@@ -75,8 +75,9 @@ def gen_random_idea(client_name="cz.example.warden.test"):
"WinEndTime": get_precise_timestamp(), "WinEndTime": get_precise_timestamp(),
"EventTime": get_precise_timestamp(), "EventTime": get_precise_timestamp(),
"CeaseTime": get_precise_timestamp(), "CeaseTime": get_precise_timestamp(),
"Category": ["Abusive.Spam","Fraud.Copyright","Test"], #"Category": ["Abusive.Spam","Abusive.Harassment","Malware","Fraud.Copyright","Test","Fraud.Phishing","Fraud.Scam"],
# "Category": ["Abusive.Spam","Fraud.Copyright"], # "Category": ["Abusive.Spam","Fraud.Copyright"],
"Category": [choice(["Abusive.Spam","Abusive.Harassment","Malware","Fraud.Copyright","Test","Fraud.Phishing","Fraud.Scam"]) for dummy in range(randint(1, 3))],
"Ref": ["cve:CVE-%s-%s" % (randstr(string.digits, 4), randstr()), "http://www.example.com/%s" % randstr()], "Ref": ["cve:CVE-%s-%s" % (randstr(string.digits, 4), randstr()), "http://www.example.com/%s" % randstr()],
"Confidence": random(), "Confidence": random(),
"Note": "Random event", "Note": "Random event",
...@@ -117,7 +118,7 @@ def gen_random_idea(client_name="cz.example.warden.test"): ...@@ -117,7 +118,7 @@ def gen_random_idea(client_name="cz.example.warden.test"):
"Node": [ "Node": [
{ {
"Name": client_name, "Name": client_name,
"Tags": ["Protocol", "Honeypot"], "Tags": [choice(["Data", "Protocol", "Honeypot", "Heuristic", "Log"]) for dummy in range(randint(1, 3))],
"SW": ["Kippo"], "SW": ["Kippo"],
"AggrWin": "00:05:00" "AggrWin": "00:05:00"
} }
...@@ -141,12 +142,28 @@ def main(): ...@@ -141,12 +142,28 @@ def main():
# idstore="MyClient.id", # idstore="MyClient.id",
# name="cz.example.warden.test") # name="cz.example.warden.test")
print "=== Debug ==="
info = wclient.getDebug()
if not isinstance(info, Error):
pprint(info)
print "=== Server info ==="
info = wclient.getInfo()
if not isinstance(info, Error):
pprint(info)
print "=== Sending 10 event(s) ==="
start = time()
ret = wclient.sendEvents([gen_random_idea(client_name=wclient.name) for i in range(10)])
print ret
print "Time: %f" % (time()-start)
print "=== Getting 10 events ===" print "=== Getting 10 events ==="
start = time() start = time()
# cat = ['Availability', 'Abusive.Spam','Attempt.Login'] # cat = ['Availability', 'Abusive.Spam','Attempt.Login']
# cat = ['Attempt', 'Information','Fraud.Scam','Malware.Virus'] # cat = ['Attempt', 'Information','Fraud.Scam','Malware.Virus']
cat = ['Attempt'] cat = ['Fraud', 'Abusive.Spam']
nocat = ['Availability', 'Information', 'Fraud.Scam'] nocat = ['Availability', 'Information', 'Fraud.Scam']
tag = ['Log', 'Data'] tag = ['Log', 'Data']
...@@ -155,31 +172,14 @@ def main(): ...@@ -155,31 +172,14 @@ def main():
group = ['cz.tul.ward.kippo','cz.vsb.buldog.kippo'] group = ['cz.tul.ward.kippo','cz.vsb.buldog.kippo']
nogroup = ['cz.zcu.civ.afrodita','cz.vutbr.net.bee.hpscan'] nogroup = ['cz.zcu.civ.afrodita','cz.vutbr.net.bee.hpscan']
ret = wclient.getEvents(count=10, cat=None, nocat=None, tag=None, notag=None, group=None, nogroup=nogroup) ret = wclient.getEvents(count=10, cat=cat, nocat=None, tag=tag, notag=None, group=None, nogroup=nogroup)
ret = wclient.getEvents(count=10)
print "Time: %f" % (time()-start) print "Time: %f" % (time()-start)
print "Got %i events" % len(ret) print "Got %i events" % len(ret)
for e in ret: for e in ret:
print e print e["Category"], e["Node"][0]["Tags"], e["Node"][0]["Name"]
if isinstance(ret, Error): if isinstance(ret, Error):
print ret print ret
print "=== Sending 1 event(s) ==="
start = time()
ret = wclient.sendEvents([gen_random_idea(client_name=wclient.name) for i in range(1)])
print ret
print "Time: %f" % (time()-start)
print "=== Server info ==="
info = wclient.getInfo()
if not isinstance(info, Error):
pprint(info)
print "=== Debug ==="
info = wclient.getDebug()
if not isinstance(info, Error):
pprint(info)
if __name__ == "__main__": if __name__ == "__main__":
main() main()
...@@ -39,40 +39,50 @@ CREATE TABLE IF NOT EXISTS `categories` ( ...@@ -39,40 +39,50 @@ CREATE TABLE IF NOT EXISTS `categories` (
-- --
INSERT INTO `categories` (`id`, `category`, `subcategory`, `cat_subcat`) VALUES INSERT INTO `categories` (`id`, `category`, `subcategory`, `cat_subcat`) VALUES
(100, 'Abusive', NULL, 'Abusive'),
(101, 'Abusive', 'Spam', 'Abusive.Spam'), (101, 'Abusive', 'Spam', 'Abusive.Spam'),
(102, 'Abusive', 'Harassment', 'Abusive.Harassment'), (102, 'Abusive', 'Harassment', 'Abusive.Harassment'),
(103, 'Abusive', 'Child', 'Abusive.Child'), (103, 'Abusive', 'Child', 'Abusive.Child'),
(104, 'Abusive', 'Sexual', 'Abusive.Sexual'), (104, 'Abusive', 'Sexual', 'Abusive.Sexual'),
(105, 'Abusive', 'Violence', 'Abusive.Violence'), (105, 'Abusive', 'Violence', 'Abusive.Violence'),
(200, 'Malware', NULL, 'Malware'),
(201, 'Malware', 'Virus', 'Malware.Virus'), (201, 'Malware', 'Virus', 'Malware.Virus'),
(202, 'Malware', 'Worm', 'Malware.Worm'), (202, 'Malware', 'Worm', 'Malware.Worm'),
(203, 'Malware', 'Trojan', 'Malware.Trojan'), (203, 'Malware', 'Trojan', 'Malware.Trojan'),
(204, 'Malware', 'Spyware', 'Malware.Spyware'), (204, 'Malware', 'Spyware', 'Malware.Spyware'),
(205, 'Malware', 'Dialer', 'Malware.Dialer'), (205, 'Malware', 'Dialer', 'Malware.Dialer'),
(206, 'Malware', 'Rootkit', 'Malware.Rootkit'), (206, 'Malware', 'Rootkit', 'Malware.Rootkit'),
(300, 'Recon', NULL, 'Recon'),
(301, 'Recon', 'Scanning', 'Recon.Scanning'), (301, 'Recon', 'Scanning', 'Recon.Scanning'),
(302, 'Recon', 'Sniffing', 'Recon.Sniffing'), (302, 'Recon', 'Sniffing', 'Recon.Sniffing'),
(303, 'Recon', 'SocialEngineering', 'Recon.SocialEngineering'), (303, 'Recon', 'SocialEngineering', 'Recon.SocialEngineering'),
(304, 'Recon', 'Searching', 'Recon.Searching'), (304, 'Recon', 'Searching', 'Recon.Searching'),
(400, 'Attempt', NULL, 'Attempt'),
(401, 'Attempt', 'Exploit', 'Attempt.Exploit'), (401, 'Attempt', 'Exploit', 'Attempt.Exploit'),
(402, 'Attempt', 'Login', 'Attempt.Login'), (402, 'Attempt', 'Login', 'Attempt.Login'),
(403, 'Attempt', 'NewSignature', 'Attempt.NewSignature'), (403, 'Attempt', 'NewSignature', 'Attempt.NewSignature'),
(500, 'Intrusion', NULL, 'Intrusion'),
(501, 'Intrusion', 'AdminCompromise', 'Intrusion.AdminCompromise'), (501, 'Intrusion', 'AdminCompromise', 'Intrusion.AdminCompromise'),
(502, 'Intrusion', 'UserCompromise', 'Intrusion.UserCompromise'), (502, 'Intrusion', 'UserCompromise', 'Intrusion.UserCompromise'),
(503, 'Intrusion', 'AppCompromise', 'Intrusion.AppCompromise'), (503, 'Intrusion', 'AppCompromise', 'Intrusion.AppCompromise'),
(504, 'Intrusion', 'Botnet', 'Intrusion.Botnet'), (504, 'Intrusion', 'Botnet', 'Intrusion.Botnet'),
(600, 'Availability', NULL, 'Availability'),
(601, 'Availability', 'DoS', 'Availability.DoS'), (601, 'Availability', 'DoS', 'Availability.DoS'),
(602, 'Availability', 'DDoS', 'Availability.DDoS'), (602, 'Availability', 'DDoS', 'Availability.DDoS'),
(603, 'Availability', 'Sabotage', 'Availability.Sabotage'), (603, 'Availability', 'Sabotage', 'Availability.Sabotage'),
(604, 'Availability', 'Outage', 'Availability.Outage'), (604, 'Availability', 'Outage', 'Availability.Outage'),
(700, 'Information', NULL, 'Information'),
(701, 'Information', 'UnauthorizedAccess', 'Information.UnauthorizedAccess'), (701, 'Information', 'UnauthorizedAccess', 'Information.UnauthorizedAccess'),
(702, 'Information', 'UnauthorizedModification', 'Information.UnauthorizedModification'), (702, 'Information', 'UnauthorizedModification', 'Information.UnauthorizedModification'),
(800, 'Fraud', NULL, 'Fraud'),
(801, 'Fraud', 'UnauthorizedUsage', 'Fraud.UnauthorizedUsage'), (801, 'Fraud', 'UnauthorizedUsage', 'Fraud.UnauthorizedUsage'),
(802, 'Fraud', 'Copyright', 'Fraud.Copyright'), (802, 'Fraud', 'Copyright', 'Fraud.Copyright'),
(803, 'Fraud', 'Masquerade', 'Fraud.Masquerade'), (803, 'Fraud', 'Masquerade', 'Fraud.Masquerade'),
(804, 'Fraud', 'Phishing', 'Fraud.Phishing'), (804, 'Fraud', 'Phishing', 'Fraud.Phishing'),
(805, 'Fraud', 'Scam', 'Fraud.Scam'), (805, 'Fraud', 'Scam', 'Fraud.Scam'),
(900, 'Vulnerable', NULL, 'Vulnerable'),
(901, 'Vulnerable', 'Open', 'Vulnerable.Open'), (901, 'Vulnerable', 'Open', 'Vulnerable.Open'),
(1000, 'Anomaly', NULL, 'Anomaly'),
(1001, 'Anomaly', 'Traffic', 'Anomaly.Traffic'), (1001, 'Anomaly', 'Traffic', 'Anomaly.Traffic'),
(1002, 'Anomaly', 'Connection', 'Anomaly.Connection'), (1002, 'Anomaly', 'Connection', 'Anomaly.Connection'),
(1003, 'Anomaly', 'Protocol', 'Anomaly.Protocol'), (1003, 'Anomaly', 'Protocol', 'Anomaly.Protocol'),
......
...@@ -294,7 +294,6 @@ class X509Authenticator(NoAuthenticator): ...@@ -294,7 +294,6 @@ class X509Authenticator(NoAuthenticator):
identity = args.get("client", [None])[0] identity = args.get("client", [None])[0]
secret = args.get("secret", [None])[0] secret = args.get("secret", [None])[0]
args["secret"] = ["..."] # Prevent to spill it over logs
client = self.db.get_client_by_name(cert_names, identity, secret) client = self.db.get_client_by_name(cert_names, identity, secret)
...@@ -412,6 +411,14 @@ class MySQL(ObjectReq): ...@@ -412,6 +411,14 @@ class MySQL(ObjectReq):
type(self).__name__, type(self.req).__name__, self.host, self.user, self.dbname, self.port, self.catmap_filename, self.tagmap_filename) type(self).__name__, type(self.req).__name__, self.host, self.user, self.dbname, self.port, self.catmap_filename, self.tagmap_filename)
def _get_comma_perc(self, l):
return ','.join(['%s'] * len(l))
def _get_not(self, b):
return "" if b else "NOT"
def get_client_by_name(self, cert_names, identity=None, secret=None): def get_client_by_name(self, cert_names, identity=None, secret=None):
query = ["SELECT id, registered, requestor, hostname, service, note, identity, secret, `read`, debug, `write`, test FROM clients WHERE valid = 1"] query = ["SELECT id, registered, requestor, hostname, service, note, identity, secret, `read`, debug, `write`, test FROM clients WHERE valid = 1"]
params = [] params = []
...@@ -421,7 +428,7 @@ class MySQL(ObjectReq): ...@@ -421,7 +428,7 @@ class MySQL(ObjectReq):
if secret: if secret:
query.append(" AND secret = %s") query.append(" AND secret = %s")
params.append(secret) params.append(secret)
query.append(" AND hostname IN (%s)" % ','.join(['%s'] * len(cert_names))) query.append(" AND hostname IN (%s)" % self._get_comma_perc(cert_names))
params.extend(cert_names) params.extend(cert_names)
self.crs.execute("".join(query), params) self.crs.execute("".join(query), params)
rows = self.crs.fetchall() rows = self.crs.fetchall()
...@@ -446,30 +453,16 @@ class MySQL(ObjectReq): ...@@ -446,30 +453,16 @@ class MySQL(ObjectReq):
} }
def generateDynamicQuery(self, section, query_string, variables, parent_cats = []): def getMaps(self, section, variables):
variables_id = [] maps = []
for v in variables: for v in variables:
try: try:
mapped_id = section[v] mapped = section[v]
except KeyError: except KeyError:
raise self.req.error("Wrong tag or category used in query.", 422, raise self.req.error("Wrong tag or category used in query.", 422,
sys.exc_info(), detail={"key": v}) sys.exc_info(), detail={"key": v})
maps.append(mapped)
if mapped_id % 100: return set(maps) # unique
variables_id.append(mapped_id)
else:
parent_cats.append(mapped_id)
temp_string = ""
if len(variables_id) > 0:
format_strings = ','.join(['%s'] * len(variables_id))
logging.debug("query_string: %s" % query_string)
logging.debug("format_strings: %s" % format_strings)
temp_string = query_string % format_strings
return temp_string, variables_id
def fetch_events(self, client, id, count, def fetch_events(self, client, id, count,
...@@ -489,42 +482,41 @@ class MySQL(ObjectReq): ...@@ -489,42 +482,41 @@ class MySQL(ObjectReq):
raise self.req.error("Unrealizable conditions. Choose group or nogroup option.", 422, raise self.req.error("Unrealizable conditions. Choose group or nogroup option.", 422,
detail={'tag': cat, 'notag' : nocat}) detail={'tag': cat, 'notag' : nocat})
sqlwhere = [] query = ["SELECT e.id, e.data FROM clients c RIGHT JOIN events e ON c.id = e.client_id WHERE e.id > %s"]
sqlparams = [] params = [id or 0]
sqlwhere.append("SELECT e.id, e.data FROM clients c RIGHT JOIN events e ON c.id = e.client_id WHERE e.id > %s")
sqlparams.append(id or 0)
if cat or nocat: if cat or nocat:
parent_cats = [] cats = self.getMaps(self.catmap, (cat or nocat))
sqltemp, sqlpar = self.generateDynamicQuery(self.catmap, "category_id IN (%s)", (cat or nocat), parent_cats) query.append(
for pcat in parent_cats: " AND e.id %s IN (SELECT event_id FROM event_category_mapping WHERE category_id IN (%s))" % (
sqltemp += " %s (category_id > %s AND category_id < %s) " % (("OR" if sqltemp else ""), pcat, pcat + 100) self._get_not(cat), self._get_comma_perc(cats)))
params.extend(cats)
sqlwhere.append(" AND e.id %s IN (SELECT event_id FROM event_category_mapping WHERE %s)" % (("NOT" if nocat else ""), sqltemp))
sqlparams.extend(sqlpar)
if tag or notag: if tag or notag:
sqltemp, sqlpar = self.generateDynamicQuery(self.tagmap, "tag_id IN (%s)", (tag or notag)) tags = self.getMaps(self.tagmap, (tag or notag))
query.append(
sqlwhere.append(" AND e.id %s IN (SELECT event_id FROM event_tag_mapping WHERE %s)" % (("NOT" if notag else ""), sqltemp)) " AND e.id %s IN (SELECT event_id FROM event_tag_mapping WHERE tag_id IN (%s))" % (
sqlparams.extend(sqlpar) self._get_not(tag), self._get_comma_perc(tags)))
params.extend(tags)
if group or nogroup: if group or nogroup:
not_op = "" if group else "NOT" subquery = []
for identity in (group or nogroup): for identity in (group or nogroup):
sqlwhere.append(" AND c.identity %s LIKE %%s" % not_op) subquery.append("c.identity = %s") # exact client
sqlparams.append(identity + "%") params.append(identity)
subquery.append("c.identity LIKE %s") # whole subtree
params.append(identity + ".%")
query.append(" AND %s (%s)" % (self._get_not(group), " OR ".join(subquery)))
sqlwhere.append(" AND e.valid = 1 LIMIT %s") query.append(" AND e.valid = 1 LIMIT %s")
sqlparams.append(count) params.append(count)
sqlwhere_string = "".join(sqlwhere) query_string = "".join(query)
logging.debug("fetch_events: query - %s" % sqlwhere_string) logging.debug("fetch_events: query - %s" % query_string)
logging.debug("fetch_events: params - %s", str(sqlparams)) logging.debug("fetch_events: params - %s", str(params))
self.crs.execute(sqlwhere_string, sqlparams) self.crs.execute(query_string, params)
row = self.crs.fetchall() row = self.crs.fetchall()
if row: if row:
...@@ -544,11 +536,11 @@ class MySQL(ObjectReq): ...@@ -544,11 +536,11 @@ class MySQL(ObjectReq):
try: try:
self.crs.execute("INSERT INTO events (received,client_id,data) VALUES (NOW(), %s, %s)", (client.id, json.dumps(event))) self.crs.execute("INSERT INTO events (received,client_id,data) VALUES (NOW(), %s, %s)", (client.id, json.dumps(event)))
lastid = self.crs.lastrowid lastid = self.crs.lastrowid
logging.debug("store_event: Last ID in events - %i" % lastid)
for cat in event.get('Category', ["Other"]): catlist = event.get('Category', ["Other"])
cats = set(catlist) | {cat.split(".", 1)[0] for cat in catlist}
for cat in cats:
cat_id = self.catmap.get(cat, self.catmap_other) cat_id = self.catmap.get(cat, self.catmap_other)
logging.debug("store_event: Category \"%s\" translated to %i" % (cat, cat_id))
self.crs.execute("INSERT INTO event_category_mapping (event_id,category_id) VALUES (%s, %s)", (lastid, cat_id)) self.crs.execute("INSERT INTO event_category_mapping (event_id,category_id) VALUES (%s, %s)", (lastid, cat_id))
try: try:
...@@ -558,7 +550,6 @@ class MySQL(ObjectReq): ...@@ -558,7 +550,6 @@ class MySQL(ObjectReq):
for tag in tags: for tag in tags:
tag_id = self.tagmap.get(tag, self.tagmap_other) tag_id = self.tagmap.get(tag, self.tagmap_other)
logging.debug("store_event: Tag \"%s\" translated to %i" % (tag, tag_id))
self.crs.execute("INSERT INTO event_tag_mapping (event_id,tag_id) VALUES (%s, %s)", (lastid, tag_id)) self.crs.execute("INSERT INTO event_tag_mapping (event_id,tag_id) VALUES (%s, %s)", (lastid, tag_id))
self.con.commit() self.con.commit()
...@@ -671,8 +662,6 @@ class Server(ObjectReq): ...@@ -671,8 +662,6 @@ class Server(ObjectReq):
if not auth: if not auth:
raise self.req.error("I'm watching. Not authorized.", 403, detail={"client": client.identity}) raise self.req.error("I'm watching. Not authorized.", 403, detail={"client": client.identity})
logging.debug("arguments: %s" % str(args))
# These args are not for handler # These args are not for handler
args.pop("client", None) args.pop("client", None)
args.pop("secret", None) args.pop("secret", None)
...@@ -812,11 +801,11 @@ class WardenHandler(ObjectReq): ...@@ -812,11 +801,11 @@ class WardenHandler(ObjectReq):
count = min(count, self.get_events_limit) count = min(count, self.get_events_limit)
res = self.db.fetch_events(self.req.client, id, count, cat, nocat, tag, notag, group, nogroup) res = self.db.fetch_events(self.req.client, id, count, cat, nocat, tag, notag, group, nogroup)
logging.info("fetch_events(%d, %d, %s, %s, %s, %s, %s, %s) returned %d events" % (
id, count, cat, nocat, tag, notag, group, nogroup, len(res["events"])))
self.db.insertLastReceivedId(self.req.client, res['lastid']) self.db.insertLastReceivedId(self.req.client, res['lastid'])
logging.info("sending %d events, lastid is %i" % (len(res["events"]), res["lastid"]))
return res return res
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment