Parent categories are now also saved in event_category_mapping, which...

Parent categories are now also saved in event_category_mapping, which simplifies fetch_events. Fixed AND->OR in group checking. Group now also matches only at "." boundary.

warden3/warden_client/warden_client_test.py

@@ -75,8 +75,9 @@ def gen_random_idea(client_name="cz.example.warden.test"):
        "WinEndTime": get_precise_timestamp(),
        "EventTime": get_precise_timestamp(),
        "CeaseTime": get_precise_timestamp(),
-       "Category": ["Abusive.Spam","Fraud.Copyright","Test"],
+       #"Category": ["Abusive.Spam","Abusive.Harassment","Malware","Fraud.Copyright","Test","Fraud.Phishing","Fraud.Scam"],
        # "Category": ["Abusive.Spam","Fraud.Copyright"],
+       "Category": [choice(["Abusive.Spam","Abusive.Harassment","Malware","Fraud.Copyright","Test","Fraud.Phishing","Fraud.Scam"]) for dummy in range(randint(1, 3))],
        "Ref": ["cve:CVE-%s-%s" % (randstr(string.digits, 4), randstr()), "http://www.example.com/%s" % randstr()],
        "Confidence": random(),
        "Note": "Random event",
@@ -117,7 +118,7 @@ def gen_random_idea(client_name="cz.example.warden.test"):
        "Node": [
           {
              "Name": client_name,
-             "Tags": ["Protocol", "Honeypot"],
+             "Tags": [choice(["Data", "Protocol", "Honeypot", "Heuristic", "Log"]) for dummy in range(randint(1, 3))],
              "SW": ["Kippo"],
              "AggrWin": "00:05:00"
           }
@@ -141,12 +142,28 @@ def main():
     #     idstore="MyClient.id",
     #     name="cz.example.warden.test")
 
+    print "=== Debug ==="
+    info = wclient.getDebug()
+    if not isinstance(info, Error):
+        pprint(info)
+
+    print "=== Server info ==="
+    info = wclient.getInfo()
+    if not isinstance(info, Error):
+        pprint(info)
+
+    print "=== Sending 10 event(s) ==="
+    start = time()
+    ret = wclient.sendEvents([gen_random_idea(client_name=wclient.name) for i in range(10)])
+    print ret
+    print "Time: %f" % (time()-start)
+
     print "=== Getting 10 events ==="
     start = time()
 
     # cat = ['Availability', 'Abusive.Spam','Attempt.Login']
     # cat = ['Attempt', 'Information','Fraud.Scam','Malware.Virus']
-    cat = ['Attempt']
+    cat = ['Fraud', 'Abusive.Spam']
     nocat = ['Availability', 'Information', 'Fraud.Scam']
 
     tag = ['Log', 'Data']
@@ -155,31 +172,14 @@ def main():
     group = ['cz.tul.ward.kippo','cz.vsb.buldog.kippo']
     nogroup = ['cz.zcu.civ.afrodita','cz.vutbr.net.bee.hpscan']
 
-    ret = wclient.getEvents(count=10, cat=None, nocat=None, tag=None, notag=None, group=None, nogroup=nogroup)
-    ret = wclient.getEvents(count=10)
+    ret = wclient.getEvents(count=10, cat=cat, nocat=None, tag=tag, notag=None, group=None, nogroup=nogroup)
     print "Time: %f" % (time()-start)
     print "Got %i events" % len(ret)
     for e in ret:
-        print e
+        print e["Category"], e["Node"][0]["Tags"], e["Node"][0]["Name"]
     if isinstance(ret, Error):
         print ret
 
-    print "=== Sending 1 event(s) ==="
-    start = time()
-    ret = wclient.sendEvents([gen_random_idea(client_name=wclient.name) for i in range(1)])
-    print ret
-    print "Time: %f" % (time()-start)
-
-    print "=== Server info ==="
-    info = wclient.getInfo()
-    if not isinstance(info, Error):
-        pprint(info)
-
-    print "=== Debug ==="
-    info = wclient.getDebug()
-    if not isinstance(info, Error):
-        pprint(info)
-
 
 if __name__ == "__main__":
     main()

warden3/warden_server/warden3.0-alpha.sql

@@ -39,40 +39,50 @@ CREATE TABLE IF NOT EXISTS `categories` (
 --
 
 INSERT INTO `categories` (`id`, `category`, `subcategory`, `cat_subcat`) VALUES
+(100, 'Abusive', NULL, 'Abusive'),
 (101, 'Abusive', 'Spam', 'Abusive.Spam'),
 (102, 'Abusive', 'Harassment', 'Abusive.Harassment'),
 (103, 'Abusive', 'Child', 'Abusive.Child'),
 (104, 'Abusive', 'Sexual', 'Abusive.Sexual'),
 (105, 'Abusive', 'Violence', 'Abusive.Violence'),
+(200, 'Malware', NULL, 'Malware'),
 (201, 'Malware', 'Virus', 'Malware.Virus'),
 (202, 'Malware', 'Worm', 'Malware.Worm'),
 (203, 'Malware', 'Trojan', 'Malware.Trojan'),
 (204, 'Malware', 'Spyware', 'Malware.Spyware'),
 (205, 'Malware', 'Dialer', 'Malware.Dialer'),
 (206, 'Malware', 'Rootkit', 'Malware.Rootkit'),
+(300, 'Recon', NULL, 'Recon'),
 (301, 'Recon', 'Scanning', 'Recon.Scanning'),
 (302, 'Recon', 'Sniffing', 'Recon.Sniffing'),
 (303, 'Recon', 'SocialEngineering', 'Recon.SocialEngineering'),
 (304, 'Recon', 'Searching', 'Recon.Searching'),
+(400, 'Attempt', NULL, 'Attempt'),
 (401, 'Attempt', 'Exploit', 'Attempt.Exploit'),
 (402, 'Attempt', 'Login', 'Attempt.Login'),
 (403, 'Attempt', 'NewSignature', 'Attempt.NewSignature'),
+(500, 'Intrusion', NULL, 'Intrusion'),
 (501, 'Intrusion', 'AdminCompromise', 'Intrusion.AdminCompromise'),
 (502, 'Intrusion', 'UserCompromise', 'Intrusion.UserCompromise'),
 (503, 'Intrusion', 'AppCompromise', 'Intrusion.AppCompromise'),
 (504, 'Intrusion', 'Botnet', 'Intrusion.Botnet'),
+(600, 'Availability', NULL, 'Availability'),
 (601, 'Availability', 'DoS', 'Availability.DoS'),
 (602, 'Availability', 'DDoS', 'Availability.DDoS'),
 (603, 'Availability', 'Sabotage', 'Availability.Sabotage'),
 (604, 'Availability', 'Outage', 'Availability.Outage'),
+(700, 'Information', NULL, 'Information'),
 (701, 'Information', 'UnauthorizedAccess', 'Information.UnauthorizedAccess'),
 (702, 'Information', 'UnauthorizedModification', 'Information.UnauthorizedModification'),
+(800, 'Fraud', NULL, 'Fraud'),
 (801, 'Fraud', 'UnauthorizedUsage', 'Fraud.UnauthorizedUsage'),
 (802, 'Fraud', 'Copyright', 'Fraud.Copyright'),
 (803, 'Fraud', 'Masquerade', 'Fraud.Masquerade'),
 (804, 'Fraud', 'Phishing', 'Fraud.Phishing'),
 (805, 'Fraud', 'Scam', 'Fraud.Scam'),
+(900, 'Vulnerable', NULL, 'Vulnerable'),
 (901, 'Vulnerable', 'Open', 'Vulnerable.Open'),
+(1000, 'Anomaly', NULL, 'Anomaly'),
 (1001, 'Anomaly', 'Traffic', 'Anomaly.Traffic'),
 (1002, 'Anomaly', 'Connection', 'Anomaly.Connection'),
 (1003, 'Anomaly', 'Protocol', 'Anomaly.Protocol'),

warden3/warden_server/warden_server.py

@@ -294,7 +294,6 @@ class X509Authenticator(NoAuthenticator):
 
         identity = args.get("client", [None])[0]
         secret =  args.get("secret", [None])[0]
-        args["secret"] = ["..."]    # Prevent to spill it over logs
 
         client = self.db.get_client_by_name(cert_names, identity, secret)
 
@@ -412,6 +411,14 @@ class MySQL(ObjectReq):
             type(self).__name__, type(self.req).__name__, self.host, self.user, self.dbname, self.port, self.catmap_filename, self.tagmap_filename)
 
 
+    def _get_comma_perc(self, l):
+        return ','.join(['%s'] * len(l))
+
+
+    def _get_not(self, b):
+        return "" if b else "NOT"
+
+
     def get_client_by_name(self, cert_names, identity=None, secret=None):
         query = ["SELECT id, registered, requestor, hostname, service, note, identity, secret, `read`, debug, `write`, test FROM clients WHERE valid = 1"]
         params = []
@@ -421,7 +428,7 @@ class MySQL(ObjectReq):
         if secret:
             query.append(" AND secret = %s")
             params.append(secret)
-        query.append(" AND hostname IN (%s)" % ','.join(['%s'] * len(cert_names)))
+        query.append(" AND hostname IN (%s)" % self._get_comma_perc(cert_names))
         params.extend(cert_names)
         self.crs.execute("".join(query), params)
         rows = self.crs.fetchall()
@@ -446,30 +453,16 @@ class MySQL(ObjectReq):
         }
 
 
-    def generateDynamicQuery(self, section, query_string, variables, parent_cats = []):
-        variables_id = []
-    
+    def getMaps(self, section, variables):
+        maps = []
         for v in variables:
             try:
-                mapped_id = section[v]
+                mapped = section[v]
             except KeyError:
                 raise self.req.error("Wrong tag or category used in query.", 422,
                     sys.exc_info(), detail={"key": v})
-
-            if mapped_id % 100:
-                variables_id.append(mapped_id)
-            else:
-                parent_cats.append(mapped_id)
-    
-        temp_string = ""
-
-        if len(variables_id) > 0:
-            format_strings = ','.join(['%s'] * len(variables_id))
-            logging.debug("query_string: %s" % query_string)
-            logging.debug("format_strings: %s" % format_strings)
-            temp_string = query_string % format_strings
-        
-        return temp_string, variables_id
+            maps.append(mapped)
+        return set(maps)    # unique
 
 
     def fetch_events(self, client, id, count,
@@ -489,42 +482,41 @@ class MySQL(ObjectReq):
             raise self.req.error("Unrealizable conditions. Choose group or nogroup option.", 422,
                         detail={'tag': cat, 'notag' : nocat})
 
-        sqlwhere = []
-        sqlparams = []
-
-        sqlwhere.append("SELECT e.id, e.data FROM clients c RIGHT JOIN events e ON c.id = e.client_id WHERE e.id > %s")
-        sqlparams.append(id or 0)
+        query = ["SELECT e.id, e.data FROM clients c RIGHT JOIN events e ON c.id = e.client_id WHERE e.id > %s"]
+        params = [id or 0]
 
         if cat or nocat:
-            parent_cats = []
-            sqltemp, sqlpar = self.generateDynamicQuery(self.catmap, "category_id IN (%s)", (cat or nocat), parent_cats)
-            for pcat in parent_cats:
-                sqltemp += " %s (category_id > %s AND category_id < %s) " % (("OR" if sqltemp else ""), pcat, pcat + 100)
-             
-            sqlwhere.append(" AND e.id %s IN (SELECT event_id FROM event_category_mapping WHERE %s)" % (("NOT" if nocat else ""), sqltemp))
-            sqlparams.extend(sqlpar)
+            cats = self.getMaps(self.catmap, (cat or nocat))
+            query.append(
+                " AND e.id %s IN (SELECT event_id FROM event_category_mapping WHERE category_id IN (%s))" % (
+                    self._get_not(cat), self._get_comma_perc(cats)))
+            params.extend(cats)
 
         if tag or notag:
-            sqltemp, sqlpar = self.generateDynamicQuery(self.tagmap, "tag_id IN (%s)", (tag or notag))
-
-            sqlwhere.append(" AND e.id %s IN (SELECT event_id FROM event_tag_mapping WHERE %s)" % (("NOT" if notag else ""), sqltemp))
-            sqlparams.extend(sqlpar)
+            tags = self.getMaps(self.tagmap, (tag or notag))
+            query.append(
+                " AND e.id %s IN (SELECT event_id FROM event_tag_mapping WHERE tag_id IN (%s))" % (
+                    self._get_not(tag), self._get_comma_perc(tags)))
+            params.extend(tags)
 
         if group or nogroup:
-            not_op = "" if group else "NOT"
-            
+            subquery = []
             for identity in (group or nogroup):
-                sqlwhere.append(" AND c.identity %s LIKE %%s" % not_op)
-                sqlparams.append(identity + "%")
+                subquery.append("c.identity = %s")      # exact client
+                params.append(identity)
+                subquery.append("c.identity LIKE %s")   # whole subtree
+                params.append(identity + ".%")
+
+            query.append(" AND %s (%s)" % (self._get_not(group), " OR ".join(subquery)))
 
-        sqlwhere.append(" AND e.valid = 1 LIMIT %s")
-        sqlparams.append(count)
+        query.append(" AND e.valid = 1 LIMIT %s")
+        params.append(count)
 
-        sqlwhere_string = "".join(sqlwhere)
-        logging.debug("fetch_events: query - %s" % sqlwhere_string)
-        logging.debug("fetch_events: params - %s", str(sqlparams))
+        query_string = "".join(query)
+        logging.debug("fetch_events: query - %s" % query_string)
+        logging.debug("fetch_events: params - %s", str(params))
 
-        self.crs.execute(sqlwhere_string, sqlparams)
+        self.crs.execute(query_string, params)
         row = self.crs.fetchall()
 
         if row:
@@ -544,11 +536,11 @@ class MySQL(ObjectReq):
         try:
             self.crs.execute("INSERT INTO events (received,client_id,data) VALUES (NOW(), %s, %s)", (client.id, json.dumps(event)))
             lastid = self.crs.lastrowid
-            logging.debug("store_event: Last ID in events - %i" % lastid)
 
-            for cat in event.get('Category', ["Other"]):
+            catlist = event.get('Category', ["Other"])
+            cats = set(catlist) | {cat.split(".", 1)[0] for cat in catlist}
+            for cat in cats:
                 cat_id = self.catmap.get(cat, self.catmap_other)
-                logging.debug("store_event: Category \"%s\" translated to %i" % (cat, cat_id))
                 self.crs.execute("INSERT INTO event_category_mapping (event_id,category_id) VALUES (%s, %s)", (lastid, cat_id))
                 
             try:
@@ -558,7 +550,6 @@ class MySQL(ObjectReq):
                 
             for tag in tags:
                 tag_id = self.tagmap.get(tag, self.tagmap_other)
-                logging.debug("store_event: Tag \"%s\" translated to %i" % (tag, tag_id))
                 self.crs.execute("INSERT INTO event_tag_mapping (event_id,tag_id) VALUES (%s, %s)", (lastid, tag_id))    
 
             self.con.commit()
@@ -671,8 +662,6 @@ class Server(ObjectReq):
             if not auth:
                 raise self.req.error("I'm watching. Not authorized.", 403, detail={"client": client.identity})
 
-            logging.debug("arguments: %s" % str(args))
-
             # These args are not for handler
             args.pop("client", None)
             args.pop("secret", None)
@@ -812,11 +801,11 @@ class WardenHandler(ObjectReq):
             count = min(count, self.get_events_limit)
 
         res = self.db.fetch_events(self.req.client, id, count, cat, nocat, tag, notag, group, nogroup)
-        logging.info("fetch_events(%d, %d, %s, %s, %s, %s, %s, %s) returned %d events" % (
-            id, count, cat, nocat, tag, notag, group, nogroup, len(res["events"])))
 
         self.db.insertLastReceivedId(self.req.client, res['lastid'])
 
+        logging.info("sending %d events, lastid is %i" % (len(res["events"]), res["lastid"]))
+
         return res