diff --git a/warden3/warden_client/warden_client_test.py b/warden3/warden_client/warden_client_test.py
index 1cc4b4dd7dd34e0992b920f9f8e2b33c7a99985f..3eb85c43af1a3775193c2bd4e3872980a04fba38 100755
--- a/warden3/warden_client/warden_client_test.py
+++ b/warden3/warden_client/warden_client_test.py
@@ -75,8 +75,9 @@ def gen_random_idea(client_name="cz.example.warden.test"):
"WinEndTime": get_precise_timestamp(),
"EventTime": get_precise_timestamp(),
"CeaseTime": get_precise_timestamp(),
- "Category": ["Abusive.Spam","Fraud.Copyright","Test"],
+ #"Category": ["Abusive.Spam","Abusive.Harassment","Malware","Fraud.Copyright","Test","Fraud.Phishing","Fraud.Scam"],
# "Category": ["Abusive.Spam","Fraud.Copyright"],
+ "Category": [choice(["Abusive.Spam","Abusive.Harassment","Malware","Fraud.Copyright","Test","Fraud.Phishing","Fraud.Scam"]) for dummy in range(randint(1, 3))],
"Ref": ["cve:CVE-%s-%s" % (randstr(string.digits, 4), randstr()), "http://www.example.com/%s" % randstr()],
"Confidence": random(),
"Note": "Random event",
@@ -117,7 +118,7 @@ def gen_random_idea(client_name="cz.example.warden.test"):
"Node": [
{
"Name": client_name,
- "Tags": ["Protocol", "Honeypot"],
+ "Tags": [choice(["Data", "Protocol", "Honeypot", "Heuristic", "Log"]) for dummy in range(randint(1, 3))],
"SW": ["Kippo"],
"AggrWin": "00:05:00"
}
@@ -141,12 +142,28 @@ def main():
# idstore="MyClient.id",
# name="cz.example.warden.test")
+ print "=== Debug ==="
+ info = wclient.getDebug()
+ if not isinstance(info, Error):
+ pprint(info)
+
+ print "=== Server info ==="
+ info = wclient.getInfo()
+ if not isinstance(info, Error):
+ pprint(info)
+
+ print "=== Sending 10 event(s) ==="
+ start = time()
+ ret = wclient.sendEvents([gen_random_idea(client_name=wclient.name) for i in range(10)])
+ print ret
+ print "Time: %f" % (time()-start)
+
print "=== Getting 10 events ==="
start = time()
# cat = ['Availability', 'Abusive.Spam','Attempt.Login']
# cat = ['Attempt', 'Information','Fraud.Scam','Malware.Virus']
- cat = ['Attempt']
+ cat = ['Fraud', 'Abusive.Spam']
nocat = ['Availability', 'Information', 'Fraud.Scam']
tag = ['Log', 'Data']
@@ -155,31 +172,14 @@ def main():
group = ['cz.tul.ward.kippo','cz.vsb.buldog.kippo']
nogroup = ['cz.zcu.civ.afrodita','cz.vutbr.net.bee.hpscan']
- ret = wclient.getEvents(count=10, cat=None, nocat=None, tag=None, notag=None, group=None, nogroup=nogroup)
- ret = wclient.getEvents(count=10)
+ ret = wclient.getEvents(count=10, cat=cat, nocat=None, tag=tag, notag=None, group=None, nogroup=nogroup)
print "Time: %f" % (time()-start)
print "Got %i events" % len(ret)
for e in ret:
- print e
+ print e["Category"], e["Node"][0]["Tags"], e["Node"][0]["Name"]
if isinstance(ret, Error):
print ret
- print "=== Sending 1 event(s) ==="
- start = time()
- ret = wclient.sendEvents([gen_random_idea(client_name=wclient.name) for i in range(1)])
- print ret
- print "Time: %f" % (time()-start)
-
- print "=== Server info ==="
- info = wclient.getInfo()
- if not isinstance(info, Error):
- pprint(info)
-
- print "=== Debug ==="
- info = wclient.getDebug()
- if not isinstance(info, Error):
- pprint(info)
-
if __name__ == "__main__":
main()
diff --git a/warden3/warden_server/warden3.0-alpha.sql b/warden3/warden_server/warden3.0-alpha.sql
index 16d78b539dda27566524a20caa6ecbce8905ce9c..cfd494b1f919a89db9e1aa1d3ca5cd9b2f643bbd 100644
--- a/warden3/warden_server/warden3.0-alpha.sql
+++ b/warden3/warden_server/warden3.0-alpha.sql
@@ -39,40 +39,50 @@ CREATE TABLE IF NOT EXISTS `categories` (
--
INSERT INTO `categories` (`id`, `category`, `subcategory`, `cat_subcat`) VALUES
+(100, 'Abusive', NULL, 'Abusive'),
(101, 'Abusive', 'Spam', 'Abusive.Spam'),
(102, 'Abusive', 'Harassment', 'Abusive.Harassment'),
(103, 'Abusive', 'Child', 'Abusive.Child'),
(104, 'Abusive', 'Sexual', 'Abusive.Sexual'),
(105, 'Abusive', 'Violence', 'Abusive.Violence'),
+(200, 'Malware', NULL, 'Malware'),
(201, 'Malware', 'Virus', 'Malware.Virus'),
(202, 'Malware', 'Worm', 'Malware.Worm'),
(203, 'Malware', 'Trojan', 'Malware.Trojan'),
(204, 'Malware', 'Spyware', 'Malware.Spyware'),
(205, 'Malware', 'Dialer', 'Malware.Dialer'),
(206, 'Malware', 'Rootkit', 'Malware.Rootkit'),
+(300, 'Recon', NULL, 'Recon'),
(301, 'Recon', 'Scanning', 'Recon.Scanning'),
(302, 'Recon', 'Sniffing', 'Recon.Sniffing'),
(303, 'Recon', 'SocialEngineering', 'Recon.SocialEngineering'),
(304, 'Recon', 'Searching', 'Recon.Searching'),
+(400, 'Attempt', NULL, 'Attempt'),
(401, 'Attempt', 'Exploit', 'Attempt.Exploit'),
(402, 'Attempt', 'Login', 'Attempt.Login'),
(403, 'Attempt', 'NewSignature', 'Attempt.NewSignature'),
+(500, 'Intrusion', NULL, 'Intrusion'),
(501, 'Intrusion', 'AdminCompromise', 'Intrusion.AdminCompromise'),
(502, 'Intrusion', 'UserCompromise', 'Intrusion.UserCompromise'),
(503, 'Intrusion', 'AppCompromise', 'Intrusion.AppCompromise'),
(504, 'Intrusion', 'Botnet', 'Intrusion.Botnet'),
+(600, 'Availability', NULL, 'Availability'),
(601, 'Availability', 'DoS', 'Availability.DoS'),
(602, 'Availability', 'DDoS', 'Availability.DDoS'),
(603, 'Availability', 'Sabotage', 'Availability.Sabotage'),
(604, 'Availability', 'Outage', 'Availability.Outage'),
+(700, 'Information', NULL, 'Information'),
(701, 'Information', 'UnauthorizedAccess', 'Information.UnauthorizedAccess'),
(702, 'Information', 'UnauthorizedModification', 'Information.UnauthorizedModification'),
+(800, 'Fraud', NULL, 'Fraud'),
(801, 'Fraud', 'UnauthorizedUsage', 'Fraud.UnauthorizedUsage'),
(802, 'Fraud', 'Copyright', 'Fraud.Copyright'),
(803, 'Fraud', 'Masquerade', 'Fraud.Masquerade'),
(804, 'Fraud', 'Phishing', 'Fraud.Phishing'),
(805, 'Fraud', 'Scam', 'Fraud.Scam'),
+(900, 'Vulnerable', NULL, 'Vulnerable'),
(901, 'Vulnerable', 'Open', 'Vulnerable.Open'),
+(1000, 'Anomaly', NULL, 'Anomaly'),
(1001, 'Anomaly', 'Traffic', 'Anomaly.Traffic'),
(1002, 'Anomaly', 'Connection', 'Anomaly.Connection'),
(1003, 'Anomaly', 'Protocol', 'Anomaly.Protocol'),
diff --git a/warden3/warden_server/warden_server.py b/warden3/warden_server/warden_server.py
index 9c34759a69faeb0d99e020e91cb9b29d27e6823e..07e7df14936815dfd7c8f88ad6af0beabceb58e2 100755
--- a/warden3/warden_server/warden_server.py
+++ b/warden3/warden_server/warden_server.py
@@ -294,7 +294,6 @@ class X509Authenticator(NoAuthenticator):
identity = args.get("client", [None])[0]
secret = args.get("secret", [None])[0]
- args["secret"] = ["..."] # Prevent to spill it over logs
client = self.db.get_client_by_name(cert_names, identity, secret)
@@ -412,6 +411,14 @@ class MySQL(ObjectReq):
type(self).__name__, type(self.req).__name__, self.host, self.user, self.dbname, self.port, self.catmap_filename, self.tagmap_filename)
+ def _get_comma_perc(self, l):
+ return ','.join(['%s'] * len(l))
+
+
+ def _get_not(self, b):
+ return "" if b else "NOT"
+
+
def get_client_by_name(self, cert_names, identity=None, secret=None):
query = ["SELECT id, registered, requestor, hostname, service, note, identity, secret, `read`, debug, `write`, test FROM clients WHERE valid = 1"]
params = []
@@ -421,7 +428,7 @@ class MySQL(ObjectReq):
if secret:
query.append(" AND secret = %s")
params.append(secret)
- query.append(" AND hostname IN (%s)" % ','.join(['%s'] * len(cert_names)))
+ query.append(" AND hostname IN (%s)" % self._get_comma_perc(cert_names))
params.extend(cert_names)
self.crs.execute("".join(query), params)
rows = self.crs.fetchall()
@@ -446,30 +453,16 @@ class MySQL(ObjectReq):
}
- def generateDynamicQuery(self, section, query_string, variables, parent_cats = []):
- variables_id = []
-
+ def getMaps(self, section, variables):
+ maps = []
for v in variables:
try:
- mapped_id = section[v]
+ mapped = section[v]
except KeyError:
raise self.req.error("Wrong tag or category used in query.", 422,
sys.exc_info(), detail={"key": v})
-
- if mapped_id % 100:
- variables_id.append(mapped_id)
- else:
- parent_cats.append(mapped_id)
-
- temp_string = ""
-
- if len(variables_id) > 0:
- format_strings = ','.join(['%s'] * len(variables_id))
- logging.debug("query_string: %s" % query_string)
- logging.debug("format_strings: %s" % format_strings)
- temp_string = query_string % format_strings
-
- return temp_string, variables_id
+ maps.append(mapped)
+ return set(maps) # unique
def fetch_events(self, client, id, count,
@@ -489,42 +482,41 @@ class MySQL(ObjectReq):
raise self.req.error("Unrealizable conditions. Choose group or nogroup option.", 422,
detail={'tag': cat, 'notag' : nocat})
- sqlwhere = []
- sqlparams = []
-
- sqlwhere.append("SELECT e.id, e.data FROM clients c RIGHT JOIN events e ON c.id = e.client_id WHERE e.id > %s")
- sqlparams.append(id or 0)
+ query = ["SELECT e.id, e.data FROM clients c RIGHT JOIN events e ON c.id = e.client_id WHERE e.id > %s"]
+ params = [id or 0]
if cat or nocat:
- parent_cats = []
- sqltemp, sqlpar = self.generateDynamicQuery(self.catmap, "category_id IN (%s)", (cat or nocat), parent_cats)
- for pcat in parent_cats:
- sqltemp += " %s (category_id > %s AND category_id < %s) " % (("OR" if sqltemp else ""), pcat, pcat + 100)
-
- sqlwhere.append(" AND e.id %s IN (SELECT event_id FROM event_category_mapping WHERE %s)" % (("NOT" if nocat else ""), sqltemp))
- sqlparams.extend(sqlpar)
+ cats = self.getMaps(self.catmap, (cat or nocat))
+ query.append(
+ " AND e.id %s IN (SELECT event_id FROM event_category_mapping WHERE category_id IN (%s))" % (
+ self._get_not(cat), self._get_comma_perc(cats)))
+ params.extend(cats)
if tag or notag:
- sqltemp, sqlpar = self.generateDynamicQuery(self.tagmap, "tag_id IN (%s)", (tag or notag))
-
- sqlwhere.append(" AND e.id %s IN (SELECT event_id FROM event_tag_mapping WHERE %s)" % (("NOT" if notag else ""), sqltemp))
- sqlparams.extend(sqlpar)
+ tags = self.getMaps(self.tagmap, (tag or notag))
+ query.append(
+ " AND e.id %s IN (SELECT event_id FROM event_tag_mapping WHERE tag_id IN (%s))" % (
+ self._get_not(tag), self._get_comma_perc(tags)))
+ params.extend(tags)
if group or nogroup:
- not_op = "" if group else "NOT"
-
+ subquery = []
for identity in (group or nogroup):
- sqlwhere.append(" AND c.identity %s LIKE %%s" % not_op)
- sqlparams.append(identity + "%")
+ subquery.append("c.identity = %s") # exact client
+ params.append(identity)
+ subquery.append("c.identity LIKE %s") # whole subtree
+ params.append(identity + ".%")
+
+ query.append(" AND %s (%s)" % (self._get_not(group), " OR ".join(subquery)))
- sqlwhere.append(" AND e.valid = 1 LIMIT %s")
- sqlparams.append(count)
+ query.append(" AND e.valid = 1 LIMIT %s")
+ params.append(count)
- sqlwhere_string = "".join(sqlwhere)
- logging.debug("fetch_events: query - %s" % sqlwhere_string)
- logging.debug("fetch_events: params - %s", str(sqlparams))
+ query_string = "".join(query)
+ logging.debug("fetch_events: query - %s" % query_string)
+ logging.debug("fetch_events: params - %s", str(params))
- self.crs.execute(sqlwhere_string, sqlparams)
+ self.crs.execute(query_string, params)
row = self.crs.fetchall()
if row:
@@ -544,11 +536,11 @@ class MySQL(ObjectReq):
try:
self.crs.execute("INSERT INTO events (received,client_id,data) VALUES (NOW(), %s, %s)", (client.id, json.dumps(event)))
lastid = self.crs.lastrowid
- logging.debug("store_event: Last ID in events - %i" % lastid)
- for cat in event.get('Category', ["Other"]):
+ catlist = event.get('Category', ["Other"])
+ cats = set(catlist) | {cat.split(".", 1)[0] for cat in catlist}
+ for cat in cats:
cat_id = self.catmap.get(cat, self.catmap_other)
- logging.debug("store_event: Category \"%s\" translated to %i" % (cat, cat_id))
self.crs.execute("INSERT INTO event_category_mapping (event_id,category_id) VALUES (%s, %s)", (lastid, cat_id))
try:
@@ -558,7 +550,6 @@ class MySQL(ObjectReq):
for tag in tags:
tag_id = self.tagmap.get(tag, self.tagmap_other)
- logging.debug("store_event: Tag \"%s\" translated to %i" % (tag, tag_id))
self.crs.execute("INSERT INTO event_tag_mapping (event_id,tag_id) VALUES (%s, %s)", (lastid, tag_id))
self.con.commit()
@@ -671,8 +662,6 @@ class Server(ObjectReq):
if not auth:
raise self.req.error("I'm watching. Not authorized.", 403, detail={"client": client.identity})
- logging.debug("arguments: %s" % str(args))
-
# These args are not for handler
args.pop("client", None)
args.pop("secret", None)
@@ -812,11 +801,11 @@ class WardenHandler(ObjectReq):
count = min(count, self.get_events_limit)
res = self.db.fetch_events(self.req.client, id, count, cat, nocat, tag, notag, group, nogroup)
- logging.info("fetch_events(%d, %d, %s, %s, %s, %s, %s, %s) returned %d events" % (
- id, count, cat, nocat, tag, notag, group, nogroup, len(res["events"])))
self.db.insertLastReceivedId(self.req.client, res['lastid'])
+ logging.info("sending %d events, lastid is %i" % (len(res["events"]), res["lastid"]))
+
return res