From 096a10cb8f6318f24c1af1ff94fa27f7e68fd339 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Pavel=20K=C3=A1cha?= <ph@cesnet.cz>
Date: Wed, 14 Jan 2015 18:42:04 +0100
Subject: [PATCH] fetch_events and friends modified to use SQL placeholders
---
warden3/warden_server/warden_server.py | 174 ++++++++++---------------
1 file changed, 70 insertions(+), 104 deletions(-)
diff --git a/warden3/warden_server/warden_server.py b/warden3/warden_server/warden_server.py
index f9b0a04..2d43bb4 100755
--- a/warden3/warden_server/warden_server.py
+++ b/warden3/warden_server/warden_server.py
@@ -307,7 +307,7 @@ class MySQL(Object):
client["services"] = services
- logging.debug("Client/services: " + str(client))
+ logging.debug("get_client_by_name: %s", str(client))
return client
@@ -341,73 +341,77 @@ class MySQL(Object):
}
+ def generateDynamicQuery(self, section, query_string, variables, parent_cats = []):
+ variables_id = []
+
+ for v in variables:
+ mapped_id = self.map_id(section, v)
+ if mapped_id % 100:
+ variables_id.append(mapped_id)
+ else:
+ parent_cats.append(mapped_id)
+
+ format_strings = ','.join(['%s'] * len(variables_id))
+ temp_string = query_string % format_strings
+
+ return temp_string, variables_id
+
+
def fetch_events(self, client, id, count,
cat=None, nocat=None,
tag=None, notag=None,
group=None, nogroup=None):
- sqlwhere = []
- sqltemp = {}
-
logging.debug("fetch_events: id=%i, count=%i, cat=%s, nocat=%s, tag=%s, notag=%s, group=%s, nogroup=%s" % (id, count, str(cat), str(nocat), str(tag), str(notag), str(group), str(nogroup)))
- if cat is not None and nocat is not None:
+ if cat and nocat:
raise Error("Unrealizable conditions. Choose cat or nocat option.", 500, method='getEvents',
exc=sys.exc_info(), detail={'cat': cat, 'nocat' : nocat})
-
- if cat is not None or nocat is not None:
- if cat is not None:
- parent_cats = []
- sqltemp['cat'] = self.generateDynamicQuery("Category", "category_id IN (%s)", cat, parent_cats)
- for pcats in parent_cats:
- sqltemp['cat'] += " %s category_id DIV %s = 1 " % (("OR" if sqltemp['cat'] else ""), pcats)
- if nocat is not None:
- parent_cats = []
- sqltemp['cat'] = self.generateDynamicQuery("Category", "category_id NOT IN (%s)", nocat, parent_cats)
- for pcats in parent_cats:
- sqltemp['cat'] += " %s category_id DIV %s = 1 " % (("OR" if sqltemp['cat'] else ""), pcats)
-
- sqlwhere.append("e.id IN (SELECT event_id FROM event_category_mapping WHERE %s)" % sqltemp['cat'])
-
- if tag is not None and notag is not None:
+ if tag and notag:
raise Error("Unrealizable conditions. Choose tag or notag option.", 500, method='getEvents',
exc=sys.exc_info(), detail={'tag': cat, 'notag' : nocat})
-
- if tag is not None or notag is not None:
- if tag is not None:
- sqltemp['tag'] = self.generateDynamicQuery("Tag", "tag_id IN (%s)", tag)
- if notag is not None:
- sqltemp['tag'] = self.generateDynamicQuery("Tag", "tag_id NOT IN (%s)", notag)
-
- sqlwhere.append("e.id IN (SELECT event_id FROM event_tag_mapping WHERE %s)" % sqltemp['tag'])
-
-
- if group is not None and nogroup is not None:
+ if group and nogroup:
raise Error("Unrealizable conditions. Choose group or nogroup option.", 500, method='getEvents',
exc=sys.exc_info(), detail={'tag': cat, 'notag' : nocat})
- if group is not None or nogroup is not None:
- sqltemp['group'] = ""
+ sqlwhere = []
+ sqlparams = []
- if group is not None:
- for identity in group:
- sqltemp['group'] += ("s.identity LIKE '%s' AND " % (identity))
- if nogroup is not None:
- for identity in nogroup:
- sqltemp['group'] += ("s.identity NOT LIKE '%s' AND " % (identity))
+ sqlwhere.append("SELECT e.id, e.data FROM services s RIGHT JOIN events e ON s.service_id = e.service_id WHERE e.id > %s")
+ sqlparams.append(id or 0)
- # logging.debug(sqltemp['group'][:-4])
- sqlwhere.append(sqltemp['group'][:-4])
+ if cat or nocat:
+ not_op = "" if cat else "NOT"
+ parent_cats = []
+ sqltemp, sqlpar = self.generateDynamicQuery("Category", "category_id %s IN (%%s)" % not_op, (cat or nocat), parent_cats)
+ for pcats in parent_cats:
+ sqltemp += " %s category_id DIV %s = 1 " % (("OR" if sqltemp else ""), pcats)
+
+ sqlwhere.append(" AND e.id IN (SELECT event_id FROM event_category_mapping WHERE %s)" % sqltemp)
+ sqlparams.extend(sqlpar)
- sqlwhere_string = (" AND " . join(sqlwhere))
- # logging.debug(sqlwhere_string)
- # logging.debug(' AND ' . join(sqlwhere))
-
- #sqlwhere = sqlwhere[:-4]
- and_op = "" if not sqlwhere_string else "AND"
+ if tag or notag:
+ not_op = "" if tag else "NOT"
+ sqltemp, sqlpar = self.generateDynamicQuery("Tag", "tag_id %s IN (%%s)" % not_op, (tag or notag))
+
+ sqlwhere.append(" AND e.id IN (SELECT event_id FROM event_tag_mapping WHERE %s)" % sqltemp)
+ sqlparams.extend(sqlpar)
+
+ if group or nogroup:
+ not_op = "" if group else "NOT"
+
+ for identity in (group or nogroup):
+ sqlwhere.append(" AND s.identity %s LIKE %%s" % not_op)
+ sqlparams.append(identity + "%")
- logging.debug("SELECT e.id, e.data FROM services s RIGHT JOIN events e ON s.id = e.service_id WHERE e.id > %s AND %s %s e.valid = 1 LIMIT %s" % (str(id or 0), sqlwhere_string, and_op, str(count)))
- self.crs.execute("SELECT e.id, e.data FROM services s RIGHT JOIN events e ON s.service_id = e.service_id WHERE e.id > %s AND %s %s e.valid = 1 LIMIT %s" % (str(id or 0), sqlwhere_string, and_op, str(count)))
+ sqlwhere.append(" AND e.valid = 1 LIMIT %s")
+ sqlparams.append(count)
+
+ sqlwhere_string = "".join(sqlwhere)
+ logging.debug("fetch_events: query - %s" % sqlwhere_string)
+ logging.debug("fetch_events: params - %s", str(sqlparams))
+
+ self.crs.execute(sqlwhere_string, sqlparams)
row = self.crs.fetchall()
if row:
@@ -415,14 +419,11 @@ class MySQL(Object):
else:
maxid = self.getLastEventId()
- # logging.debug("MAX ID = %s", str(maxid))
- #for r in row:
- # logging.debug(json.loads(r["data"]))
+ events = [json.loads(r["data"]) for r in row]
return {
"lastid": maxid,
- # "lastid": row[-1]['id'] if row else str(id),
- "events": [json.loads(r["data"]) for r in row]
+ "events": events
}
@@ -455,8 +456,8 @@ class MySQL(Object):
def insertLastReceivedId(self, client, id):
- logging.debug("INSERT INTO last_events(client_id, event_id, timestamp) VALUES(%s, %s, NOW())" % (str(client["id"]), id))
- self.crs.execute("INSERT INTO last_events(client_id, event_id, timestamp) VALUES(%s, %s, NOW())" % (str(client["id"]), id))
+ logging.debug("insertLastReceivedId: id %i for client %i(%s)" % (id, client["id"], client["hostname"]))
+ self.crs.execute("INSERT INTO last_events(client_id, event_id, timestamp) VALUES(%s, %s, NOW())", (client["id"], id))
def getLastEventId(self):
self.crs.execute("SELECT MAX(id) as id FROM events")
@@ -465,16 +466,13 @@ class MySQL(Object):
return row['id'] if row['id'] is not None else 0
def getLastReceivedId(self, client):
- client_id = client["id"]
- logging.debug("getLastReceivedId (client_id) = %s", str(client_id))
-
- logging.debug("SELECT MAX(event_id) as id FROM last_events WHERE client_id = %s" % (str(client_id)))
- self.crs.execute("SELECT MAX(event_id) as id FROM last_events WHERE client_id = %s" % (str(client_id)))
+ self.crs.execute("SELECT MAX(event_id) as id FROM last_events WHERE client_id = %s", client["id"])
row = self.crs.fetchone()
- logging.debug("getLastReceivedId - %s" % str(row['id']))
+ id = row['id'] if row is not None else 0
+ logging.debug("getLastReceivedId: id %i for client %i(%s)" % (id, client["id"], client["hostname"]))
- return row['id'] if row is not None else 0
+ return id
def map_id (self, section, key):
@@ -556,24 +554,6 @@ class MySQL(Object):
return data[section][key]
except KeyError:
return 0
-
-
- def generateDynamicQuery(self, section, query_string, variables, parent_cats = []):
- variables_id = []
- # parent_cats = []
-
- for v in variables:
- mapped_id = self.map_id(section, v)
- if mapped_id % 100 != 0:
- variables_id.append(mapped_id)
- else:
- parent_cats.append(mapped_id)
-
- # variables_id = [self.map_id(section, v) for v in variables if self.map_id(section, v) % 100 != 0]
- format_strings = ','.join(['\'%s\''] * len(variables_id))
- temp_string = query_string % format_strings
-
- return temp_string % tuple(variables_id)
@@ -643,21 +623,16 @@ class Server(Object):
exc=sys.exc_info(), detail={"args": injson})
args = parse_qs(environ.get('QUERY_STRING', ""))
- #for k, v in args.iteritems():
- # args[k] = v[0]
logging.debug("%s called with %s" % (path, str(args)))
if events:
args["events"] = events
- # if not self.auth.authorize(environ, client, path, args):
- # raise Error("I'm watching YOU.", 403, method=path, detail={"client": client})
-
args = self.sanitize_args(path, method, args)
result = method(_env=environ, _client=client, **args) # call requested method
try:
- # 'default': takes care of non JSON serializable objects,
- # which could (although shouldn't) appear in handler code
+ # 'default': takes care of non JSON serializable objects,
+ # which could (although shouldn't) appear in handler code
output = json.dumps(result, default=lambda v: str(v))
except Exception as e:
raise Error("Serialization error", 500, method=path,
@@ -748,25 +723,18 @@ class WardenHandler(Object):
try:
id = int(id[0])
except (ValueError, TypeError, IndexError):
- id = 0
+ id = None
- if not id:
+ if id is None:
try:
id = self.db.getLastReceivedId(_client)
- # logging.debug("Last received ID for %s is %s" % (_client['hostname'], str(id)))
except Exception, e:
- logging.error("getEvents: cannot getLastReceivedId - " + type(e).__name__ + ": " + e)
- id = 0
+ logging.info("getEvents: cannot getLastReceivedId - " + type(e).__name__ + ": " + e)
- if not id:
- try:
- id = self.db.getLastEventId()
- except Exception as e:
- raise Error("Last event id receiving error", 500, detail={"client": _client})
-
- # First access, remember the guy
+ if id is None:
+ # First access, remember the guy and get him last event
+ id = self.db.getLastEventId()
self.db.insertLastReceivedId(_client, id)
-
return {
"lastid": id,
"events": []
@@ -780,13 +748,11 @@ class WardenHandler(Object):
if self.get_events_limit:
count = min(count, self.get_events_limit)
- logging.debug("getEvents - count: %s" % count)
res = self.db.fetch_events(_client, id, count, cat, nocat, tag, notag, group, nogroup)
logging.info("getEvents(%d, %d, %s, %s, %s, %s, %s, %s): sending %d events" % (
id, count, cat, nocat, tag, notag, group, nogroup, len(res["events"])))
self.db.insertLastReceivedId(_client, res['lastid'])
- logging.debug("lastid inserting: %s" % {'lastid': res['lastid'], 'client' : _client})
return res
--
GitLab