(Hopefully) more robust database reconnection

warden3/warden_server/warden_server.py

@@ -527,6 +527,8 @@ class MySQL(ObjectBase):
         crs = kwargs.pop("crs", None)
         while True:
             try:
+                if self.con is None:
+                    self.connect()
                 if crs is None:
                     crs = self.con.cursor()
                 self.log.debug("execute: %s %s" % (args, kwargs))
@@ -540,9 +542,16 @@ class MySQL(ObjectBase):
                 self.log.info("execute: Database down, trying to reconnect (%d attempts left)..." % countdown)
                 if countdown<self.retry_count:
                     sleep(self.retry_pause)    # no need to melt down server on longer outage
+                try:
+                    crs.close()
+                except Exception:
+                    pass
+                try:
                     self.close()
-                self.connect()
+                except Exception:
+                    pass
                 crs = None
+                self.con = None
                 countdown -= 1