Server: Make code compatible with Py3 (and Py2)

* All input files (config, maps, idea schema) are expected in UTF-8,
* Add helper get_method_params() hiding 2vs3 internal differences,
* Report Python version in getInfo,
* Argparse compatibility fixes.

warden3/warden_server/warden_server.py

@@ -8,6 +8,7 @@ from __future__ import print_function
 
 import sys
 import os
+import io
 from os import path
 import logging
 import logging.handlers
@@ -25,10 +26,19 @@ import MySQLdb.cursors as mycursors
 if sys.version_info[0] >= 3:
     import configparser as ConfigParser
     from urllib.parse import parse_qs
+    unicode = str
+
+    def get_method_params(method):
+        return method.__code__.co_varnames[:method.__code__.co_argcount]
+
 else:
     import ConfigParser
     from urlparse import parse_qs
 
+    def get_method_params(method):
+        return method.func_code.co_varnames[:method.func_code.co_argcount]
+
+
 # for local version of up to date jsonschema
 sys.path.append(path.join(path.dirname(__file__), "..", "lib"))
 
@@ -212,7 +222,7 @@ Client = namedtuple("Client", [
 class Object(object):
 
     def __str__(self):
-        attrs = self.__init__.func_code.co_varnames[1:self.__init__.func_code.co_argcount]
+        attrs = get_method_params(self.__init__)[1:]
         eq_str = ["%s=%r" % (attr, getattr(self, attr, None)) for attr in attrs]
         return "%s(%s)" % (type(self).__name__, ", ".join(eq_str))
 
@@ -435,7 +445,7 @@ class JSONSchemaValidator(NoValidator):
     def __init__(self, req, log, filename=None):
         NoValidator.__init__(self, req, log)
         self.path = filename or path.join(path.dirname(__file__), "idea.schema")
-        with open(self.path) as f:
+        with io.open(self.path, "r", encoding="utf-8") as f:
             self.schema = json.load(f)
         self.validator = Draft4Validator(self.schema)
 
@@ -477,11 +487,11 @@ class MySQL(ObjectBase):
         self.catmap_filename = catmap_filename
         self.tagmap_filename = tagmap_filename
 
-        with open(catmap_filename, "r") as catmap_fd:
+        with io.open(catmap_filename, "r", encoding="utf-8") as catmap_fd:
             self.catmap = json.load(catmap_fd)
             self.catmap_other = self.catmap["Other"]    # Catch error soon, avoid lookup later
 
-        with open(tagmap_filename, "r") as tagmap_fd:
+        with io.open(tagmap_filename, "r", encoding="utf-8") as tagmap_fd:
             self.tagmap = json.load(tagmap_fd)
             self.tagmap_other = self.catmap["Other"]    # Catch error soon, avoid lookup later
 
@@ -806,10 +816,10 @@ class MySQL(ObjectBase):
     def load_maps(self):
         with self as db:
             db.query("DELETE FROM tags")
-            for tag, num in self.tagmap.iteritems():
+            for tag, num in self.tagmap.items():
                 db.query("INSERT INTO tags(id, tag) VALUES (%s, %s)", (num, tag))
             db.query("DELETE FROM categories")
-            for cat_subcat, num in self.catmap.iteritems():
+            for cat_subcat, num in self.catmap.items():
                 catsplit = cat_subcat.split(".", 1)
                 category = catsplit[0]
                 subcategory = catsplit[1] if len(catsplit) > 1 else None
@@ -931,10 +941,15 @@ class Server(ObjectBase):
 
         # Make sure everything is properly encoded - JSON and various function
         # may spit out unicode instead of str and it gets propagated up (str
-        # + unicode = unicode). However, the right thing would be to be unicode
-        # correct among whole source and always decode on input (json module
-        # does that for us) and on output here.
-        if isinstance(status, unicode):
+        # + unicode = unicode).
+        # For Python2 the right thing would be to be unicode correct among whole
+        # source and always decode on input (json module does that for us) and
+        # on output here.
+        # For Python3 strings are internally unicode so no decoding on input is
+        # necessary. For output, "status" must be unicode string, "output" must
+        # be encoded bytes array, what is done here. Important: for Python 3 we
+        # define: unicode = str
+        if isinstance(status, unicode) and sys.version_info[0] < 3:
             status = status.encode("utf-8")
         if isinstance(output, unicode):
             output = output.encode("utf-8")
@@ -947,11 +962,10 @@ class Server(ObjectBase):
 
 
 def json_wrapper(method):
-
     def meth_deco(self, post, **args):
-        if "events" in method.func_code.co_varnames[0:method.func_code.co_argcount]:
+        if "events" in get_method_params(method):
             try:
-                events = json.loads(post) if post else None
+                events = json.loads(post.decode('utf-8')) if post else None
             except Exception as e:
                 raise self.req.error(
                     message="Deserialization error.", error=400,
@@ -973,7 +987,7 @@ def json_wrapper(method):
     try:
         meth_deco.arguments = method.arguments
     except AttributeError:
-        meth_deco.arguments = method.func_code.co_varnames[:method.func_code.co_argcount]
+        meth_deco.arguments = get_method_params(method)
     return meth_deco
 
 
@@ -997,13 +1011,14 @@ class WardenHandler(ObjectBase):
     def getDebug(self):
         return {
             "environment": self.req.env,
-            "client": self.req.client.__dict__,
+            "client": self.req.client._asdict(),
             "database": self.db.get_debug(),
             "system": {
+                "python": sys.version,
                 "uname": os.uname()
             },
             "process": {
-                "cwd": os.getcwdu(),
+                "cwd": unicode(os.getcwd()),
                 "pid": os.getpid(),
                 "ppid": os.getppid(),
                 "pgrp": os.getpgrp(),
@@ -1177,15 +1192,15 @@ def read_ini(path):
 
 
 def read_cfg(path):
-    with open(path, "r") as f:
+    with io.open(path, "r", encoding="utf-8") as f:
         stripcomments = "\n".join((l for l in f if not l.lstrip().startswith(("#", "//"))))
         conf = json.loads(stripcomments)
 
     # Lowercase keys
     conf = dict((
         sect.lower(), dict(
-            (subkey.lower(), val) for subkey, val in subsect.iteritems())
-    ) for sect, subsect in conf.iteritems())
+            (subkey.lower(), val) for subkey, val in subsect.items())
+    ) for sect, subsect in conf.items())
 
     return conf
 
@@ -1360,7 +1375,7 @@ def build_server(conf, section_order=section_order, section_def=section_def, par
 
         # Process parameters
         kwargs = {}
-        for name, definition in params.iteritems():
+        for name, definition in params.items():
             raw_val = config.get(name, definition["default"])
             try:
                 type_callable = conv_dict[definition["type"]]
@@ -1573,7 +1588,8 @@ def get_args():
     argp.add_argument(
         "-c", "--config",
         help="path to configuration file")
-    subargp = argp.add_subparsers(title="commands")
+    subargp = argp.add_subparsers(title="commands", dest="command")
+    subargp.required = True
 
     subargp_check = subargp.add_parser(
         "check", add_help=False,