Implemented the pybabel translation commands as built-in Flask CLI commands.

Pybabel translations are now implemented as built-in Flask CLI commands and accessible from 'hawat-cli intl' utility. This approach enables users to easily customize translations for their custom Mentat system installations, or adding new languages. The drawback of this approach is, that the commands are executed within the Flask`s application context, so the application must be properly installed on target system for everything to work properly. I have kept original translation commands in the makefile for future reference.

(Redmine issue: #4789,#4216)

Makefile

@@ -124,10 +124,8 @@ help:
 	@echo "  * $(GREEN)presentations$(NC): project presentations"
 	@echo ""
 	@echo "  * $(GREEN)pybabel-patch$(NC): patch babel library"
-	@echo "  * ${GREEN}hpybabel-init INIT_LOCALE=lc${NC}: init Hawat translations for given locale $(FAINT)lc$(NC)"
-	@echo "  * $(GREEN)hpybabel-pull$(NC): extract and update Hawat translations"
-	@echo "      - $(ORANGE)hpybabel-extract$(NC): extract Hawat translations"
-	@echo "      - $(ORANGE)hpybabel-update$(NC): update Hawat translations"
+	@echo "  * ${GREEN}hpybabel-init INIT_LOCALE=lc${NC}: extract and init Hawat translations for given locale $(FAINT)lc$(NC)"
+	@echo "  * $(GREEN)hpybabel-update$(NC): extract and update Hawat translations"
 	@echo "  * $(GREEN)hpybabel-compile$(NC): compile Hawat translations"
 	@echo "  * ${GREEN}mpybabel-init INIT_LOCALE=lc${NC}: init Mentat translations for given locale $(FAINT)lc$(NC)"
 	@echo "  * $(GREEN)mpybabel-pull$(NC): extract and update Mentat translations"
@@ -449,31 +447,31 @@ pybabel-patch: FORCE
 	@cd / && patch -p0 -i /var/tmp/babel.messages.frontend.py.patch
 
 #
-# When doing translation into another language, you may use following command to
-# initialize the translations directory:
+# Original solution that used the pybabel utility directly:
+#   @echo "\n$(GREEN)*** Initializing translations for new locale for Hawat user interface ***$(NC)\n"
+#   @echo "Locale name: $(INIT_LOCALE)"
+#   @cd $(DIR_LIB_HAWAT) && $(PYBABEL) extract -F babel.cfg -o messages.pot -k lazy_gettext -k tr_ .
+#   @cd $(DIR_LIB_HAWAT) && $(PYBABEL) init -i messages.pot -d translations -l $(INIT_LOCALE)
 #
-# 1. Enter directory containing the messages.pot file.
-# 2. pybabel init -i messages.pot -d translations -l [locale]
-#
-hpybabel-extract: FORCE
-	@echo "\n$(GREEN)*** Extracting babel translations for Hawat user interface ***$(NC)\n"
-	@cd $(DIR_LIB_HAWAT) && $(PYBABEL) extract -F babel.cfg -o messages.pot -k lazy_gettext -k tr_ .
-
 hpybabel-init: FORCE
-	@echo "\n$(GREEN)*** Initializing translations for new locale for Hawat user interface ***$(NC)\n"
-	@echo "Locale name: $(INIT_LOCALE)"
-	@cd $(DIR_LIB_HAWAT) && $(PYBABEL) init -i messages.pot -d translations -l $(INIT_LOCALE)
-	@echo ""
+	APP_ROOT_PATH=$(shell realpath ./chroot) hawat-cli intl init $(INIT_LOCALE)
 
+#
+# Original solution that used the pybabel utility directly:
+#   @echo "\n$(GREEN)*** Updating translations for Hawat user interface ***$(NC)\n"
+#   @cd $(DIR_LIB_HAWAT) && $(PYBABEL) extract -F babel.cfg -o messages.pot -k lazy_gettext -k tr_ .
+#   @cd $(DIR_LIB_HAWAT) && $(PYBABEL) update -i messages.pot -d translations
+#
 hpybabel-update: FORCE
-	@echo "\n$(GREEN)*** Updating translations for Hawat user interface ***$(NC)\n"
-	@cd $(DIR_LIB_HAWAT) && $(PYBABEL) update -i messages.pot -d translations -l cs
-
-hpybabel-pull: hpybabel-extract hpybabel-update
+	APP_ROOT_PATH=$(shell realpath ./chroot) hawat-cli intl update
 
+#
+# Original solution that used the pybabel utility directly:
+#   @echo "\n$(GREEN)*** Compiling translations for Hawat user interface ***$(NC)\n"
+#   @cd $(DIR_LIB_HAWAT) && $(PYBABEL) compile -d translations
+#
 hpybabel-compile: FORCE
-	@echo "\n$(GREEN)*** Compiling translations for Hawat user interface ***$(NC)\n"
-	@cd $(DIR_LIB_HAWAT) && $(PYBABEL) compile -d translations
+	APP_ROOT_PATH=$(shell realpath ./chroot) hawat-cli intl compile
 
 
 mpybabel-extract: FORCE
@@ -492,9 +490,9 @@ mpybabel-init: FORCE
 
 mpybabel-update: FORCE
 	@echo "\n$(GREEN)*** Updating translations for Mentat ***$(NC)\n"
-	@$(PYBABEL) update -i $(DIR_TEMPLATES_INFORMANT)/messages.pot -d $(DIR_TEMPLATES_INFORMANT)/translations -l cs
-	@$(PYBABEL) update -i $(DIR_TEMPLATES_REPORTER)/messages.pot -d $(DIR_TEMPLATES_REPORTER)/translations -l cs
-	@$(PYBABEL) update -i $(DIR_TEMPLATES_UTEST)/messages.pot -d $(DIR_TEMPLATES_UTEST)/translations -l cs
+	@$(PYBABEL) update -i $(DIR_TEMPLATES_INFORMANT)/messages.pot -d $(DIR_TEMPLATES_INFORMANT)/translations
+	@$(PYBABEL) update -i $(DIR_TEMPLATES_REPORTER)/messages.pot -d $(DIR_TEMPLATES_REPORTER)/translations
+	@$(PYBABEL) update -i $(DIR_TEMPLATES_UTEST)/messages.pot -d $(DIR_TEMPLATES_UTEST)/translations
 
 mpybabel-pull: mpybabel-extract mpybabel-update
 

lib/hawat/app.py

@@ -56,6 +56,7 @@ import hawat.acl
 import hawat.log
 import hawat.mailer
 import hawat.db
+import hawat.intl
 import hawat.events
 import hawat.errors
 from hawat.models.user import GuiUserModel
@@ -110,7 +111,7 @@ def create_app_full(
     _setup_app_eventdb(app)
     _setup_app_auth(app)
     _setup_app_acl(app)
-    _setup_app_babel(app)
+    _setup_app_intl(app)
     _setup_app_menu(app)
     _setup_app_blueprints(app)
 
@@ -717,7 +718,7 @@ def _setup_app_acl(app):
     return app
 
 
-def _setup_app_babel(app):
+def _setup_app_intl(app):
     """
     Setup application`s internationalization sybsystem.
 
@@ -725,8 +726,9 @@ def _setup_app_babel(app):
     :return: Modified Hawat application
     :rtype: hawat.base.HawatApp
     """
-    babel = flask_babel.Babel(app)
-    app.set_resource(hawat.const.RESOURCE_BABEL, babel)
+    hawat.intl.BABEL.init_app(app)
+    app.set_resource(hawat.const.RESOURCE_BABEL, hawat.intl.BABEL)
+    app.cli.add_command(hawat.intl.INTL_CLI)
 
     @app.route('/locale/<code>')
     def locale(code):  # pylint: disable=locally-disabled,unused-variable
@@ -765,94 +767,6 @@ def _setup_app_babel(app):
             )
         )
 
-    @babel.localeselector
-    def get_locale():  # pylint: disable=locally-disabled,unused-variable
-        """
-        Implementation of locale selector for :py:mod:`flask_babel`.
-        """
-        # If a user is logged in, try to use the locale from the user settings.
-        if flask_login.current_user.is_authenticated:
-            if hasattr(flask_login.current_user, 'locale') and flask_login.current_user.locale:
-                flask.session['locale'] = flask_login.current_user.locale
-
-        # Store the best locale selection into the session.
-        if 'locale' not in flask.session:
-            flask.session['locale'] = flask.request.accept_languages.best_match(app.config['SUPPORTED_LOCALES'].keys())
-
-        return flask.session['locale']
-
-    @babel.timezoneselector
-    def get_timezone():  # pylint: disable=locally-disabled,unused-variable
-        """
-        Implementation of timezone selector for :py:mod:`flask_babel`.
-        """
-        # If a user is logged in, try to use the timezone from the user settings.
-        if flask_login.current_user.is_authenticated:
-            if hasattr(flask_login.current_user, 'timezone') and flask_login.current_user.timezone:
-                flask.session['timezone'] = flask_login.current_user.timezone
-
-        # Store the default timezone selection into the session.
-        if 'timezone' not in flask.session:
-            flask.session['timezone'] = flask.current_app.config['BABEL_DEFAULT_TIMEZONE']
-
-        return flask.session['timezone']
-
-    @app.before_request
-    def before_request():  # pylint: disable=locally-disabled,unused-variable
-        """
-        Use Flask`s :py:func:`flask.Flask.before_request` hook for storing
-        currently selected locale and timezone to request`s global storage.
-        """
-        flask.g.locale = flask.session.get('locale', get_locale())
-        flask.g.timezone = flask.session.get('timezone', get_timezone())
-
-    def babel_format_bytes(size, unit = 'B', step_size = 1024):
-        """
-        Format given numeric value to human readable string describing size in
-        B/KB/MB/GB/TB.
-
-        :param int size: Number to be formatted.
-        :param enum unit: Starting unit, possible values are ['B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB'].
-        :param int step_size: Size of the step between units.
-        :return: Formatted and localized string.
-        :rtype: string
-        """
-        units = ['B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB']
-        idx_max = len(units) - 1
-        unit = unit.upper()
-        for idx, val in enumerate(units):
-            # Skip the last step, there is no next unit defined after exabyte.
-            if idx == idx_max:
-                break
-            if size > step_size:
-                if unit == val:
-                    size = size / step_size
-                    unit = units[idx+1]
-            else:
-                break
-        return '{} {}'.format(
-            flask_babel.format_decimal(size),
-            unit
-        )
-
-    def babel_translate_locale(locale_id, with_current = False):
-        """
-        Translate given locale language. By default return language in locale`s
-        language. Optionaly return language in given locale`s language.
-        """
-        locale_obj = Locale.parse(locale_id)
-        if not with_current:
-            return locale_obj.language_name
-        return locale_obj.get_language_name(flask_babel.get_locale())
-
-    def babel_language_in_locale(locale_id = 'en'):
-        """
-        Translate given locale language. By default return language in locale`s
-        language. Optionaly return language in given locale`s language.
-        """
-        locale_obj = Locale.parse(flask_babel.get_locale())
-        return locale_obj.get_language_name(locale_id)
-
     @app.context_processor
     def utility_processor():  # pylint: disable=locally-disabled,unused-variable
         """
@@ -871,9 +785,9 @@ def _setup_app_babel(app):
             babel_format_timedelta   = flask_babel.format_timedelta,
             babel_format_decimal     = flask_babel.format_decimal,
             babel_format_percent     = flask_babel.format_percent,
-            babel_format_bytes       = babel_format_bytes,
-            babel_translate_locale   = babel_translate_locale,
-            babel_language_in_locale = babel_language_in_locale
+            babel_format_bytes       = hawat.intl.babel_format_bytes,
+            babel_translate_locale   = hawat.intl.babel_translate_locale,
+            babel_language_in_locale = hawat.intl.babel_language_in_locale
         )
 
     return app

lib/hawat/const.py

@@ -17,11 +17,21 @@ __author__ = "Jan Mach <jan.mach@cesnet.cz>"
 __credits__ = "Pavel Kácha <pavel.kacha@cesnet.cz>, Andrea Kropáčová <andrea.kropacova@cesnet.cz>"
 
 
+import re
 import datetime
 
 
-HAWAT_EMAIL_RE = r"^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$"
-"""Regular expression for email address format validation."""
+CRE_LOGIN = re.compile('^[-_@.a-zA-Z0-9]+$')
+"""Compiled regular expression for login validation."""
+
+CRE_EMAIL = re.compile(r"^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$")
+"""Compiled regular expression for email address format validation."""
+
+CRE_COUNTRY_CODE = re.compile('^[a-zA-Z]{2,3}$')
+"""Compiled regular expression for validating language/country codes."""
+
+CRE_LANG_CODE = re.compile('^[a-zA-Z]{2}(_[a-zA-Z]{2})?$')
+"""Compiled regular expression for validating language codes."""
 
 
 HAWAT_DEFAULT_LOCALE = 'en'

lib/hawat/intl.py

+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#-------------------------------------------------------------------------------
+# This file is part of Mentat system (https://mentat.cesnet.cz/).
+#
+# Copyright (C) since 2011 CESNET, z.s.p.o (http://www.ces.net/)
+# Use of this source is governed by the MIT license, see LICENSE file.
+#-------------------------------------------------------------------------------
+
+
+"""
+This module contains very thin database abstraction layer and access functions for
+Hawat. It is a wrapper around `SQLAlchemy <http://www.sqlalchemy.org/>`__ library.
+"""
+
+
+__author__ = "Jan Mach <jan.mach@cesnet.cz>"
+__credits__ = "Pavel Kácha <pavel.kacha@cesnet.cz>, Andrea Kropáčová <andrea.kropacova@cesnet.cz>"
+
+
+import os
+
+#
+# Flask related modules.
+#
+import click
+import flask
+from flask.cli import AppGroup
+import flask_babel
+import flask_login
+from babel import Locale
+
+
+import hawat.const
+
+
+BABEL = flask_babel.Babel()
+@BABEL.localeselector
+def get_locale():  # pylint: disable=locally-disabled,unused-variable
+    """
+    Implementation of locale selector for :py:mod:`flask_babel`.
+    """
+    # If a user is logged in, try to use the locale from the user settings.
+    if flask_login.current_user.is_authenticated:
+        if hasattr(flask_login.current_user, 'locale') and flask_login.current_user.locale:
+            flask.session['locale'] = flask_login.current_user.locale
+
+    # Store the best locale selection into the session.
+    if 'locale' not in flask.session:
+        flask.session['locale'] = flask.request.accept_languages.best_match(
+            flask.current_app.config['SUPPORTED_LOCALES'].keys()
+        )
+
+    return flask.session['locale'] or flask.current_app.config['BABEL_DEFAULT_LOCALE']
+
+@BABEL.timezoneselector
+def get_timezone():  # pylint: disable=locally-disabled,unused-variable
+    """
+    Implementation of timezone selector for :py:mod:`flask_babel`.
+    """
+    # If a user is logged in, try to use the timezone from the user settings.
+    if flask_login.current_user.is_authenticated:
+        if hasattr(flask_login.current_user, 'timezone') and flask_login.current_user.timezone:
+            flask.session['timezone'] = flask_login.current_user.timezone
+
+    # Store the default timezone selection into the session.
+    if 'timezone' not in flask.session:
+        flask.session['timezone'] = flask.current_app.config['BABEL_DEFAULT_TIMEZONE']
+
+    return flask.session['timezone']
+
+
+def babel_format_bytes(size, unit = 'B', step_size = 1024):
+    """
+    Format given numeric value to human readable string describing size in
+    B/KB/MB/GB/TB.
+
+    :param int size: Number to be formatted.
+    :param enum unit: Starting unit, possible values are ['B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB'].
+    :param int step_size: Size of the step between units.
+    :return: Formatted and localized string.
+    :rtype: string
+    """
+    units = ['B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB']
+    idx_max = len(units) - 1
+    unit = unit.upper()
+    for idx, val in enumerate(units):
+        # Skip the last step, there is no next unit defined after exabyte.
+        if idx == idx_max:
+            break
+        if size > step_size:
+            if unit == val:
+                size = size / step_size
+                unit = units[idx+1]
+        else:
+            break
+    return '{} {}'.format(
+        flask_babel.format_decimal(size),
+        unit
+    )
+
+def babel_translate_locale(locale_id, with_current = False):
+    """
+    Translate given locale language. By default return language in locale`s
+    language. Optionaly return language in given locale`s language.
+    """
+    locale_obj = Locale.parse(locale_id)
+    if not with_current:
+        return locale_obj.language_name
+    return locale_obj.get_language_name(flask_babel.get_locale())
+
+def babel_language_in_locale(locale_id = 'en'):
+    """
+    Translate given locale language. By default return language in locale`s
+    language. Optionaly return language in given locale`s language.
+    """
+    locale_obj = Locale.parse(flask_babel.get_locale())
+    return locale_obj.get_language_name(locale_id)
+
+
+#-------------------------------------------------------------------------------
+
+
+TRANSLATIONS_ROOT = os.path.realpath(os.path.dirname(__file__))
+TRANSLATIONS_CFG  = os.path.join(TRANSLATIONS_ROOT, 'babel.cfg')
+TRANSLATIONS_POT  = os.path.join(TRANSLATIONS_ROOT, 'messages.pot')
+TRANSLATIONS_DIR  = os.path.join(TRANSLATIONS_ROOT, 'translations')
+
+CMD_BABEL_EXTRACT = 'pybabel extract -F {} -o {} -k lazy_gettext -k tr_ {}'.format(
+    TRANSLATIONS_CFG,
+    TRANSLATIONS_POT,
+    TRANSLATIONS_ROOT
+)
+CMD_BABEL_UPDATE = 'pybabel update -i {} -d {}'.format(
+    TRANSLATIONS_POT,
+    TRANSLATIONS_DIR,
+)
+CMD_BABEL_COMPILE = 'pybabel compile -d {}'.format(
+    TRANSLATIONS_DIR
+)
+CMD_BABEL_INIT = 'pybabel init -i {} -d {} -l {{}}'.format(
+    TRANSLATIONS_POT,
+    TRANSLATIONS_DIR,
+)
+
+
+INTL_CLI = AppGroup('intl', help = "Web interface translation module.")
+
+
+@INTL_CLI.command('update')
+def intl_update():
+    """Update all existing translation message catalogs."""
+    _extract()
+    _update()
+    _clean()
+
+
+@INTL_CLI.command('compile')
+def intl_compile():
+    """Compile all existing message translation catalogs."""
+    _compile()
+
+
+def validate_lang(ctx, param, value):
+    """Validate ``login/email`` command line parameter."""
+    if value:
+        if hawat.const.CRE_LANG_CODE.match(value):
+            return value
+        raise click.BadParameter(
+            "Value '{}' does not look like valid language code.".format(value)
+        )
+
+@INTL_CLI.command()
+@click.argument('lang', callback = validate_lang)
+def init(lang):
+    """Initialize a new language translation."""
+    _extract()
+    _init(lang)
+    _clean()
+
+
+#-------------------------------------------------------------------------------
+
+
+def _extract():
+    click.secho("\n***Extracting web interface translations ***\n", fg = 'yellow')
+    click.echo("$ {}\n".format(CMD_BABEL_EXTRACT))
+    if os.system(CMD_BABEL_EXTRACT):
+        raise RuntimeError('pybabel extract command failed')
+    click.secho("[OK] Successfully extracted translations", fg = 'green')
+
+def _update():
+    click.secho("\n*** Updating all web interface message translation catalogs ***\n", fg = 'yellow')
+    click.echo("$ {}\n".format(CMD_BABEL_UPDATE))
+    if os.system(CMD_BABEL_UPDATE):
+        raise RuntimeError('pybabel update command failed')
+    click.secho("[OK] Successfully updated all message translation catalogs", fg = 'green')
+
+def _compile():
+    click.secho("\n*** Compiling all web interface message translation catalogs ***\n", fg = 'yellow')
+    click.echo("$ {}\n".format(CMD_BABEL_COMPILE))
+    if os.system(CMD_BABEL_COMPILE):
+        raise RuntimeError('pybabel compile command failed')
+    click.secho("[OK] Successfully compiled all message translation catalogs", fg = 'green')
+
+def _init(lang):
+    click.secho("\n*** Initializing new web interface translation ***\n", fg = 'yellow')
+    click.echo("Locale name: {}\n".format(lang))
+    click.echo("$ {}\n".format(CMD_BABEL_INIT.format(lang)))
+    if os.system(CMD_BABEL_INIT.format(lang)):
+        raise RuntimeError('pybabel init command failed')
+    click.secho(
+        "[OK] Successfully initialized translations for '{}' language.".format(lang),
+        fg = 'green'
+    )
+
+def _clean():
+    os.remove(TRANSLATIONS_POT)