Implemented better algorithm in stop command of mentat-controller.py module.

The module now makes more and better attempts to stop all running Mentat modules. (Redmine issue: #3443)

lib/mentat/module/controller.py

@@ -382,30 +382,55 @@ class MentatControllerScript(pyzenkit.zenscript.ZenScript):
         modules should be affected by the command. All modules will be affected
         by default.
         """
-        status = self.get_system_status()
-
         self.logger.info("Stopping all configured Mentat modules:")
-        complist = self._get_target()
-        for mname in complist:
-            stat = status['modules'][mname]
-            pidf = status['pid_files'].get(mname, None)
-            proc = status['processes'].get(mname, None)
-
-            if stat[0] == mentat.system.STATUS_RT_NOT_RUNNING:
-                self.logger.info("Module '%s': Module is already not running, nothing to do", mname)
-            elif stat[0] in (mentat.system.STATUS_RT_RUNNING_OK,
-                             mentat.system.STATUS_RT_RUNNING_PF_MISSING,
-                             mentat.system.STATUS_RT_RUNNING_PID_MISSMATCH,
-                             mentat.system.STATUS_RT_RUNNING_INCONSISTENT,
-                             mentat.system.STATUS_RT_RUNNING_FEWER,
-                             mentat.system.STATUS_RT_RUNNING_MORE):
-                self.logger.info("Module '%s': Stopping module", mname)
-                self._module_signal(self.modules[mname], pidf, proc, signal.SIGINT)
-            else:
-                self.logger.error("Module '%s': Module is in weird state, unable to perform automatic shutdown", mname)
-
-        self.logger.info("Waiting for modules to fully shut down")
-        time.sleep(1)
+        modulelist = self._get_target()
+
+        # We will try to make at most 20 attempts to stop all Mentat modules.
+        for i in range(1, 20):
+            status = self.get_system_status()
+
+            nextmodulelist = []
+            if not modulelist:
+                break
+
+            # Process all modules from the list.
+            for mname in modulelist:
+                stat = status['modules'][mname]
+                pidf = status['pid_files'].get(mname, None)
+                proc = status['processes'].get(mname, None)
+
+                if stat[0] == mentat.system.STATUS_RT_NOT_RUNNING:
+                    # In case this is a first attempt the module was already not running.
+                    if i == 1:
+                        self.logger.info("Module '%s': Module is already not running, nothing to do", mname)
+                    # Otherwise the stop was successfull.
+                    else:
+                        self.logger.info("Module '%s': Module successfully stopped", mname)
+                    continue
+
+                if stat[0] in (mentat.system.STATUS_RT_RUNNING_OK,
+                               mentat.system.STATUS_RT_RUNNING_PF_MISSING,
+                               mentat.system.STATUS_RT_RUNNING_PID_MISSMATCH,
+                               mentat.system.STATUS_RT_RUNNING_INCONSISTENT,
+                               mentat.system.STATUS_RT_RUNNING_FEWER,
+                               mentat.system.STATUS_RT_RUNNING_MORE):
+                    # Perform attemt to stop the module and mark it for another check in next iteration.
+                    self.logger.info("Module '%s': Stopping module, attempt #%s", mname, i)
+                    self._module_signal(self.modules[mname], pidf, proc, signal.SIGINT)
+                    nextmodulelist.append(mname)
+                    continue
+
+                self.logger.error("Module '%s': Module is in weird state, unable to perform full automatic shutdown", mname)
+
+            # In case there are any modules marked for check in next iteration, wait for them to shut down.
+            if nextmodulelist:
+                self.logger.info("Waiting for modules to fully shut down")
+                time.sleep(1)
+
+            # Prepare module list for another iteration.
+            modulelist = nextmodulelist
+
+        # Review the overall Mentat system status.
         status = self.get_system_status()
         self.rc = status['resultm'][0]