Rewrite kolla-ansible CLI to python

Moving the CLI to python allows for easier
maintenance and larger feature-set.

This patch introduces a few breaking changes!
The changes stem the nature of the cliff package.
- the order of parameters must be
  kolla-ansible <action> <arguments>
- mariadb_backup and mariadb_recovery now are
  mariadb-backup and mariadb-recovery

Closes-bug: #1589020
Signed-off-by: Roman Krček <roman.krcek@tietoevry.com>
Change-Id: I9749b320d4f5eeec601a055b597dfa7d8fb97ce2

doc/source/reference/deployment-and-bootstrapping/bifrost.rst

@@ -284,13 +284,15 @@ For development:
 
 .. code-block:: console
 
-   cd kolla-ansible
+   pip install -e ./kolla-ansible
-   tools/kolla-ansible deploy-bifrost
+   kolla-ansible deploy-bifrost
+
 
 For Production:
 
 .. code-block:: console
 
+   pip install -U ./kolla-ansible
    kolla-ansible deploy-bifrost
 
 Deploy Bifrost manually
@@ -376,12 +378,14 @@ For Development:
 
 .. code-block:: console
 
-   tools/kolla-ansible deploy-servers
+   pip install -e ./kolla-ansible
+   kolla-ansible deploy-servers
 
 For Production:
 
 .. code-block:: console
 
+   pip install -U ./kolla-ansible
    kolla-ansible deploy-servers
 
 Manually

kolla_ansible/ansible.py

@@ -0,0 +1,318 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import logging
+import os
+import subprocess  # nosec
+import sys
+
+from kolla_ansible import utils
+from typing import List
+from typing import Tuple
+
+
+DEFAULT_CONFIG_PATH = "/etc/kolla"
+
+CONFIG_PATH_ENV = "KOLLA_CONFIG_PATH"
+
+LOG = logging.getLogger(__name__)
+
+
+def add_ansible_args(parser):
+    """Add arguments required for running Ansible playbooks to a parser."""
+    parser.add_argument(
+        "-b",
+        "--become",
+        action="store_true",
+        help="run operations with become (nopasswd implied)",
+    )
+    parser.add_argument(
+        "-C",
+        "--check",
+        action="store_true",
+        help="don't make any changes; instead, try to predict "
+        "some of the changes that may occur",
+    )
+    parser.add_argument(
+        "-D",
+        "--diff",
+        action="store_true",
+        help="when changing (small) files and templates, show "
+        "the differences in those files; works great "
+        "with --check",
+    )
+    parser.add_argument(
+        "-e",
+        "--extra-vars",
+        metavar="EXTRA_VARS",
+        action="append",
+        help="set additional variables as key=value or "
+        "YAML/JSON",
+    )
+    parser.add_argument(
+        "-i",
+        "--inventory",
+        metavar="INVENTORY",
+        action="append",
+        help="specify inventory host path ",
+    )
+    parser.add_argument(
+        "-l",
+        "--limit",
+        metavar="SUBSET",
+        help="further limit selected hosts to an additional "
+        "pattern",
+    )
+    parser.add_argument(
+        "--skip-tags",
+        metavar="TAGS",
+        help="only run plays and tasks whose tags do not "
+        "match these values",
+    )
+    parser.add_argument(
+        "-t",
+        "--tags",
+        metavar="TAGS",
+        help="only run plays and tasks tagged with these "
+        "values",
+    )
+    parser.add_argument(
+        "-lt",
+        "--list-tasks",
+        action="store_true",
+        help="only print names of tasks, don't run them, "
+        "note this has no affect on kolla-ansible.",
+    )
+    parser.add_argument(
+        "-p", "--playbook",
+        metavar="PLAYBOOKS",
+        action="append",
+        help="Specify custom playbooks for kolla ansible "
+        "to use"
+    ),
+    parser.add_argument(
+        "--vault-id",
+        metavar="VAULT_IDS",
+        action="append",
+        help="the vault identity to use. "
+             "This argument may be specified multiple times.",
+        default=[]
+    ),
+    parser.add_argument(
+        "--vault-password-file",
+        "--vault-pass-file",
+        metavar="VAULT_APSSWORD_FILES",
+        action="append",
+        help="vault password file",
+        default=[]
+    ),
+    parser.add_argument(
+        "-J",
+        "--ask-vault-password",
+        "--ask-vault-pass",
+        action="store_true",
+        help="ask for vault password"
+    )
+
+
+def add_kolla_ansible_args(parser):
+    """Add arguments required for running Kolla Ansible to a parser."""
+    default_config_path = os.getenv(CONFIG_PATH_ENV, DEFAULT_CONFIG_PATH)
+    parser.add_argument(
+        "--configdir",
+        default=default_config_path,
+        dest="kolla_config_path",
+        help="path to Kolla configuration."
+        "(default=$%s or %s)" % (CONFIG_PATH_ENV, DEFAULT_CONFIG_PATH),
+    )
+    parser.add_argument(
+        "--passwords",
+        dest="kolla_passwords",
+        help="Path to the kolla ansible passwords file"
+    )
+
+
+def _get_inventory_paths(parsed_args) -> List[str]:
+    """Return path to the Kolla Ansible inventory."""
+    if parsed_args.inventory:
+        return parsed_args.inventory
+
+    default_inventory = os.path.join(
+        os.path.abspath(parsed_args.kolla_config_path),
+        "ansible", "inventory", "all-in-one")
+    return [default_inventory]
+
+
+def _validate_args(parsed_args, playbooks: list) -> None:
+    """Validate Kolla Ansible arguments."""
+    result = utils.is_readable_dir(
+        os.path.abspath(parsed_args.kolla_config_path))
+    if not result["result"]:
+        LOG.error(
+            "Kolla Ansible configuration path %s is invalid: %s",
+            os.path.abspath(parsed_args.kolla_config_path),
+            result["message"],
+        )
+        sys.exit(1)
+
+    inventories = _get_inventory_paths(parsed_args)
+    for inventory in inventories:
+        result = utils.is_readable_dir(inventory)
+        if not result["result"]:
+            # NOTE(mgoddard): Previously the inventory was a file, now it is a
+            # directory to allow us to support inventory host_vars. Support
+            # both formats for now.
+            result_f = utils.is_readable_file(inventory)
+            if not result_f["result"]:
+                LOG.error(
+                    "Kolla inventory %s is invalid: %s",
+                    inventory, result["message"]
+                )
+                sys.exit(1)
+
+    for playbook in playbooks:
+        result = utils.is_readable_file(playbook)
+        if not result["result"]:
+            LOG.error(
+                "Kolla Ansible playbook %s is invalid: %s",
+                playbook, result["message"]
+            )
+            sys.exit(1)
+
+    if parsed_args.kolla_passwords:
+        passwd_file = parsed_args.kolla_passwords
+    else:
+        passwd_file = os.path.join(
+            os.path.abspath(parsed_args.kolla_config_path), "passwords.yml")
+    result = utils.is_readable_file(passwd_file)
+    if not result["result"]:
+        LOG.error("Kolla Ansible passwords file %s is invalid: %s",
+                  passwd_file, result["message"])
+
+    globals_file = os.path.join(os.path.abspath(
+        os.path.abspath(parsed_args.kolla_config_path)), "globals.yml")
+    result = utils.is_readable_file(globals_file)
+    if not result["result"]:
+        LOG.error("Kolla ansible globals file %s is invalid %s",
+                  globals_file, result["message"])
+
+
+def _get_vars_files(config_path: os.path) -> List[str]:
+    """Return a list of Kolla Ansible configuration variable files.
+
+    The globals.d directory in config path is searched to create the list of
+    variable files. The files will be sorted alphabetically by name for each
+    file, but ordering of file is kept to allow overrides.
+    """
+    vars_path = os.path.join(config_path, "globals.d")
+    result = utils.is_readable_dir(vars_path)
+    if not result["result"]:
+        return []
+
+    vars_files = []
+    for vars_file in os.listdir(vars_path):
+        abs_path = os.path.join(vars_path, vars_file)
+        if utils.is_readable_file(abs_path)["result"]:
+            root, ext = os.path.splitext(vars_file)
+            if ext in (".yml", ".yaml", ".json"):
+                vars_files.append(abs_path)
+
+    return sorted(vars_files)
+
+
+def build_args(parsed_args,
+               playbooks: list,
+               extra_vars: dict = {},
+               verbose_level: int = None) -> Tuple[str, List[str]]:
+    """Build arguments required for running Ansible playbooks."""
+    args = list()
+    if verbose_level:
+        args += ["-" + "v" * verbose_level]
+    if parsed_args.list_tasks:
+        args += ["--list-tasks"]
+    inventories = _get_inventory_paths(parsed_args)
+    for inventory in inventories:
+        args += ["--inventory", inventory]
+    args += ["-e", "@%s" % os.path.join(
+        os.path.abspath(parsed_args.kolla_config_path),
+        "globals.yml")]
+    args += ["-e", "@%s" % os.path.join(
+        os.path.abspath(parsed_args.kolla_config_path),
+        "passwords.yml")]
+    for vault_id in parsed_args.vault_id:
+        args += ["--vault-id", vault_id]
+    for vault_pass_file in parsed_args.vault_password_file:
+        args += ["--vault-password-file", vault_pass_file]
+    if parsed_args.ask_vault_password:
+        args += "--ask-vault-password"
+    vars_files = _get_vars_files(
+        os.path.abspath(parsed_args.kolla_config_path))
+    for vars_file in vars_files:
+        args += ["-e", "@%s" % vars_file]
+    if parsed_args.extra_vars:
+        for extra_var in parsed_args.extra_vars:
+            args += ["-e", extra_var]
+    if extra_vars:
+        for extra_var_name, extra_var_value in extra_vars.items():
+            args += ["-e", "%s=%s" % (extra_var_name, extra_var_value)]
+    args += ["-e", "CONFIG_DIR=%s" %
+             os.path.abspath(parsed_args.kolla_config_path)]
+    if parsed_args.become:
+        args += ["--become"]
+    if parsed_args.check:
+        args += ["--check"]
+    if parsed_args.diff:
+        args += ["--diff"]
+    if parsed_args.limit:
+        args += ["--limit", parsed_args.limit]
+    if parsed_args.skip_tags:
+        args += ["--skip-tags", parsed_args.skip_tags]
+    if parsed_args.tags:
+        args += ["--tags", parsed_args.tags]
+    args += [" ".join(playbooks)]
+    return ("ansible-playbook", args)
+
+
+def run_playbooks(parsed_args, playbooks: list, extra_vars: dict = {},
+                  quiet: bool = False, verbose_level: int = 0) -> None:
+    """Run a Kolla Ansible playbook."""
+    LOG.debug("Parsed arguments: %s" % parsed_args)
+    _validate_args(parsed_args, playbooks)
+    (executable, args) = build_args(
+        parsed_args,
+        playbooks,
+        extra_vars=extra_vars,
+        verbose_level=verbose_level,
+    )
+
+    try:
+        utils.run_command(executable, args, quiet=quiet)
+    except subprocess.CalledProcessError as e:
+        LOG.error(
+            "Kolla Ansible playbook(s) %s exited %d", ", ".join(
+                playbooks), e.returncode
+        )
+        sys.exit(e.returncode)
+
+
+def install_galaxy_collections(force: bool = True) -> None:
+    """Install Ansible Galaxy collection dependencies.
+
+    Installs collection dependencies specified in kolla-ansible,
+    and if present, in kolla-ansibnle configuration.
+
+    :param force: Whether to force reinstallation of roles.
+    """
+    requirements = utils.get_data_files_path("requirements.yml")
+    requirements_core = utils.get_data_files_path("requirements-core.yml")
+    utils.galaxy_collection_install(requirements, force=force)
+    utils.galaxy_collection_install(requirements_core, force=force)

kolla_ansible/cli/commands.py

@@ -0,0 +1,470 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from cliff.command import Command
+
+from kolla_ansible import ansible
+from kolla_ansible import utils
+
+# Serial is not recommended and disabled by default.
+# Users can enable it by configuring the variable.
+ANSIBLE_SERIAL = 0
+
+
+def _get_playbook_path(playbook):
+    """Return the absolute path of Kolla Ansible playbook"""
+    return utils.get_data_files_path("ansible", "%s.yml" % playbook)
+
+
+def _choose_playbooks(parsed_args, kolla_playbook="site"):
+    """Return user defined playbook if set, otherwise return Kolla playbook"""
+    if parsed_args.playbook:
+        playbooks = parsed_args.playbook
+    else:
+        playbooks = [_get_playbook_path(kolla_playbook)]
+    return playbooks
+
+
+class KollaAnsibleMixin:
+    """Mixin class for commands running Kolla Ansible."""
+
+    def get_parser(self, prog_name):
+        parser = super(KollaAnsibleMixin, self).get_parser(prog_name)
+        ansible_group = parser.add_argument_group("Ansible arguments")
+        ka_group = parser.add_argument_group("Kolla Ansible arguments")
+        self.add_ansible_args(ansible_group)
+        self.add_kolla_ansible_args(ka_group)
+        return parser
+
+    def add_kolla_ansible_args(self, group):
+        ansible.add_kolla_ansible_args(group)
+
+    def add_ansible_args(self, group):
+        ansible.add_ansible_args(group)
+
+    def _get_verbosity_args(self):
+        """Add quietness and verbosity level arguments."""
+        # Cliff's default verbosity level is 1, 0 means quiet.
+        verbosity_args = {}
+        if self.app.options.verbose_level:
+            ansible_verbose_level = self.app.options.verbose_level - 1
+            verbosity_args["verbose_level"] = ansible_verbose_level
+        else:
+            verbosity_args["quiet"] = True
+        return verbosity_args
+
+    def run_playbooks(self, parsed_args, *args, **kwargs):
+        kwargs.update(self._get_verbosity_args())
+        return ansible.run_playbooks(parsed_args, *args, **kwargs)
+
+
+class GatherFacts(KollaAnsibleMixin, Command):
+    """Gather Ansible facts on hosts"""
+
+    def take_action(self, parsed_args):
+        self.app.LOG.info("Gathering Ansible facts")
+
+        playbooks = _choose_playbooks(parsed_args, "gather-facts")
+
+        self.run_playbooks(parsed_args, playbooks)
+
+
+class InstallDeps(KollaAnsibleMixin, Command):
+    """Install Ansible Galaxy dependencies"""
+
+    def take_action(self, parsed_args):
+        self.app.LOG.info("Installing Ansible Galaxy dependencies")
+        ansible.install_galaxy_collections()
+
+
+class Prechecks(KollaAnsibleMixin, Command):
+    """Do pre-deployment checks for hosts"""
+
+    def take_action(self, parsed_args):
+        self.app.LOG.info("Pre-deployment checking")
+
+        extra_vars = {}
+        extra_vars["kolla_action"] = "precheck"
+
+        playbooks = _choose_playbooks(parsed_args,)
+
+        self.run_playbooks(parsed_args, playbooks, extra_vars=extra_vars)
+
+
+class GenConfig(KollaAnsibleMixin, Command):
+    """Generate configuration files for services. No container changes!"""
+
+    def take_action(self, parsed_args):
+        self.app.LOG.info(
+            "Generate configuration files for enabled OpenStack services")
+
+        extra_vars = {}
+        extra_vars["kolla_action"] = "config"
+
+        playbooks = _choose_playbooks(parsed_args)
+
+        self.run_playbooks(parsed_args, playbooks, extra_vars=extra_vars)
+
+
+class Reconfigure(KollaAnsibleMixin, Command):
+    """Reconfigure enabled OpenStack service"""
+
+    def take_action(self, parsed_args):
+        self.app.LOG.info("Reconfigure OpenStack service")
+
+        extra_vars = {}
+        extra_vars["kolla_action"] = "reconfigure"
+        extra_vars["kolla_serial"] = ANSIBLE_SERIAL
+
+        playbooks = _choose_playbooks(parsed_args)
+
+        self.run_playbooks(parsed_args, playbooks, extra_vars=extra_vars)
+
+
+class ValidateConfig(KollaAnsibleMixin, Command):
+    """Validate configuration files for enabled OpenStack services"""
+
+    def take_action(self, parsed_args):
+        self.app.LOG.info("Validate configuration files for enabled "
+                          "OpenStack services")
+
+        extra_vars = {}
+        extra_vars["kolla_action"] = "config_validate"
+
+        playbooks = _choose_playbooks(parsed_args)
+
+        self.run_playbooks(parsed_args, playbooks, extra_vars=extra_vars)
+
+
+class BootstrapServers(KollaAnsibleMixin, Command):
+    """Bootstrap servers with Kolla Ansible deploy dependencies"""
+
+    def take_action(self, parsed_args):
+        self.app.LOG.info("Bootstrapping servers")
+
+        extra_vars = {}
+        extra_vars["kolla_action"] = "bootstrap-servers"
+
+        playbooks = _choose_playbooks(parsed_args, "kolla-host")
+
+        self.run_playbooks(parsed_args, playbooks, extra_vars=extra_vars)
+
+
+class Pull(KollaAnsibleMixin, Command):
+    """Pull all images for containers. Only pulls, no container changes."""
+
+    def take_action(self, parsed_args):
+        self.app.LOG.info("Pulling Docker images")
+
+        extra_vars = {}
+        extra_vars["kolla_action"] = "pull"
+
+        playbooks = _choose_playbooks(parsed_args)
+
+        self.run_playbooks(parsed_args, playbooks, extra_vars=extra_vars)
+
+
+class Certificates(KollaAnsibleMixin, Command):
+    """Generate self-signed certificate for TLS *For Development Only*"""
+
+    def take_action(self, parsed_args):
+        self.app.LOG.info("Generate TLS Certificates")
+
+        playbooks = _choose_playbooks(parsed_args, "certificates")
+
+        self.run_playbooks(parsed_args, playbooks)
+
+
+class OctaviaCertificates(KollaAnsibleMixin, Command):
+    """Generate certificates for octavia deployment"""
+
+    def get_parser(self, prog_name):
+        parser = super().get_parser(prog_name)
+        group = parser.add_argument_group("Octavia certificates action")
+        group.add_argument(
+            "--check-expiry",
+            type=int,
+            help="Check if the certificates will expire "
+                 "within given number of days",
+        )
+        return parser
+
+    def take_action(self, parsed_args):
+        extra_vars = {}
+
+        if hasattr(parsed_args, "check_expiry"):
+            self.app.LOG.info("Checking if certificates expire "
+                              "within given number of days.")
+            extra_vars["octavia_certs_check_expiry"] = "yes"
+            extra_vars["octavia_certs_expiry_limit"] = parsed_args.check_expiry
+        else:
+            self.app.LOG.info("Generate octavia Certificates")
+
+        playbooks = _choose_playbooks(parsed_args, "octavia-certificates")
+
+        self.run_playbooks(parsed_args, playbooks, extra_vars=extra_vars)
+
+
+class Deploy(KollaAnsibleMixin, Command):
+    """Generate config, bootstrap and start all Kolla Ansible containers"""
+
+    def take_action(self, parsed_args):
+        self.app.LOG.info("Deploying Playbooks")
+
+        extra_vars = {}
+        extra_vars["kolla_action"] = "deploy"
+
+        playbooks = _choose_playbooks(parsed_args)
+
+        self.run_playbooks(parsed_args, playbooks, extra_vars=extra_vars)
+
+
+class DeployContainers(KollaAnsibleMixin, Command):
+    """Only deploy and start containers (no config updates or bootstrapping)"""
+
+    def take_action(self, parsed_args):
+        self.app.LOG.info("Deploying Containers")
+
+        extra_vars = {}
+        extra_vars["kolla_action"] = "deploy-containers"
+
+        playbooks = _choose_playbooks(parsed_args)
+
+        self.run_playbooks(parsed_args, playbooks, extra_vars=extra_vars)
+
+
+class Postdeploy(KollaAnsibleMixin, Command):
+    """Do post deploy on deploy node"""
+
+    def take_action(self, parsed_args):
+        self.app.LOG.info("Post-Deploying Playbooks")
+
+        playbooks = _choose_playbooks(parsed_args, "post-deploy")
+
+        self.run_playbooks(parsed_args, playbooks)
+
+
+class Upgrade(KollaAnsibleMixin, Command):
+    """Upgrades existing OpenStack Environment"""
+
+    def take_action(self, parsed_args):
+        self.app.LOG.info("Upgrading OpenStack Environment")
+
+        extra_vars = {}
+        extra_vars["kolla_action"] = "upgrade"
+        extra_vars["kolla_serial"] = ANSIBLE_SERIAL
+
+        playbooks = _choose_playbooks(parsed_args)
+
+        self.run_playbooks(parsed_args, playbooks, extra_vars=extra_vars)
+
+
+class Stop(KollaAnsibleMixin, Command):
+    """Stop Kolla Ansible containers"""
+
+    def get_parser(self, prog_name):
+        parser = super().get_parser(prog_name)
+        group = parser.add_argument_group("Stop action")
+        group.add_argument(
+            "--yes-i-really-really-mean-it",
+            action="store_true",
+            required=True,
+            help="WARNING! This action will remove the Openstack deployment!",
+        )
+        return parser
+
+    def take_action(self, parsed_args):
+        self.app.LOG.info("Stop Kolla containers")
+
+        extra_vars = {}
+        extra_vars["kolla_action"] = "stop"
+
+        playbooks = _choose_playbooks(parsed_args)
+
+        self.run_playbooks(parsed_args, playbooks, extra_vars=extra_vars)
+
+
+class Destroy(KollaAnsibleMixin, Command):
+    """Destroy Kolla Ansible containers, volumes and host configuration!"""
+
+    def get_parser(self, prog_name):
+        parser = super().get_parser(prog_name)
+        group = parser.add_argument_group("Destroy action")
+        group.add_argument(
+            "--yes-i-really-really-mean-it",
+            action="store_true",
+            required=True,
+            help="WARNING! This action will remove the Openstack deployment!",
+        )
+        group.add_argument(
+            "--include-dev",
+            action="store_true",
+            help="Remove devevelopment environment",
+        )
+        group.add_argument(
+            "--include-images",
+            action="store_true",
+            help="Remove leftover container images",
+        )
+        return parser
+
+    def take_action(self, parsed_args):
+        self.app.LOG.warning("WARNING: This will PERMANENTLY DESTROY "
+                             "all deployed kolla containers, volumes "
+                             "and host configuration. There is no way "
+                             "to recover from this action!")
+
+        extra_vars = {}
+        extra_vars["kolla_action"] = "destroy"
+        extra_vars["destroy_include_dev"] = (
+            "yes" if parsed_args.include_dev else "no"
+        )
+        extra_vars["destroy_include_images"] = (
+            "yes" if parsed_args.include_images else "no"
+        )
+
+        playbooks = _choose_playbooks(parsed_args, "destroy")
+
+        self.run_playbooks(parsed_args, playbooks, extra_vars=extra_vars)
+
+
+class PruneImages(KollaAnsibleMixin, Command):
+    """Prune orphaned Kolla Ansible docker images"""
+
+    def get_parser(self, prog_name):
+        parser = super().get_parser(prog_name)
+        group = parser.add_argument_group("Prune images action")
+        group.add_argument(
+            "--yes-i-really-really-mean-it",
+            action="store_true",
+            required=True,
+            help="WARNING! This action will remove all orphaned images!",
+        )
+        return parser
+
+    def take_action(self, parsed_args):
+        self.app.LOG.info("Prune orphaned Kolla images")
+
+        playbooks = _choose_playbooks(parsed_args, "prune-images")
+
+        self.run_playbooks(parsed_args, playbooks)
+
+
+class BifrostDeploy(KollaAnsibleMixin, Command):
+    """Deploy and start bifrost container"""
+
+    def take_action(self, parsed_args):
+        self.app.LOG.info("Deploying Bifrost")
+
+        extra_vars = {}
+        extra_vars["kolla_action"] = "deploy"
+
+        playbooks = _choose_playbooks(parsed_args, "bifrost")
+
+        self.run_playbooks(parsed_args, playbooks, extra_vars=extra_vars)
+
+
+class BifrostDeployServers(KollaAnsibleMixin, Command):
+    """Enroll and deploy servers with bifrost"""
+
+    def take_action(self, parsed_args):
+        self.app.LOG.info("Deploying servers with bifrost")
+
+        extra_vars = {}
+        extra_vars["kolla_action"] = "deploy-servers"
+
+        playbooks = _choose_playbooks(parsed_args, "bifrost")
+
+        self.run_playbooks(parsed_args, playbooks, extra_vars=extra_vars)
+
+
+class BifrostUpgrade(KollaAnsibleMixin, Command):
+    """Upgrades an existing bifrost container"""
+
+    def take_action(self, parsed_args):
+        self.app.LOG.info("Upgrading Bifrost")
+
+        extra_vars = {}
+        extra_vars["kolla_action"] = "upgrade"
+
+        playbooks = _choose_playbooks(parsed_args, "bifrost")
+
+        self.run_playbooks(parsed_args, playbooks, extra_vars=extra_vars)
+
+
+class RabbitMQResetState(KollaAnsibleMixin, Command):
+    """Force reset the state of RabbitMQ"""
+
+    def take_action(self, parsed_args):
+        self.app.LOG.info("Force reset the state of RabbitMQ")
+
+        playbooks = _choose_playbooks(parsed_args, "rabbitmq-reset-state")
+
+        self.run_playbooks(parsed_args, playbooks)
+
+
+class MariaDBBackup(KollaAnsibleMixin, Command):
+    """Take a backup of MariaDB databases. See help for options."""
+
+    def get_parser(self, prog_name):
+        parser = super().get_parser(prog_name)
+        group = parser.add_argument_group("MariaDB backup type")
+        group.add_argument(
+            "--full",
+            action="store_const",
+            const="full",
+            dest="mariadb_backup_type",
+            default="full"
+        )
+        group.add_argument(
+            "--incremental",
+            action="store_const",
+            const="incremental",
+            dest="mariadb_backup_type"
+        )
+        return parser
+
+    def take_action(self, parsed_args):
+        self.app.LOG.info("Backup MariaDB databases")
+
+        extra_vars = {}
+        extra_vars["kolla_action"] = "backup"
+        extra_vars["mariadb_backup_type"] = parsed_args.mariadb_backup_type
+
+        playbooks = _choose_playbooks(parsed_args, "mariadb_backup")
+
+        self.run_playbooks(parsed_args, playbooks, extra_vars=extra_vars)
+
+
+class MariaDBRecovery(KollaAnsibleMixin, Command):
+    """Recover a completely stopped MariaDB cluster"""
+
+    def take_action(self, parsed_args):
+        self.app.LOG.info("Attempting to restart MariaDB cluster")
+
+        extra_vars = {}
+        extra_vars["kolla_action"] = "deploy"
+
+        playbooks = _choose_playbooks(parsed_args, "mariadb_recovery")
+
+        self.run_playbooks(parsed_args, playbooks, extra_vars=extra_vars)
+
+
+class NovaLibvirtCleanup(KollaAnsibleMixin, Command):
+    """Clean up disabled nova_libvirt containers"""
+
+    def take_action(self, parsed_args):
+        self.app.LOG.info("Cleanup disabled nova_libvirt containers")
+
+        playbooks = _choose_playbooks(parsed_args, "nova-libvirt-cleanup")
+
+        self.run_playbooks(parsed_args, playbooks)

kolla_ansible/cmd/kolla_ansible.py

@@ -0,0 +1,50 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import sys
+
+from cliff.app import App
+from cliff.commandmanager import CommandManager
+
+from kolla_ansible import version
+
+
+class KollaAnsibleApp(App):
+
+    def __init__(self):
+        release_version = version.version_info.release_string()
+        super().__init__(
+            description="Kolla Ansible Command Line Interface (CLI)",
+            version=release_version,
+            command_manager=CommandManager("kolla_ansible.cli"),
+            deferred_help=True,
+        )
+
+    def initialize_app(self, argv):
+        self.LOG.debug("initialize_app")
+
+    def prepare_to_run_command(self, cmd):
+        self.LOG.debug("prepare_to_run_command %s", cmd.__class__.__name__)
+
+    def clean_up(self, cmd, result, err):
+        self.LOG.debug("clean_up %s", cmd.__class__.__name__)
+        if err:
+            self.LOG.debug("got an error: %s", err)
+
+
+def main(argv=sys.argv[1:]):
+    myapp = KollaAnsibleApp()
+    return myapp.run(argv)
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))

kolla_ansible/utils.py

@@ -0,0 +1,173 @@
+# Copyright (c) 2017 StackHPC Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import glob
+import json
+import logging
+import os
+import subprocess  # nosec
+import sys
+import yaml
+
+from importlib.metadata import Distribution
+
+LOG = logging.getLogger(__name__)
+
+
+def get_data_files_path(*relative_path) -> os.path:
+    """Given a relative path to a data file, return the absolute path"""
+    # Detect editable pip install / python setup.py develop and use a path
+    # relative to the source directory
+    return os.path.join(_get_base_path(), *relative_path)
+
+
+def _detect_install_prefix(path: os.path) -> str:
+    script_path = os.path.realpath(path)
+    script_path = os.path.normpath(script_path)
+    components = script_path.split(os.sep)
+    # use heuristic: anything before the last 'lib' in path is the prefix
+    if 'lib' not in components:
+        return None
+    last_lib = len(components) - 1 - components[::-1].index('lib')
+    prefix = components[:last_lib]
+    prefix_path = os.sep.join(prefix)
+    return prefix_path
+
+
+def _get_direct_url(dist: Distribution) -> str:
+    direct_url = os.path.join(dist._path, 'direct_url.json')
+    if os.path.isfile(direct_url):
+        with open(direct_url, 'r') as f:
+            direct_url_content = json.loads(f.readline().strip())
+            url = direct_url_content['url']
+            prefix = 'file://'
+            if url.startswith(prefix):
+                return url[len(prefix):]
+
+    return None
+
+
+def _get_base_path() -> os.path:
+    """Return location where kolla-ansible package is installed."""
+    override = os.environ.get("KOLLA_ANSIBLE_DATA_FILES_PATH")
+    if override:
+        return os.path.join(override)
+
+    kolla_ansible_dist = list(Distribution.discover(name="kolla_ansible"))
+    if kolla_ansible_dist:
+        direct_url = _get_direct_url(kolla_ansible_dist[0])
+        if direct_url:
+            return direct_url
+
+    egg_glob = os.path.join(
+        sys.prefix, 'lib*', 'python*', '*-packages', 'kolla-ansible.egg-link'
+    )
+    egg_link = glob.glob(egg_glob)
+    if egg_link:
+        with open(egg_link[0], "r") as f:
+            realpath = f.readline().strip()
+        return os.path.join(realpath)
+
+    prefix = _detect_install_prefix(__file__)
+    if prefix:
+        return os.path.join(prefix, "share", "kolla-ansible")
+
+    # Assume uninstalled
+    return os.path.join(os.path.dirname(os.path.realpath(__file__)), "..")
+
+
+def galaxy_collection_install(requirements_file: str,
+                              collections_path: str = None,
+                              force: bool = False) -> None:
+    """Install ansible collections needed by kolla-ansible roles."""
+    requirements = read_yaml_file(requirements_file)
+    if not isinstance(requirements, dict):
+        # Handle legacy role list format, which causes the command to fail.
+        return
+    args = ["collection", "install"]
+    if collections_path:
+        args += ["--collections-path", collections_path]
+    args += ["--requirements-file", requirements_file]
+    if force:
+        args += ["--force"]
+    try:
+        run_command("ansible-galaxy", args)
+    except subprocess.CalledProcessError as e:
+        LOG.error("Failed to install Ansible collections from %s via Ansible "
+                  "Galaxy: returncode %d", requirements_file, e.returncode)
+        sys.exit(e.returncode)
+
+
+def read_file(path: os.path, mode: str = "r") -> str | bytes:
+    """Read the content of a file."""
+    with open(path, mode) as f:
+        return f.read()
+
+
+def read_yaml_file(path: os.path):
+    """Read and decode a YAML file."""
+    try:
+        content = read_file(path)
+    except IOError as e:
+        print("Failed to open YAML file %s: %s" %
+              (path, repr(e)))
+        sys.exit(1)
+    try:
+        return yaml.safe_load(content)
+    except yaml.YAMLError as e:
+        print("Failed to decode YAML file %s: %s" %
+              (path, repr(e)))
+        sys.exit(1)
+
+
+def is_readable_dir(path: os.path) -> bool:
+    """Check whether a path references a readable directory."""
+    if not os.path.exists(path):
+        return {"result": False, "message": "Path does not exist"}
+    if not os.path.isdir(path):
+        return {"result": False, "message": "Path is not a directory"}
+    if not os.access(path, os.R_OK):
+        return {"result": False, "message": "Directory is not readable"}
+    return {"result": True}
+
+
+def is_readable_file(path: os.path) -> bool:
+    """Check whether a path references a readable file."""
+    if not os.path.exists(path):
+        return {"result": False, "message": "Path does not exist"}
+    if not os.path.isfile(path):
+        return {"result": False, "message": "Path is not a file"}
+    if not os.access(path, os.R_OK):
+        return {"result": False, "message": "File is not readable"}
+    return {"result": True}
+
+
+def run_command(executable: str,
+                args: list,
+                quiet: bool = False,
+                **kwargs) -> None:
+    """Run a command, checking the output.
+
+    :param quiet: Redirect output to /dev/null
+    """
+    full_cmd = [executable] + args
+    cmd_string = " ".join(full_cmd)
+    LOG.debug("Running command: %s", cmd_string)
+
+    if quiet:
+        kwargs["stdout"] = subprocess.DEVNULL
+        kwargs["stderr"] = subprocess.DEVNULL
+        subprocess.run(full_cmd, shell=False, **kwargs)  # nosec
+    else:
+        subprocess.run(full_cmd, shell=False, **kwargs)  # nosec

releasenotes/notes/python-cli-3e568065b8706e73.yaml

@@ -0,0 +1,16 @@
+---
+upgrade:
+  - |
+    Rewrite kolla-ansible CLI to python
+
+    Moving the CLI to python allows for easier
+    maintenance and larger feature set.
+    The CLI was built using the cliff package
+    that is used in openstack-cli and kayobe-cli.
+
+    This patch introduces a few breaking changes.
+    The changes stem the nature of the cliff package.
+    1. the order of parameters must be
+    kolla-ansible <action> <arguments>
+    2. mariadb_backup and mariadb_recovery now are
+    mariadb-backup and mariadb-recovery

requirements.txt

@@ -23,7 +23,5 @@ oslo.utils>=3.33.0 # Apache-2.0
 # Password hashing
 passlib[bcrypt]>=1.0.0 # BSD
 
-pbr!=2.1.0,>=2.0.0 # Apache-2.0
+# CLI
-
+cliff>=4.7.0 # Apache-2.0
-# YAML parsing
-PyYAML>=3.12 # MIT

setup.cfg

@@ -38,12 +38,37 @@ data_files =
     share/kolla-ansible = requirements.yml
     share/kolla-ansible = requirements-core.yml
 
-scripts =
-    tools/kolla-ansible
-
 [entry_points]
 console_scripts =
     kolla-genpwd = kolla_ansible.cmd.genpwd:main
     kolla-mergepwd = kolla_ansible.cmd.mergepwd:main
     kolla-writepwd = kolla_ansible.cmd.writepwd:main
     kolla-readpwd = kolla_ansible.cmd.readpwd:main
+    kolla-ansible = kolla_ansible.cmd.kolla_ansible:main
+
+kolla-ansible.cli =
+    gather-facts = kolla_ansible.cli.commands:GatherFacts
+    install-deps = kolla_ansible.cli.commands:InstallDeps
+    prechecks = kolla_ansible.cli.commands:Prechecks
+    genconfig = kolla_ansible.cli.commands:GenConfig
+    reconfigure = kolla_ansible.cli.commands:Reconfigure
+    validate-config = kolla_ansible.cli.commands:ValidateConfig
+    bootstrap-servers = kolla_ansible.cli.commands:BootstrapServers
+    pull = kolla_ansible.cli.commands:Pull
+    certificates = kolla_ansible.cli.commands:Certificates
+    octavia-certificates = kolla_ansible.cli.commands:OctaviaCertificates
+    deploy = kolla_ansible.cli.commands:Deploy
+    deploy-containers = kolla_ansible.cli.commands:DeployContainers
+    post-deploy = kolla_ansible.cli.commands:Postdeploy
+    upgrade = kolla_ansible.cli.commands:Upgrade
+    stop = kolla_ansible.cli.commands:Stop
+    destroy = kolla_ansible.cli.commands:Destroy
+    prune-images = kolla_ansible.cli.commands:PruneImages
+    deploy-bifrost = kolla_ansible.cli.commands:BifrostDeploy
+    deploy-servers = kolla_ansible.cli.commands:BifrostDeployServers
+    upgrade-bifrost = kolla_ansible.cli.commands:BifrostUpgrade
+    rabbitmq-reset-state = kolla_ansible.cli.commands:RabbitMQResetState
+    mariadb-backup = kolla_ansible.cli.commands:MariaDBBackup
+    mariadb-recovery = kolla_ansible.cli.commands:MariaDBRecovery
+    nova-libvirt-cleanup = kolla_ansible.cli.commands:NovaLibvirtCleanup
+

tests/deploy-bifrost.sh

@@ -16,7 +16,7 @@ function deploy_bifrost {
     # Deploy the bifrost container.
     # TODO(mgoddard): add pull action when we have a local registry service in
     # CI.
-    kolla-ansible -i ${RAW_INVENTORY} -vvv deploy-bifrost &> /tmp/logs/ansible/deploy-bifrost
+    kolla-ansible deploy-bifrost -i ${RAW_INVENTORY} -vvv  &> /tmp/logs/ansible/deploy-bifrost
 }
 
 

tests/deploy.sh

@@ -41,7 +41,7 @@ function certificates {
 
     # generate self-signed certificates for the optional internal TLS tests
     if [[ "$TLS_ENABLED" = "True" ]]; then
-        kolla-ansible -i ${RAW_INVENTORY} -vvv certificates > /tmp/logs/ansible/certificates
+        kolla-ansible certificates -i ${RAW_INVENTORY} -vvv > /tmp/logs/ansible/certificates
     fi
     if [[ "$LE_ENABLED" = "True" ]]; then
         init_pebble
@@ -64,13 +64,13 @@ function deploy {
     certificates
 
     # Actually do the deployment
-    kolla-ansible -i ${RAW_INVENTORY} -vvv prechecks &> /tmp/logs/ansible/deploy-prechecks
+    kolla-ansible prechecks -i ${RAW_INVENTORY} -vvv &> /tmp/logs/ansible/deploy-prechecks
-    kolla-ansible -i ${RAW_INVENTORY} -vvv pull &> /tmp/logs/ansible/pull
+    kolla-ansible pull -i ${RAW_INVENTORY} -vvv &> /tmp/logs/ansible/pull
-    kolla-ansible -i ${RAW_INVENTORY} -vvv deploy &> /tmp/logs/ansible/deploy
+    kolla-ansible deploy -i ${RAW_INVENTORY} -vvv &> /tmp/logs/ansible/deploy
-    kolla-ansible -i ${RAW_INVENTORY} -vvv post-deploy &> /tmp/logs/ansible/post-deploy
+    kolla-ansible post-deploy -i ${RAW_INVENTORY} -vvv &> /tmp/logs/ansible/post-deploy
 
     if [[ $HAS_UPGRADE == 'no' ]]; then
-        kolla-ansible -i ${RAW_INVENTORY} -vvv validate-config &> /tmp/logs/ansible/validate-config
+        kolla-ansible validate-config -i ${RAW_INVENTORY} -vvv &> /tmp/logs/ansible/validate-config
     fi
 }
 

tests/reconfigure.sh

@@ -18,8 +18,8 @@ function reconfigure {
     if [[ $SCENARIO == "ovn" ]]; then
         sudo ${CONTAINER_ENGINE} rm -f ovn_nb_db ovn_sb_db && sudo ${CONTAINER_ENGINE} volume rm ovn_nb_db ovn_sb_db
     fi
-    kolla-ansible -i ${RAW_INVENTORY} -vvv prechecks &> /tmp/logs/ansible/reconfigure-prechecks
+    kolla-ansible prechecks -i ${RAW_INVENTORY} -vvv &> /tmp/logs/ansible/reconfigure-prechecks
-    kolla-ansible -i ${RAW_INVENTORY} -vvv reconfigure &> /tmp/logs/ansible/reconfigure
+    kolla-ansible reconfigure -i ${RAW_INVENTORY} -vvv &> /tmp/logs/ansible/reconfigure
 }
 
 

tests/setup_gate.sh

@@ -116,7 +116,7 @@ EOF
 RAW_INVENTORY=/etc/kolla/inventory
 
 source $KOLLA_ANSIBLE_VENV_PATH/bin/activate
-kolla-ansible -i ${RAW_INVENTORY} -vvv bootstrap-servers &> /tmp/logs/ansible/bootstrap-servers
+kolla-ansible bootstrap-servers -i ${RAW_INVENTORY} -vvv  &> /tmp/logs/ansible/bootstrap-servers
 deactivate
 
 prepare_images

tests/test-magnum.sh

@@ -125,7 +125,7 @@ EOF
     deactivate
     source $KOLLA_ANSIBLE_VENV_PATH/bin/activate
     echo 'designate_enable_notifications_sink: "yes"' >> /etc/kolla/globals.yml
-    kolla-ansible -i ${RAW_INVENTORY} --tags designate,nova,nova-cell,neutron -vvv reconfigure &> /tmp/logs/ansible/reconfigure-designate
+    kolla-ansible reconfigure -i ${RAW_INVENTORY} --tags designate,nova,nova-cell,neutron -vvv &> /tmp/logs/ansible/reconfigure-designate
     deactivate
     source ~/openstackclient-venv/bin/activate
 

tests/test-mariadb.sh

@@ -11,7 +11,7 @@ export PYTHONUNBUFFERED=1
 
 function mariadb_stop {
     echo "Stopping the database cluster"
-    kolla-ansible -i ${RAW_INVENTORY} -vvv stop --yes-i-really-really-mean-it --tags mariadb --skip-tags common
+    kolla-ansible stop -i ${RAW_INVENTORY} -vvv --yes-i-really-really-mean-it --tags mariadb --skip-tags common
     if [[ $(sudo ${container_engine} ps -q | grep mariadb | wc -l) -ne 0 ]]; then
         echo "Failed to stop MariaDB cluster"
         return 1
@@ -21,7 +21,7 @@ function mariadb_stop {
 function mariadb_recovery {
     # Recover the database cluster.
     echo "Recovering the database cluster"
-    kolla-ansible -i ${RAW_INVENTORY} -vvv mariadb_recovery --tags mariadb --skip-tags common
+    kolla-ansible mariadb-recovery -i ${RAW_INVENTORY} -vvv --tags mariadb --skip-tags common
 }
 
 function test_recovery {
@@ -32,7 +32,7 @@ function test_recovery {
 
 function test_backup {
     echo "Performing full backup"
-    kolla-ansible -i ${RAW_INVENTORY} -vvv mariadb_backup --full
+    kolla-ansible mariadb-backup -i ${RAW_INVENTORY} -vvv --full
     # Sleep for 30 seconds, not because it's absolutely necessary.
     # The full backup is already completed at this point, as the
     # ansible job is waiting for the completion of the backup script
@@ -42,7 +42,7 @@ function test_backup {
     # data gets written within those 30 seconds.
     echo "Sleeping for 30 seconds"
     sleep 30
-    kolla-ansible -i ${RAW_INVENTORY} -vvv mariadb_backup --incremental
+    kolla-ansible mariadb-backup -i ${RAW_INVENTORY} -vvv --incremental
 }
 
 function test_backup_with_retries {

tests/upgrade-bifrost.sh

@@ -17,7 +17,7 @@ function upgrade_bifrost {
     # CI.
     # TODO(mgoddard): make some configuration file changes and trigger a real
     # upgrade.
-    kolla-ansible -i ${RAW_INVENTORY} -vvv deploy-bifrost &> /tmp/logs/ansible/upgrade-bifrost
+    kolla-ansible deploy-bifrost -i ${RAW_INVENTORY} -vvv &> /tmp/logs/ansible/upgrade-bifrost
 }
 
 

tests/upgrade.sh

@@ -12,14 +12,14 @@ function upgrade {
 
     source $KOLLA_ANSIBLE_VENV_PATH/bin/activate
 
-    kolla-ansible -i ${RAW_INVENTORY} -vvv certificates &> /tmp/logs/ansible/certificates
+    kolla-ansible certificates -i ${RAW_INVENTORY} -vvv &> /tmp/logs/ansible/certificates
-    kolla-ansible -i ${RAW_INVENTORY} -vvv prechecks &> /tmp/logs/ansible/upgrade-prechecks
+    kolla-ansible prechecks -i ${RAW_INVENTORY} -vvv &> /tmp/logs/ansible/upgrade-prechecks
-    kolla-ansible -i ${RAW_INVENTORY} -vvv pull &> /tmp/logs/ansible/pull-upgrade
+    kolla-ansible pull -i ${RAW_INVENTORY} -vvv &> /tmp/logs/ansible/pull-upgrade
-    kolla-ansible -i ${RAW_INVENTORY} -vvv upgrade &> /tmp/logs/ansible/upgrade
+    kolla-ansible upgrade -i ${RAW_INVENTORY} -vvv &> /tmp/logs/ansible/upgrade
 
-    kolla-ansible -i ${RAW_INVENTORY} -vvv post-deploy &> /tmp/logs/ansible/upgrade-post-deploy
+    kolla-ansible post-deploy -i ${RAW_INVENTORY} -vvv &> /tmp/logs/ansible/upgrade-post-deploy
 
-    kolla-ansible -i ${RAW_INVENTORY} -vvv validate-config &> /tmp/logs/ansible/validate-config
+    kolla-ansible validate-config -i ${RAW_INVENTORY} -vvv &> /tmp/logs/ansible/validate-config
 }
 
 

tools/kolla-ansible

@@ -1,608 +0,0 @@
-#!/usr/bin/env bash
-#
-# This script can be used to interact with kolla via ansible.
-
-set -o errexit
-
-# do not use _PYTHON_BIN directly, use $(get_python_bin) instead
-_PYTHON_BIN=""
-
-ANSIBLE_VERSION_MIN=2.16
-ANSIBLE_VERSION_MAX=2.17
-
-function get_python_bin {
-    if [ -n "$_PYTHON_BIN" ]; then
-      echo -n "$_PYTHON_BIN"
-      return
-    fi
-
-    local ansible_path
-    ansible_path=$(which ansible)
-
-    if [[ $? -ne 0 ]]; then
-        echo "ERROR: Ansible is not installed in the current (virtual) environment." >&2
-        echo "Ansible version should be between $ANSIBLE_VERSION_MIN and $ANSIBLE_VERSION_MAX." >&2
-        exit 1
-    fi
-
-    local ansible_shebang_line
-    ansible_shebang_line=$(head -n1 "$ansible_path")
-
-    if ! echo "$ansible_shebang_line" | egrep "^#!" &>/dev/null; then
-        echo "ERROR: Ansible script is malformed (missing shebang line)." >&2
-        exit 1
-    fi
-
-    # NOTE(yoctozepto): may have multiple parts
-    _PYTHON_BIN=${ansible_shebang_line#\#\!}
-    echo -n "$_PYTHON_BIN"
-}
-
-function check_environment_coherence {
-    local ansible_python_cmdline
-    ansible_python_cmdline=$(get_python_bin)
-    ansible_python_version=$($ansible_python_cmdline -c 'import sys; print(str(sys.version_info[0])+"."+str(sys.version_info[1]))')
-
-    if ! $ansible_python_cmdline --version &>/dev/null; then
-        echo "ERROR: Ansible Python is not functional." >&2
-        echo "Tried '$ansible_python_cmdline'" >&2
-        exit 1
-    fi
-
-    # Check for existence of kolla_ansible module using Ansible's Python.
-    if ! $ansible_python_cmdline -c 'import kolla_ansible' &>/dev/null; then
-        echo "ERROR: kolla_ansible has to be available in the Ansible PYTHONPATH." >&2
-        echo "Please install both in the same (virtual) environment." >&2
-        exit 1
-    fi
-
-    local ansible_full_version
-    ansible_full_version=$($ansible_python_cmdline -c 'import ansible; print(ansible.__version__)')
-
-    if [[ $? -ne 0 ]]; then
-        echo "ERROR: Failed to obtain Ansible version:" >&2
-        echo "$ansible_full_version" >&2
-        exit 1
-    fi
-
-    local ansible_version
-    ansible_version=$(echo "$ansible_full_version" | egrep -o '^[0-9]+\.[0-9]+')
-
-    if [[ $? -ne 0 ]]; then
-        echo "ERROR: Failed to parse Ansible version:" >&2
-        echo "$ansible_full_version" >&2
-        exit 1
-    fi
-
-
-    if [[ $(printf "%s\n" "$ANSIBLE_VERSION_MIN" "$ANSIBLE_VERSION_MAX" "$ansible_version" | sort -V | head -n1) != "$ANSIBLE_VERSION_MIN" ]] ||
-       [[ $(printf "%s\n" "$ANSIBLE_VERSION_MIN" "$ANSIBLE_VERSION_MAX" "$ansible_version" | sort -V | tail -n1) != "$ANSIBLE_VERSION_MAX" ]]; then
-        echo "ERROR: Ansible version should be between $ANSIBLE_VERSION_MIN and $ANSIBLE_VERSION_MAX. Current version is $ansible_full_version which is not supported."
-        exit 1
-    fi
-}
-
-function find_base_dir {
-    local dir_name
-    local python_dir
-    dir_name=$(dirname "$0")
-    # NOTE(yoctozepto): Fix the case where dir_name is a symlink and VIRTUAL_ENV might not be. This
-    # happens with pyenv-virtualenv, see https://bugs.launchpad.net/kolla-ansible/+bug/1903887
-    dir_name=$(readlink -e "$dir_name")
-    python_dir="python${ansible_python_version}"
-    if [ -z "$SNAP" ]; then
-        if [[ ${dir_name} == "/usr/bin" ]]; then
-            if test -f /usr/lib/${python_dir}/*-packages/kolla-ansible.egg-link; then
-                # Editable install.
-                BASEDIR="$(head -n1 /usr/lib/${python_dir}/*-packages/kolla-ansible.egg-link)"
-            else
-                BASEDIR=/usr/share/kolla-ansible
-            fi
-        elif [[ ${dir_name} == "/usr/local/bin" ]]; then
-            if test -f /usr/local/lib/${python_dir}/*-packages/kolla-ansible.egg-link; then
-                # Editable install.
-                BASEDIR="$(head -n1 /usr/local/lib/${python_dir}/*-packages/kolla-ansible.egg-link)"
-            else
-                BASEDIR=/usr/local/share/kolla-ansible
-            fi
-        elif [[ ${dir_name} == ~/.local/bin ]]; then
-            if test -f ~/.local/lib/${python_dir}/*-packages/kolla-ansible.egg-link; then
-                # Editable install.
-                BASEDIR="$(head -n1 ~/.local/lib/${python_dir}/*-packages/kolla-ansible.egg-link)"
-            else
-                BASEDIR=~/.local/share/kolla-ansible
-            fi
-        elif [[ -n ${VIRTUAL_ENV} ]] && [[ ${dir_name} == "$(readlink -e "${VIRTUAL_ENV}/bin")" ]]; then
-            if test -f ${VIRTUAL_ENV}/lib/${python_dir}/site-packages/kolla-ansible.egg-link; then
-                # Editable install.
-                BASEDIR="$(head -n1 ${VIRTUAL_ENV}/lib/${python_dir}/*-packages/kolla-ansible.egg-link)"
-            else
-                BASEDIR="${VIRTUAL_ENV}/share/kolla-ansible"
-            fi
-        else
-            # Running from sources (repo).
-            BASEDIR="$(dirname ${dir_name})"
-        fi
-    else
-        BASEDIR="$SNAP/share/kolla-ansible"
-    fi
-}
-
-function install_deps {
-    echo "Installing Ansible Galaxy dependencies"
-    if pip show ansible 2>/dev/null; then
-        ansible-galaxy collection install -r ${BASEDIR}/requirements.yml --force
-    else
-        ansible-galaxy collection install -r ${BASEDIR}/requirements.yml --force
-        ansible-galaxy collection install -r ${BASEDIR}/requirements-core.yml --force
-    fi
-
-    if [[ $? -ne 0 ]]; then
-        echo "ERROR: Failed to install Ansible Galaxy dependencies" >&2
-        exit 1
-    fi
-}
-
-function process_cmd {
-    echo "$ACTION : $CMD"
-    $CMD
-    if [[ $? -ne 0 ]]; then
-        echo "Command failed $CMD"
-        exit 1
-    fi
-}
-
-function usage {
-    cat <<EOF
-Usage: $0 COMMAND [options]
-
-Options:
-    --inventory, -i <inventory_path>   Specify path to ansible inventory file. \
-Can be specified multiple times to pass multiple inventories.
-    --playbook, -p <playbook_path>     Specify path to ansible playbook file
-    --configdir <config_path>          Specify path to directory with globals.yml
-    --key -k <key_path>                Specify path to ansible vault keyfile
-    --help, -h                         Show this usage information
-    --tags, -t <tags>                  Only run plays and tasks tagged with these values
-    --skip-tags <tags>                 Only run plays and tasks whose tags do not match these values
-    --extra, -e <ansible variables>    Set additional variables as key=value or YAML/JSON passed to ansible-playbook
-    --passwords <passwords_path>       Specify path to the passwords file
-    --limit <host>                     Specify host to run plays
-    --forks <forks>                    Number of forks to run Ansible with
-    --vault-id <@prompt or path>       Specify @prompt or password file (Ansible >=  2.4)
-    --ask-vault-pass                   Ask for vault password
-    --vault-password-file <path>       Specify password file for vault decrypt
-    --check, -C                        Don't make any changes and try to predict some of the changes that may occur instead
-    --diff, -D                         Show differences in ansible-playbook changed tasks
-    --verbose, -v                      Increase verbosity of ansible-playbook
-    --version                          Show version
-
-Environment variables:
-    EXTRA_OPTS                         Additional arguments to pass to ansible-playbook
-
-Commands:
-    install-deps         Install Ansible Galaxy dependencies
-    prechecks            Do pre-deployment checks for hosts
-    mariadb_recovery     Recover a completely stopped mariadb cluster
-    mariadb_backup       Take a backup of MariaDB databases
-                             --full (default)
-                             --incremental
-    bootstrap-servers    Bootstrap servers with kolla deploy dependencies
-    destroy              Destroy Kolla containers, volumes and host configuration
-                             --include-images to also destroy Kolla images
-                             --include-dev to also destroy dev mode repos
-    deploy               Deploy and start all kolla containers
-    deploy-bifrost       Deploy and start bifrost container
-    deploy-servers       Enroll and deploy servers with bifrost
-    deploy-containers    Only deploy and start containers (no config updates or bootstrapping)
-    gather-facts         Gather Ansible facts
-    post-deploy          Do post deploy on deploy node
-    pull                 Pull all images for containers (only pulls, no running container changes)
-    rabbitmq-reset-state Force reset the state of RabbitMQ
-    rabbitmq-upgrade <version> Upgrade to a specific version of RabbitMQ
-    reconfigure          Reconfigure OpenStack service
-    stop                 Stop Kolla containers
-    certificates         Generate self-signed certificate for TLS *For Development Only*
-    octavia-certificates Generate certificates for octavia deployment
-                             --check-expiry <days> to check if certificates expire within that many days
-    upgrade              Upgrades existing OpenStack Environment
-    upgrade-bifrost      Upgrades an existing bifrost container
-    genconfig            Generate configuration files for enabled OpenStack services
-    validate-config      Validate configuration files for enabled OpenStack services
-    prune-images         Prune orphaned Kolla images
-    nova-libvirt-cleanup Clean up disabled nova_libvirt containers
-EOF
-}
-
-function bash_completion {
-cat <<EOF
---inventory -i
---playbook -p
---configdir
---key -k
---help -h
---skip-tags
---tags -t
---extra -e
---passwords
---limit
---forks
---vault-id
---ask-vault-pass
---vault-password-file
---check -C
---diff -D
---verbose -v
---version
-install-deps
-prechecks
-mariadb_recovery
-mariadb_backup
-bootstrap-servers
-destroy
-deploy
-deploy-bifrost
-deploy-containers
-deploy-servers
-gather-facts
-post-deploy
-pull
-rabbitmq-reset-state
-rabbitmq-upgrade
-reconfigure
-stop
-certificates
-octavia-certificates
-upgrade
-upgrade-bifrost
-genconfig
-validate-config
-prune-images
-nova-libvirt-cleanup
-EOF
-}
-
-function version {
-    local python_bin
-    python_bin=$(get_python_bin)
-
-    $python_bin -c 'from kolla_ansible.version import version_info; print(version_info)'
-}
-
-check_environment_coherence
-
-SHORT_OPTS="hi:p:t:k:e:CD:v"
-LONG_OPTS="help,version,inventory:,playbook:,skip-tags:,tags:,key:,extra:,check,diff,verbose,configdir:,passwords:,limit:,forks:,vault-id:,ask-vault-pass,vault-password-file:,yes-i-really-really-mean-it,include-images,include-dev:,full,incremental,check-expiry:"
-
-RAW_ARGS="$*"
-ARGS=$(getopt -o "${SHORT_OPTS}" -l "${LONG_OPTS}" --name "$0" -- "$@") || { usage >&2; exit 2; }
-
-eval set -- "$ARGS"
-
-find_base_dir
-
-INVENTORY="${BASEDIR}/ansible/inventory/all-in-one"
-PLAYBOOK="${BASEDIR}/ansible/site.yml"
-VERBOSITY=
-EXTRA_OPTS=${EXTRA_OPTS}
-CONFIG_DIR="/etc/kolla"
-DANGER_CONFIRM=
-INCLUDE_IMAGES=
-INCLUDE_DEV=
-BACKUP_TYPE="full"
-OCTAVIA_CERTS_EXPIRY=
-# Serial is not recommended and disabled by default. Users can enable it by
-# configuring ANSIBLE_SERIAL variable.
-ANSIBLE_SERIAL=${ANSIBLE_SERIAL:-0}
-INVENTORIES=()
-
-while [ "$#" -gt 0 ]; do
-    case "$1" in
-
-    (--inventory|-i)
-            INVENTORIES+=("$2")
-            shift 2
-            ;;
-
-    (--playbook|-p)
-            PLAYBOOK="$2"
-            shift 2
-            ;;
-
-    (--skip-tags)
-            EXTRA_OPTS="$EXTRA_OPTS --skip-tags $2"
-            shift 2
-            ;;
-
-    (--tags|-t)
-            EXTRA_OPTS="$EXTRA_OPTS --tags $2"
-            shift 2
-            ;;
-
-    (--check|-C)
-            EXTRA_OPTS="$EXTRA_OPTS --check"
-            shift 1
-            ;;
-
-    (--diff|-D)
-            EXTRA_OPTS="$EXTRA_OPTS --diff"
-            shift 1
-            ;;
-
-    (--verbose|-v)
-            VERBOSITY="$VERBOSITY --verbose"
-            shift 1
-            ;;
-
-    (--configdir)
-            CONFIG_DIR="$2"
-            shift 2
-            ;;
-
-    (--yes-i-really-really-mean-it)
-            if [[ ${RAW_ARGS} =~ "$1" ]]
-            then
-                DANGER_CONFIRM="$1"
-            fi
-            shift 1
-            ;;
-
-    (--include-images)
-            INCLUDE_IMAGES="$1"
-            shift 1
-            ;;
-
-    (--include-dev)
-            INCLUDE_DEV="$1"
-            shift 1
-            ;;
-
-    (--key|-k)
-            VAULT_PASS_FILE="$2"
-            EXTRA_OPTS="$EXTRA_OPTS --vault-password-file=$VAULT_PASS_FILE"
-            shift 2
-            ;;
-
-    (--extra|-e)
-            EXTRA_OPTS="$EXTRA_OPTS -e $2"
-            shift 2
-            ;;
-
-    (--passwords)
-            PASSWORDS_FILE="$2"
-            shift 2
-            ;;
-
-    (--limit)
-            EXTRA_OPTS="$EXTRA_OPTS --limit $2"
-            shift 2
-            ;;
-
-    (--forks)
-            EXTRA_OPTS="$EXTRA_OPTS --forks $2"
-            shift 2
-            ;;
-
-    (--vault-id)
-            EXTRA_OPTS="$EXTRA_OPTS --vault-id $2"
-            shift 2
-            ;;
-
-    (--ask-vault-pass)
-            VERBOSITY="$EXTRA_OPTS --ask-vault-pass"
-            shift 1
-            ;;
-
-    (--vault-password-file)
-            EXTRA_OPTS="$EXTRA_OPTS --vault-password-file $2"
-            shift 2
-            ;;
-
-    (--full)
-            BACKUP_TYPE="full"
-            shift 1
-            ;;
-
-    (--incremental)
-            BACKUP_TYPE="incremental"
-            shift 1
-            ;;
-
-    (--check-expiry)
-            OCTAVIA_CERTS_EXPIRY="$2"
-            shift 2
-            ;;
-
-    (--version)
-            version
-            exit 0
-            ;;
-
-    (--help|-h)
-            usage
-            exit 0
-            ;;
-
-    (--)
-            shift
-            break
-            ;;
-
-    (*)
-            echo "error"
-            exit 3
-            ;;
-esac
-done
-
-case "$1" in
-
-(install-deps)
-        install_deps
-        exit 0
-        ;;
-(prechecks)
-        ACTION="Pre-deployment checking"
-        EXTRA_OPTS="$EXTRA_OPTS -e kolla_action=precheck"
-        ;;
-(mariadb_recovery)
-        ACTION="Attempting to restart mariadb cluster"
-        EXTRA_OPTS="$EXTRA_OPTS -e kolla_action=deploy"
-        PLAYBOOK="${BASEDIR}/ansible/mariadb_recovery.yml"
-        ;;
-(mariadb_backup)
-        ACTION="Backup MariaDB databases"
-        EXTRA_OPTS="$EXTRA_OPTS -e kolla_action=backup -e mariadb_backup_type=${BACKUP_TYPE}"
-        PLAYBOOK="${BASEDIR}/ansible/mariadb_backup.yml"
-        ;;
-(destroy)
-        ACTION="Destroy Kolla containers, volumes and host configuration"
-        PLAYBOOK="${BASEDIR}/ansible/destroy.yml"
-
-        INVENTORIES_COMMA_SEPARATED=""
-        for INVENTORY in ${INVENTORIES[@]}; do
-            INVENTORIES_COMMA_SEPARATED="${INVENTORIES_COMMA_SEPARATED},${INVENTORY}"
-        done
-        INVENTORIES_COMMA_SEPARATED=$(echo "${INVENTORIES_COMMA_SEPARATED}" | sed -e 's/^,//g')
-
-        if [[ "${INCLUDE_IMAGES}" == "--include-images" ]]; then
-            EXTRA_OPTS="$EXTRA_OPTS -e destroy_include_images=yes"
-        fi
-        if [[ "${INCLUDE_DEV}" == "--include-dev" ]]; then
-            EXTRA_OPTS="$EXTRA_OPTS -e destroy_include_dev=yes"
-        fi
-        EXTRA_OPTS="$EXTRA_OPTS -e inventories_comma_separated=${INVENTORIES_COMMA_SEPARATED}"
-        if [[ "${DANGER_CONFIRM}" != "--yes-i-really-really-mean-it" ]]; then
-            cat << EOF
-WARNING:
-    This will PERMANENTLY DESTROY all deployed kolla containers, volumes and host configuration.
-    There is no way to recover from this action. To confirm, please add the following option:
-    --yes-i-really-really-mean-it
-EOF
-            exit 1
-        fi
-        ;;
-(bootstrap-servers)
-        ACTION="Bootstrapping servers"
-        PLAYBOOK="${BASEDIR}/ansible/kolla-host.yml"
-        EXTRA_OPTS="$EXTRA_OPTS -e kolla_action=bootstrap-servers"
-        ;;
-(deploy)
-        ACTION="Deploying Playbooks"
-        EXTRA_OPTS="$EXTRA_OPTS -e kolla_action=deploy"
-        ;;
-(deploy-bifrost)
-        ACTION="Deploying Bifrost"
-        PLAYBOOK="${BASEDIR}/ansible/bifrost.yml"
-        EXTRA_OPTS="$EXTRA_OPTS -e kolla_action=deploy"
-        ;;
-(deploy-containers)
-        ACTION="Deploying Containers"
-        EXTRA_OPTS="$EXTRA_OPTS -e kolla_action=deploy-containers"
-        ;;
-(deploy-servers)
-        ACTION="Deploying servers with bifrost"
-        PLAYBOOK="${BASEDIR}/ansible/bifrost.yml"
-        EXTRA_OPTS="$EXTRA_OPTS -e kolla_action=deploy-servers"
-        ;;
-(gather-facts)
-        ACTION="Gathering Ansible facts"
-        PLAYBOOK="${BASEDIR}/ansible/gather-facts.yml"
-        ;;
-(post-deploy)
-        ACTION="Post-Deploying Playbooks"
-        PLAYBOOK="${BASEDIR}/ansible/post-deploy.yml"
-        ;;
-(pull)
-        ACTION="Pulling Docker images"
-        EXTRA_OPTS="$EXTRA_OPTS -e kolla_action=pull"
-        ;;
-(upgrade)
-        ACTION="Upgrading OpenStack Environment"
-        EXTRA_OPTS="$EXTRA_OPTS -e kolla_action=upgrade -e kolla_serial=${ANSIBLE_SERIAL}"
-        ;;
-(upgrade-bifrost)
-        ACTION="Upgrading Bifrost"
-        PLAYBOOK="${BASEDIR}/ansible/bifrost.yml"
-        EXTRA_OPTS="$EXTRA_OPTS -e kolla_action=upgrade"
-        ;;
-(reconfigure)
-        ACTION="Reconfigure OpenStack service"
-        EXTRA_OPTS="$EXTRA_OPTS -e kolla_action=reconfigure -e kolla_serial=${ANSIBLE_SERIAL}"
-        ;;
-(stop)
-        ACTION="Stop Kolla containers"
-        EXTRA_OPTS="$EXTRA_OPTS -e kolla_action=stop"
-        if [[ "${DANGER_CONFIRM}" != "--yes-i-really-really-mean-it" ]]; then
-            cat << EOF
-WARNING:
-    This will stop all deployed kolla containers, limit with tags is possible and also with
-    skip_stop_containers variable. To confirm, please add the following option:
-    --yes-i-really-really-mean-it
-EOF
-            exit 1
-        fi
-        ;;
-(certificates)
-        ACTION="Generate TLS Certificates"
-        PLAYBOOK="${BASEDIR}/ansible/certificates.yml"
-        ;;
-(octavia-certificates)
-        ACTION="Generate octavia Certificates"
-        PLAYBOOK="${BASEDIR}/ansible/octavia-certificates.yml"
-        if [[ ! -z "${OCTAVIA_CERTS_EXPIRY}" ]]; then
-            EXTRA_OPTS="$EXTRA_OPTS -e octavia_certs_check_expiry=yes -e octavia_certs_expiry_limit=${OCTAVIA_CERTS_EXPIRY}"
-        fi
-        ;;
-(genconfig)
-        ACTION="Generate configuration files for enabled OpenStack services"
-        EXTRA_OPTS="$EXTRA_OPTS -e kolla_action=config"
-        ;;
-(validate-config)
-        ACTION="Validate configuration files for enabled OpenStack services"
-        EXTRA_OPTS="$EXTRA_OPTS -e kolla_action=config_validate"
-        ;;
-(prune-images)
-        ACTION="Prune orphaned Kolla images"
-        PLAYBOOK="${BASEDIR}/ansible/prune-images.yml"
-        if [[ "${DANGER_CONFIRM}" != "--yes-i-really-really-mean-it" ]]; then
-            cat << EOF
-WARNING:
-    This will PERMANENTLY DELETE all orphaned kolla images. To confirm, please add the following option:
-    --yes-i-really-really-mean-it
-EOF
-            exit 1
-        fi
-        ;;
-(nova-libvirt-cleanup)
-        ACTION="Cleanup disabled nova_libvirt containers"
-        PLAYBOOK="${BASEDIR}/ansible/nova-libvirt-cleanup.yml"
-        ;;
-(rabbitmq-reset-state)
-        ACTION="Force reset the state of RabbitMQ"
-        PLAYBOOK="${BASEDIR}/ansible/rabbitmq-reset-state.yml"
-        ;;
-(rabbitmq-upgrade)
-        RMQ_VERSION="$2"
-        ACTION="Upgrade to a specific version of RabbitMQ"
-        PLAYBOOK="${BASEDIR}/ansible/rabbitmq-upgrade.yml"
-        EXTRA_OPTS="$EXTRA_OPTS -e rabbitmq_version_suffix=${RMQ_VERSION}"
-        ;;
-(bash-completion)
-        bash_completion
-        exit 0
-        ;;
-(*)     usage
-        exit 3
-        ;;
-esac
-
-GLOBALS_DIR="${CONFIG_DIR}/globals.d"
-EXTRA_GLOBALS=$([ -d "${GLOBALS_DIR}" ] && find ${GLOBALS_DIR} -maxdepth 1 -type f -name '*.yml' -printf ' -e @%p' || true 2>/dev/null)
-PASSWORDS_FILE="${PASSWORDS_FILE:-${CONFIG_DIR}/passwords.yml}"
-CONFIG_OPTS="-e @${CONFIG_DIR}/globals.yml ${EXTRA_GLOBALS} -e @${PASSWORDS_FILE} -e CONFIG_DIR=${CONFIG_DIR}"
-CMD="ansible-playbook $CONFIG_OPTS $EXTRA_OPTS $PLAYBOOK $VERBOSITY"
-for INVENTORY in ${INVENTORIES[@]}; do
-    CMD="${CMD} --inventory $INVENTORY"
-done
-process_cmd