Add support of podman deployment

This change adds basic deployment based on Podman
container manager as an alternative to Docker.

Signed-off-by: Ivan Halomi <i.halomi@partner.samsung.com>
Signed-off-by: Martin Hiner <m.hiner@partner.samsung.com>
Signed-off-by: Petr Tuma <p.tuma@partner.samsung.com>
Change-Id: I2b52964906ba8b19b8b1098717b9423ab954fa3d
Depends-On: Ie4b4c1cf8fe6e7ce41eaa703b423dedcb41e3afc

ansible/group_vars/all.yml

@@ -159,6 +159,33 @@ docker_common_options:
   restart_retries: "{{ docker_restart_policy_retry }}"
   graceful_timeout: "{{ docker_graceful_timeout }}"
   client_timeout: "{{ docker_client_timeout }}"
+  container_engine: "{{ kolla_container_engine }}"
+
+# Container engine specific volume paths
+docker_volumes_path: "{{ docker_runtime_directory or '/var/lib/docker' }}/volumes"
+podman_volumes_path: "{{ docker_runtime_directory or '/var/lib/containers' }}/storage/volumes"
+container_engine_volumes_path: "{{ docker_volumes_path if kolla_container_engine == 'docker' else podman_volumes_path }}"
+
+#####################
+# Volumes under /run
+#####################
+# Podman has problem with mounting whole /run directory
+# described here: https://github.com/containers/podman/issues/16305
+run_default_volumes_podman:
+  - '/run/netns:/run/netns:shared'
+  - '/run/lock/nova:/run/lock/nova:shared'
+  - "/run/libvirt:/run/libvirt:shared"
+  - "/run/nova:/run/nova:shared"
+  - "/run/openvswitch:/run/openvswitch:shared"
+
+run_default_volumes_docker: []
+
+run_default_subdirectories:
+  - '/run/netns'
+  - '/run/lock/nova'
+  - "/run/libvirt"
+  - "/run/nova"
+  - "/run/openvswitch"
 
 ####################
 # Dimensions options
@@ -167,11 +194,20 @@ docker_common_options:
 # NOTE(mnasiadka): Lower 1073741816 nofile limit on EL9 (RHEL9/CentOS Stream 9/Rocky Linux 9)
 #                  fixes at least rabbitmq and mariadb
 default_container_dimensions: "{{ default_container_dimensions_el9 if ansible_facts.os_family == 'RedHat' else '{}' }}"
-default_container_dimensions_el9:
+default_container_dimensions_el9: "{{ default_docker_dimensions_el9 if kolla_container_engine == 'docker' else default_podman_dimensions_el9 }}"
+default_docker_dimensions_el9:
   ulimits:
     nofile:
       soft: 1048576
       hard: 1048576
+default_podman_dimensions_el9:
+  ulimits:
+    RLIMIT_NOFILE:
+      soft: 1048576
+      hard: 1048576
+    RLIMIT_NPROC:
+      soft: 1048576
+      hard: 1048576
 
 #####################
 # Healthcheck options

ansible/library/kolla_container_facts.py

@@ -13,10 +13,9 @@
 # limitations under the License.
 
 
-import docker
-
 from ansible.module_utils.basic import AnsibleModule
 
+
 DOCUMENTATION = '''
 ---
 module: kolla_container_facts
@@ -41,6 +40,11 @@ options:
       - Name or names of the containers
     required: False
     type: str or list
+  container_engine:
+    description:
+      - Name of container engine to use
+    required: True
+    type: str
 author: Jeffrey Zhang
 '''
 
@@ -49,6 +53,7 @@ EXAMPLES = '''
   tasks:
     - name: Gather docker facts
       kolla_container_facts:
+        container_engine: docker
 
     - name: Gather glance container facts
       kolla_container_facts:
@@ -56,24 +61,18 @@ EXAMPLES = '''
         name:
           - glance_api
           - glance_registry
+        container_engine: podman
 '''
 
 
 def get_docker_client():
+    import docker
     return docker.APIClient
 
 
-def main():
+def use_docker(module, results):
-    argument_spec = dict(
-        name=dict(required=False, type='list', default=[]),
-        api_version=dict(required=False, type='str', default='auto'),
-        container_engine=dict(required=True, type='str')
-    )
-
-    module = AnsibleModule(argument_spec=argument_spec)
-
-    results = dict(changed=False, _containers=[])
     client = get_docker_client()(version=module.params.get('api_version'))
+
     containers = client.containers()
     names = module.params.get('name')
     if names and not isinstance(names, list):
@@ -86,6 +85,46 @@ def main():
                 continue
             results['_containers'].append(container)
             results[container_name] = container
+
+
+def use_podman(module, results):
+    import podman.errors as pe
+    from podman import PodmanClient
+
+    client = PodmanClient(base_url="http+unix:/run/podman/podman.sock")
+
+    try:
+        containers = client.containers.list(all=True, ignore_removed=True)
+    except pe.APIError as e:
+        module.fail_json(failed=True, msg=f"Internal error: {e.explanation}")
+    names = module.params.get('name')
+    if names and not isinstance(names, list):
+        names = [names]
+
+    for container in containers:
+        container.reload()
+        container_name = container.attrs['Name']
+        if container_name not in names:
+            continue
+        results['_containers'].append(container.attrs)
+        results[container_name] = container.attrs
+
+
+def main():
+    argument_spec = dict(
+        name=dict(required=False, type='list', default=[]),
+        api_version=dict(required=False, type='str', default='auto'),
+        container_engine=dict(required=True, type='str')
+    )
+
+    module = AnsibleModule(argument_spec=argument_spec)
+
+    results = dict(changed=False, _containers=[])
+    if module.params['container_engine'] == 'podman':
+        use_podman(module, results)
+    else:
+        use_docker(module, results)
+
     module.exit_json(**results)
 
 

ansible/library/kolla_container_volume_facts.py

@@ -13,16 +13,14 @@
 # limitations under the License.
 
 
-import docker
-
 from ansible.module_utils.basic import AnsibleModule
 
 DOCUMENTATION = '''
 ---
 module: kolla_container_volume_facts
-short_description: Module for collecting Docker container volume facts
+short_description: Module for collecting container volume facts
 description:
-  - A module targeted at collecting Docker container volume facts. It is used
+  - A module targeted at collecting container volume facts. It is used
     for detecting whether the container volume exists on a host.
 options:
   container_engine:
@@ -60,9 +58,25 @@ EXAMPLES = '''
 
 
 def get_docker_client():
+    import docker
     return docker.APIClient
 
 
+def get_docker_volumes(api_version):
+    client = get_docker_client()(version=api_version)
+    return client.volumes()['Volumes']
+
+
+def get_podman_volumes():
+    from podman import PodmanClient
+
+    client = PodmanClient(base_url="http+unix:/run/podman/podman.sock")
+    volumes = []
+    for volume in client.volumes.list():
+        volumes.append(volume.attrs)
+    return volumes
+
+
 def main():
     argument_spec = dict(
         name=dict(required=False, type='list', default=[]),
@@ -73,12 +87,15 @@ def main():
     module = AnsibleModule(argument_spec=argument_spec)
 
     results = dict(changed=False, _volumes=[])
-    client = get_docker_client()(version=module.params.get('api_version'))
+    if module.params.get('container_engine') == 'docker':
-    volumes = client.volumes()
+        volumes = get_docker_volumes(module.params.get('api_version'))
+    else:
+        volumes = get_podman_volumes()
+
     names = module.params.get('name')
     if names and not isinstance(names, list):
         names = [names]
-    for volume in volumes['Volumes']:
+    for volume in volumes:
         volume_name = volume['Name']
         if names and volume_name not in names:
             continue

ansible/library/kolla_docker.py

@@ -16,22 +16,25 @@
 # a hacky way to seed most usages of kolla_docker in kolla-ansible ansible
 # playbooks - caution has to be exerted when setting "common_options"
 
-import traceback
+# FIXME(yoctozepto): restart_policy is *not* checked in the container
 
 from ansible.module_utils.basic import AnsibleModule
-from ansible.module_utils.kolla_docker_worker import DockerWorker
+import traceback
+
+from ansible.module_utils.kolla_container_worker import ContainerWorker
+
 
 DOCUMENTATION = '''
 ---
 module: kolla_docker
-short_description: Module for controlling Docker
+short_description: Module for controlling containers
 description:
-     - A module targeting at controlling Docker as used by Kolla.
+     - A module targeting at controlling container engine as used by Kolla.
 options:
   common_options:
     description:
       - A dict containing common params such as login info
-    required: False
+    required: True
     type: dict
     default: dict()
   action:
@@ -397,17 +400,23 @@ def generate_module():
 def main():
     module = generate_module()
 
-    dw = None
+    cw: ContainerWorker = None
     try:
-        dw = DockerWorker(module)
+        if module.params.get('container_engine') == 'docker':
+            from ansible.module_utils.kolla_docker_worker import DockerWorker
+            cw = DockerWorker(module)
+        else:
+            from ansible.module_utils.kolla_podman_worker import PodmanWorker
+            cw = PodmanWorker(module)
+
         # TODO(inc0): We keep it bool to have ansible deal with consistent
         # types. If we ever add method that will have to return some
         # meaningful data, we need to refactor all methods to return dicts.
-        result = bool(getattr(dw, module.params.get('action'))())
+        result = bool(getattr(cw, module.params.get('action'))())
-        module.exit_json(changed=dw.changed, result=result, **dw.result)
+        module.exit_json(changed=cw.changed, result=result, **cw.result)
     except Exception:
         module.fail_json(changed=True, msg=repr(traceback.format_exc()),
-                         **getattr(dw, 'result', {}))
+                         **getattr(cw, 'result', {}))
 
 
 if __name__ == '__main__':

ansible/library/kolla_toolbox.py

@@ -13,7 +13,6 @@
 # limitations under the License.
 
 from distutils.version import StrictVersion
-import docker
 import json
 import re
 
@@ -21,6 +20,7 @@ from ansible.module_utils.ansible_release import __version__ as ansible_version
 from ansible.module_utils.basic import AnsibleModule
 
 from ast import literal_eval
+from shlex import split
 
 DOCUMENTATION = '''
 ---
@@ -120,10 +120,19 @@ def gen_commandline(params):
         if StrictVersion(ansible_version) < StrictVersion('2.11.0'):
             module_args = params.get('module_args')
         else:
-            module_args = literal_eval(params.get('module_args'))
+            try:
+                module_args = literal_eval(params.get('module_args'))
+            except SyntaxError:
+                if not isinstance(params.get('module_args'), str):
+                    raise
+
+                # account for string arguments
+                module_args = split(params.get('module_args'))
         if isinstance(module_args, dict):
             module_args = ' '.join("{}='{}'".format(key, value)
                                    for key, value in module_args.items())
+        if isinstance(module_args, list):
+            module_args = ' '.join(module_args)
         command.extend(['-a', module_args])
     if params.get('module_extra_vars'):
         extra_vars = params.get('module_extra_vars')
@@ -134,6 +143,7 @@ def gen_commandline(params):
 
 
 def get_docker_client():
+    import docker
     return docker.APIClient
 
 
@@ -142,17 +152,7 @@ def docker_supports_environment_in_exec(client):
     return docker_version >= StrictVersion('1.25')
 
 
-def main():
+def use_docker(module):
-    specs = dict(
-        container_engine=dict(required=True, type='str'),
-        module_name=dict(required=True, type='str'),
-        module_args=dict(type='str'),
-        module_extra_vars=dict(type='json'),
-        api_version=dict(required=False, type='str', default='auto'),
-        timeout=dict(required=False, type='int', default=180),
-        user=dict(required=False, type='str'),
-    )
-    module = AnsibleModule(argument_spec=specs, bypass_checks=True)
     client = get_docker_client()(
         version=module.params.get('api_version'),
         timeout=module.params.get('timeout'))
@@ -240,7 +240,83 @@ def main():
                 # No way to know whether changed - assume yes.
                 ret['changed'] = True
 
-    module.exit_json(**ret)
+    return ret
+
+
+def get_kolla_toolbox():
+    from podman import PodmanClient
+
+    with PodmanClient(base_url="http+unix:/run/podman/podman.sock") as client:
+        for cont in client.containers.list(all=True):
+            cont.reload()
+            if cont.name == 'kolla_toolbox' and cont.status == 'running':
+                return cont
+
+
+def use_podman(module):
+    from podman.errors.exceptions import APIError
+
+    try:
+        kolla_toolbox = get_kolla_toolbox()
+        if not kolla_toolbox:
+            module.fail_json(msg='kolla_toolbox container is not running.')
+
+        kwargs = {}
+        if 'user' in module.params:
+            kwargs['user'] = module.params['user']
+        environment = {"ANSIBLE_STDOUT_CALLBACK": "json",
+                       "ANSIBLE_LOAD_CALLBACK_PLUGINS": "True"}
+        command_line = gen_commandline(module.params)
+
+        _, raw_output = kolla_toolbox.exec_run(
+            command_line,
+            environment=environment,
+            tty=True,
+            **kwargs
+        )
+    except APIError as e:
+        module.fail_json(msg=f'Encountered Podman API error: {e.explanation}')
+
+    try:
+        json_output = raw_output.decode('utf-8')
+        output = json.loads(json_output)
+    except Exception:
+        module.fail_json(
+            msg='Can not parse the inner module output: %s' % json_output)
+
+    try:
+        ret = output['plays'][0]['tasks'][0]['hosts']['localhost']
+    except (KeyError, IndexError):
+        module.fail_json(
+            msg='Ansible JSON output has unexpected format: %s' % output)
+
+    # Remove Ansible's internal variables from returned fields.
+    ret.pop('_ansible_no_log', None)
+
+    return ret
+
+
+def main():
+    specs = dict(
+        container_engine=dict(required=True, type='str'),
+        module_name=dict(required=True, type='str'),
+        module_args=dict(type='str'),
+        module_extra_vars=dict(type='json'),
+        api_version=dict(required=False, type='str', default='auto'),
+        timeout=dict(required=False, type='int', default=180),
+        user=dict(required=False, type='str'),
+    )
+    module = AnsibleModule(argument_spec=specs, bypass_checks=True)
+
+    container_engine = module.params.get('container_engine').lower()
+    if container_engine == 'docker':
+        result = use_docker(module)
+    elif container_engine == 'podman':
+        result = use_podman(module)
+    else:
+        module.fail_json(msg='Missing or invalid container engine.')
+
+    module.exit_json(**result)
 
 
 if __name__ == "__main__":

ansible/module_utils/kolla_docker_worker.py

@@ -12,8 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# FIXME(yoctozepto): restart_policy is *not* checked in the container
-
 import docker
 import json
 import os

ansible/module_utils/kolla_podman_worker.py

@@ -0,0 +1,674 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from podman.errors import APIError
+from podman import PodmanClient
+
+import shlex
+
+from ansible.module_utils.kolla_container_worker import COMPARE_CONFIG_CMD
+from ansible.module_utils.kolla_container_worker import ContainerWorker
+
+uri = "http+unix:/run/podman/podman.sock"
+
+CONTAINER_PARAMS = [
+    'name',             # string
+    'cap_add',          # list
+    'cgroupns',         # 'str',choices=['private', 'host']
+    'command',          # arrray of strings  -- docker string
+
+    # this part is hidden inside dimensions
+    'cpu_period',       # int
+    'cpu_quota',        # int
+    'cpuset_cpus',      # str
+    'cpu_shares'        # int
+    'cpuset_mems',      # str
+    'kernel_memory',    # int or string
+    'mem_limit',        # (Union[int, str])
+    'mem_reservation',  # (Union[int, str]): Memory soft limit.
+    'memswap_limit'     # (Union[int, str]): Maximum amount of memory
+                        # + swap a container is allowed to consume.
+    'ulimits',          # List[Ulimit]
+    'blkio_weight',     # int between 10 and 1000
+
+
+    'detach',           # bool
+    'entrypoint',       # string
+    'environment',      # dict docker - environment - dictionary
+    'healthcheck',      # same schema as docker -- healthcheck
+    'image',            # string
+    'ipc_mode',         # string only option is host
+
+    'labels',           # dict
+    'netns',            # dict # TODO(i.halomi) - not sure how it works
+    'network_options',  # string - none,bridge,host,container:id,
+                        # missing in docker but needs to be host
+    'pid_mode',         # "string"  host, private or ''
+    'privileged',       # bool
+    'restart_policy',   # set to none, handled by systemd
+    'remove',           # bool
+    'restart_tries',    # int doesnt matter done by systemd
+    'stop_timeout',     # int
+    'tty'               # bool
+    # VOLUMES NOT WORKING HAS TO BE DONE WITH MOUNTS
+    'volumes',          # array of dict
+    'volumes_from',     # array of strings
+]
+
+
+class PodmanWorker(ContainerWorker):
+
+    def __init__(self, module) -> None:
+        super().__init__(module)
+
+        self.pc = PodmanClient(base_url=uri)
+
+    def prepare_container_args(self):
+        args = dict(
+            network_mode='host'
+        )
+
+        command = self.params.pop('command', '')
+        self.params['command'] = shlex.split(command)
+
+        #  we have to transform volumes into mounts because podman-py
+        #  functionality is broken
+        mounts = []
+        filtered_volumes = {}
+        volumes = self.params.get('volumes', [])
+        if volumes:
+            self.parse_volumes(volumes, mounts, filtered_volumes)
+            # we can delete original volumes so it won't raise error later
+            self.params.pop('volumes', None)
+
+        args['mounts'] = mounts
+        args['volumes'] = filtered_volumes
+
+        # in case value is not string it has to be converted
+        environment = self.params.get('environment')
+        if environment:
+            for key, value in environment.items():
+                environment[key] = str(value)
+
+        healthcheck = self.params.get('healthcheck')
+        if healthcheck:
+            healthcheck = self.parse_healthcheck(healthcheck)
+            self.params.pop('healthcheck', None)
+            args.update(healthcheck)
+
+        # getting dimensions into separate parameters
+        dimensions = self.params.get('dimensions')
+        if dimensions:
+            dimensions = self.parse_dimensions(dimensions)
+            args.update(dimensions)
+
+        # NOTE(m.hiner): currently unsupported by Podman API
+        # args['tmpfs'] = self.generate_tmpfs()
+        self.params.pop('tmpfs', None)
+
+        # NOTE(m.hiner): in case containers are not privileged,
+        # they need this capability
+        if not self.params.get('privileged', False):
+            args['cap_add'] = self.params.pop('cap_add', []) + ['AUDIT_WRITE']
+
+        # maybe can be done straight away,
+        # at first it was around 6 keys that's why it is this way
+        convert_keys = dict(
+            graceful_timeout='stop_timeout',
+            cgroupns_mode='cgroupns'
+        )
+
+        # remap differing args
+        for key_orig, key_new in convert_keys.items():
+            if key_orig in self.params:
+                value = self.params.get(key_orig, None)
+
+                if value is not None:
+                    args[key_new] = value
+
+        # record remaining args
+        for key, value in self.params.items():
+            if key in CONTAINER_PARAMS and value is not None:
+                args[key] = value
+
+        args.pop('restart_policy', None)    # handled by systemd
+
+        return args
+
+    def parse_volumes(self, volumes, mounts, filtered_volumes):
+        # we can ignore empty strings
+        volumes = [item for item in volumes if item.strip()]
+
+        for item in volumes:
+            # if it starts with / it is bind not volume
+            if item[0] == '/':
+                mode = None
+                try:
+                    if item.count(':') == 2:
+                        src, dest, mode = item.split(':')
+                    else:
+                        src, dest = item.split(':')
+                except ValueError:
+                    self.module.fail_json(
+                        msg="Wrong format of volume: {}".format(item),
+                        failed=True
+                    )
+
+                mount_item = dict(
+                    source=src,
+                    target=dest,
+                    type='bind',
+                    propagation='rprivate'
+                )
+                if mode == 'ro':
+                    mount_item['read_only'] = True
+                if mode == 'shared':
+                    mount_item['propagation'] = 'shared'
+                mounts.append(mount_item)
+            else:
+                try:
+                    src, dest = item.split(':')
+                except ValueError:
+                    self.module.fail_json(
+                        msg="Wrong format of volume: {}".format(item),
+                        failed=True
+                    )
+                if src == 'devpts':
+                    mount_item = dict(
+                        target=dest,
+                        type='devpts'
+                    )
+                    mounts.append(mount_item)
+                else:
+                    filtered_volumes[src] = dict(
+                        bind=dest,
+                        mode='rw'
+                    )
+
+    def parse_dimensions(self, dimensions):
+        dimensions = dimensions.copy()
+
+        supported = {'cpu_period', 'cpu_quota', 'cpu_shares',
+                     'cpuset_cpus', 'cpuset_mems', 'mem_limit',
+                     'mem_reservation', 'memswap_limit',
+                     'kernel_memory', 'blkio_weight', 'ulimits'}
+        unsupported = set(dimensions) - supported
+        if unsupported:
+            self.module.exit_json(failed=True,
+                                  msg=repr("Unsupported dimensions"),
+                                  unsupported_dimensions=unsupported)
+
+        ulimits = dimensions.get('ulimits', {})
+        if ulimits:
+            # NOTE(m.hiner): default ulimits have to be filtered out because
+            # Podman would treat them as new ulimits and break the container
+            # as a result. Names are a copy of
+            # default_podman_dimensions_el9 in /ansible/group_vars/all.yml
+            for name in ['RLIMIT_NOFILE', 'RLIMIT_NPROC']:
+                ulimits.pop(name, None)
+
+            dimensions['ulimits'] = self.build_ulimits(ulimits)
+
+        return dimensions
+
+    def parse_healthcheck(self, healthcheck):
+        hc = super().parse_healthcheck(healthcheck)
+
+        # rename key to right format
+        if hc:
+            sp = hc['healthcheck'].pop('start_period', None)
+            if sp:
+                hc['healthcheck']['StartPeriod'] = sp
+
+        return hc
+
+    def prepare_image_args(self):
+        image, tag = self.parse_image()
+
+        args = dict(
+            repository=image,
+            tag=tag,
+            tls_verify=self.params.get('tls_verify', False),
+            stream=False
+        )
+
+        if self.params.get('auth_username', False):
+            args['auth_config'] = dict(
+                username=self.params.get('auth_username'),
+                password=self.params.get('auth_password', "")
+            )
+
+        if '/' not in image and self.params.get('auth_registry', False):
+            args['image'] = self.params['auth_registry'] + '/' + image
+        return args
+
+    def check_image(self):
+        try:
+            image = self.pc.images.get(self.params.get('image'))
+            return image.attrs
+        except APIError as e:
+            if e.status_code == 404:
+                return {}
+            else:
+                self.module.fail_json(
+                    failed=True,
+                    msg="Internal error: {}".format(
+                        e.explanation
+                    )
+                )
+
+    def check_volume(self):
+        try:
+            vol = self.pc.volumes.get(self.params.get('name'))
+            return vol.attrs
+        except APIError as e:
+            if e.status_code == 404:
+                return {}
+
+    def check_container(self):
+        name = self.params.get("name")
+        for cont in self.pc.containers.list(all=True):
+            cont.reload()
+            if name == cont.name:
+                return cont
+
+    def get_container_info(self):
+        container = self.check_container()
+        if not container:
+            return None
+
+        return container.attrs
+
+    def compare_container(self):
+        container = self.check_container()
+        if (not container or
+                self.check_container_differs() or
+                self.compare_config() or
+                self.systemd.check_unit_change()):
+            self.changed = True
+        return self.changed
+
+    def compare_pid_mode(self, container_info):
+        new_pid_mode = self.params.get('pid_mode') or self.params.get('pid')
+        current_pid_mode = container_info['HostConfig'].get('PidMode')
+
+        if not current_pid_mode:
+            current_pid_mode = None
+
+        # podman default pid_mode
+        if new_pid_mode is None and current_pid_mode == 'private':
+            return False
+
+        if new_pid_mode != current_pid_mode:
+            return True
+
+    def compare_image(self, container_info=None):
+        def parse_tag(tag):
+            splits = tag.rsplit('/', 1)
+            return splits[-1]
+
+        container_info = container_info or self.get_container_info()
+        if not container_info:
+            return True
+
+        new_image = self.check_image()
+        current_image = container_info['Image']
+        if not new_image:
+            return True
+        if new_image['Id'] != current_image:
+            return True
+        # compare name:tag
+        elif (parse_tag(self.params.get('image')) !=
+              parse_tag(container_info['Config']['Image'])):
+            return True
+
+    def compare_volumes(self, container_info):
+        def check_slash(string):
+            if not string:
+                return string
+            if string[-1] != '/':
+                return string + '/'
+            else:
+                return string
+
+        raw_volumes, binds = self.generate_volumes()
+        raw_vols, current_binds = self.generate_volumes(
+            container_info['HostConfig'].get('Binds'))
+
+        current_vols = [check_slash(vol) for vol in raw_vols if vol]
+        volumes = [check_slash(vol) for vol in raw_volumes if vol]
+
+        if not volumes:
+            volumes = list()
+        if not current_vols:
+            current_vols = list()
+        if not current_binds:
+            current_binds = list()
+
+        volumes.sort()
+        current_vols.sort()
+
+        if set(volumes).symmetric_difference(set(current_vols)):
+            return True
+
+        new_binds = list()
+        new_current_binds = list()
+        if binds:
+            for k, v in binds.items():
+                k = check_slash(k)
+                v['bind'] = check_slash(v['bind'])
+                new_binds.append(
+                    "{}:{}:{}".format(k, v['bind'], v['mode']))
+
+        if current_binds:
+            for k, v in current_binds.items():
+                k = check_slash(k)
+                v['bind'] = check_slash(v['bind'])
+                if 'ro' in v['mode']:
+                    v['mode'] = 'ro'
+                else:
+                    v['mode'] = 'rw'
+                new_current_binds.append(
+                    "{}:{}:{}".format(k, v['bind'], v['mode'][0:2]))
+
+        new_binds.sort()
+        new_current_binds.sort()
+
+        if set(new_binds).symmetric_difference(set(new_current_binds)):
+            return True
+
+    def compare_dimensions(self, container_info):
+        new_dimensions = self.params.get('dimensions')
+
+        # NOTE(mgoddard): The names used by Docker are inconsisent between
+        # configuration of a container's resources and the resources in
+        # container_info['HostConfig']. This provides a mapping between the
+        # two.
+        dimension_map = {
+            'mem_limit': 'Memory', 'mem_reservation': 'MemoryReservation',
+            'memswap_limit': 'MemorySwap', 'cpu_period': 'CpuPeriod',
+            'cpu_quota': 'CpuQuota', 'cpu_shares': 'CpuShares',
+            'cpuset_cpus': 'CpusetCpus', 'cpuset_mems': 'CpusetMems',
+            'kernel_memory': 'KernelMemory', 'blkio_weight': 'BlkioWeight',
+            'ulimits': 'Ulimits'}
+        unsupported = set(new_dimensions.keys()) - \
+            set(dimension_map.keys())
+        if unsupported:
+            self.module.exit_json(
+                failed=True, msg=repr("Unsupported dimensions"),
+                unsupported_dimensions=unsupported)
+        current_dimensions = container_info['HostConfig']
+        for key1, key2 in dimension_map.items():
+            # NOTE(mgoddard): If a resource has been explicitly requested,
+            # check for a match. Otherwise, ensure it is set to the default.
+            if key1 in new_dimensions:
+                if key1 == 'ulimits':
+                    if self.compare_ulimits(new_dimensions[key1],
+                                            current_dimensions[key2]):
+                        return True
+                elif new_dimensions[key1] != current_dimensions[key2]:
+                    return True
+            elif current_dimensions[key2]:
+                # The default values of all (except ulimits) currently
+                # supported resources are '' or 0 - both falsey.
+                return True
+
+    def compare_config(self):
+        try:
+            container = self.pc.containers.get(self.params['name'])
+            container.reload()
+            if container.status != 'running':
+                return True
+
+            rc, raw_output = container.exec_run(COMPARE_CONFIG_CMD,
+                                                user='root')
+        except APIError as e:
+            if e.is_client_error():
+                return True
+            else:
+                raise
+        # Exit codes:
+        # 0: not changed
+        # 1: changed
+        # else: error
+        if rc == 0:
+            return False
+        elif rc == 1:
+            return True
+        else:
+            raise Exception('Failed to compare container configuration: '
+                            'ExitCode: %s Message: %s' %
+                            (rc, raw_output.decode('utf-8')))
+
+    def pull_image(self):
+        args = self.prepare_image_args()
+        old_image = self.check_image()
+
+        try:
+            image = self.pc.images.pull(**args)
+
+            if image.attrs == {}:
+                self.module.fail_json(
+                    msg="The requested image does not exist: {}".format(
+                        self.params['image']),
+                    failed=True
+                )
+            self.changed = old_image != image.attrs
+        except APIError as e:
+            self.module.fail_json(
+                msg="Unknown error message: {}".format(
+                    str(e)),
+                failed=True
+            )
+
+    def remove_container(self):
+        self.changed |= self.systemd.remove_unit_file()
+        container = self.check_container()
+        if container:
+            try:
+                container.remove(force=True)
+            except APIError:
+                if self.check_container():
+                    raise
+
+    def build_ulimits(self, ulimits):
+        ulimits_opt = []
+        for key, value in ulimits.items():
+            soft = value.get('soft')
+            hard = value.get('hard')
+            # Converted to simple dictionary instead of Ulimit type
+            ulimits_opt.append(dict(Name=key,
+                                    Soft=soft,
+                                    Hard=hard))
+        return ulimits_opt
+
+    def create_container(self):
+        args = self.prepare_container_args()
+        container = self.pc.containers.create(**args)
+        if container.attrs == {}:
+            data = container.to_dict()
+            self.module.fail_json(failed=True, msg="Creation failed", **data)
+        else:
+            self.changed |= self.systemd.create_unit_file()
+        return container
+
+    def recreate_or_restart_container(self):
+        strategy = self.params.get(
+            'environment', dict()).get('KOLLA_CONFIG_STRATEGY')
+
+        container = self.get_container_info()
+        if not container:
+            self.start_container()
+            return
+
+        if strategy == 'COPY_ONCE' or self.check_container_differs():
+            self.ensure_image()
+
+            self.stop_container()
+            self.remove_container()
+            self.start_container()
+
+        elif strategy == 'COPY_ALWAYS':
+            self.restart_container()
+
+    def start_container(self):
+        self.ensure_image()
+
+        container = self.check_container()
+        if container and self.check_container_differs():
+            self.stop_container()
+            self.remove_container()
+            container = self.check_container()
+
+        if not container:
+            self.create_container()
+            container = self.check_container()
+
+        if container.status != 'running':
+            self.changed = True
+            if self.params.get('restart_policy') == 'no':
+                container = self.check_container()
+                container.start()
+            else:
+                self.systemd.create_unit_file()
+                if not self.systemd.start():
+                    self.module.fail_json(
+                        changed=True,
+                        msg="Container timed out",
+                        **self.check_container().attrs)
+
+        if not self.params.get('detach'):
+            container = self.check_container()
+            rc = container.wait()
+
+            stdout = [line.decode() for line in container.logs(stdout=True,
+                      stderr=False)]
+            stderr = [line.decode() for line in container.logs(stdout=False,
+                      stderr=True)]
+
+            self.result['rc'] = rc
+            self.result['stdout'] = "\n".join(stdout) if len(stdout) else ""
+            self.result['stderr'] = "\n".join(stderr) if len(stderr) else ""
+
+            if self.params.get('remove_on_exit'):
+                self.stop_container()
+                self.remove_container()
+            if rc != 0:
+                self.module.fail_json(
+                    changed=True,
+                    msg="Container exited with non-zero return code %s" % rc,
+                    **self.result
+                )
+
+    def stop_container(self):
+        name = self.params.get('name')
+        graceful_timeout = self.params.get('graceful_timeout')
+        if not graceful_timeout:
+            graceful_timeout = 10
+        container = self.check_container()
+        if not container:
+            ignore_missing = self.params.get('ignore_missing')
+            if not ignore_missing:
+                self.module.fail_json(
+                    msg="No such container: {} to stop".format(name))
+        elif not (container.status == 'exited' or
+                  container.status == 'stopped'):
+            self.changed = True
+            if self.params.get('restart_policy') != 'no':
+                self.systemd.create_unit_file()
+                self.systemd.stop()
+            else:
+                container.stop(timeout=str(graceful_timeout))
+
+    def stop_and_remove_container(self):
+        container = self.check_container()
+
+        if container:
+            self.stop_container()
+            self.remove_container()
+
+    def restart_container(self):
+        container = self.check_container()
+
+        if not container:
+            self.module.fail_json(
+                msg="No such container: {}".format(self.params.get('name'))
+            )
+        else:
+            self.changed = True
+            self.systemd.create_unit_file()
+
+            if not self.systemd.restart():
+                self.module.fail_json(
+                    changed=True,
+                    msg="Container timed out",
+                    **container.attrs)
+
+    def create_volume(self):
+        if not self.check_volume():
+            self.changed = True
+            args = dict(
+                name=self.params.get('name'),
+                driver='local'
+            )
+
+            vol = self.pc.volumes.create(**args)
+            self.result = vol.attrs
+
+    def remove_volume(self):
+        if self.check_volume():
+            self.changed = True
+            try:
+                self.pc.volumes.remove(self.params.get('name'))
+            except APIError as e:
+                if e.status_code == 409:
+                    self.module.fail_json(
+                        failed=True,
+                        msg="Volume named '{}' is currently in-use".format(
+                            self.params.get('name')
+                        )
+                    )
+                else:
+                    self.module.fail_json(
+                        failed=True,
+                        msg="Internal error: {}".format(
+                            e.explanation
+                        )
+                    )
+                raise
+
+    def remove_image(self):
+        if self.check_image():
+            image = self.pc.images.get(self.params['image'])
+            self.changed = True
+            try:
+                image.remove()
+            except APIError as e:
+                if e.status_code == 409:
+                    self.module.fail_json(
+                        failed=True,
+                        msg="Image '{}' is currently in-use".format(
+                            self.params.get('image')
+                        )
+                    )
+                else:
+                    self.module.fail_json(
+                        failed=True,
+                        msg="Internal error: {}".format(
+                            str(e)
+                        )
+                    )
+                raise
+
+    def ensure_image(self):
+        if not self.check_image():
+            self.pull_image()

ansible/module_utils/kolla_systemd_worker.py

@@ -21,15 +21,15 @@ TEMPLATE = '''# ${service_name}
 # autogenerated by Kolla-Ansible
 
 [Unit]
-Description=docker ${service_name}
+Description=${engine} ${service_name}
-After=docker.service
+After=${deps}
-Requires=docker.service
+Requires=${deps}
-StartLimitIntervalSec=${restart_timeout}
+StartLimitInterval=${restart_timeout}
 StartLimitBurst=${restart_retries}
 
 [Service]
-ExecStart=/usr/bin/docker start -a ${name}
+ExecStart=/usr/bin/${engine} start -a ${name}
-ExecStop=/usr/bin/docker stop ${name} -t ${graceful_timeout}
+ExecStop=/usr/bin/${engine} stop ${name} -t ${graceful_timeout}
 Restart=${restart_policy}
 RestartSec=${restart_duration}
 
@@ -46,6 +46,12 @@ class SystemdWorker(object):
         if not name:
             return None
 
+        container_engine = params.get('container_engine')
+        if container_engine == 'docker':
+            dependencies = 'docker.service'
+        else:
+            dependencies = 'network-online.target'
+
         restart_policy = params.get('restart_policy', 'no')
         if restart_policy == 'unless-stopped':
             restart_policy = 'always'
@@ -62,8 +68,8 @@ class SystemdWorker(object):
         self.container_dict = dict(
             name=name,
             service_name='kolla-' + name + '-container.service',
-            engine='docker',
+            engine=container_engine,
-            deps='docker.service',
+            deps=dependencies,
             graceful_timeout=params.get('graceful_timeout'),
             restart_policy=restart_policy,
             restart_timeout=restart_timeout,

ansible/roles/ceilometer/defaults/main.yml

@@ -22,7 +22,7 @@ ceilometer_services:
     enabled: True
     privileged: True
     image: "{{ ceilometer_compute_image_full }}"
-    volumes: "{{ ceilometer_compute_default_volumes + ceilometer_compute_extra_volumes }}"
+    volumes: "{{ ceilometer_compute_default_volumes + ceilometer_compute_extra_volumes + lookup('vars', 'run_default_volumes_' + kolla_container_engine) }}"
     dimensions: "{{ ceilometer_compute_dimensions }}"
     healthcheck: "{{ ceilometer_compute_healthcheck }}"
   ceilometer-ipmi:
@@ -136,7 +136,7 @@ ceilometer_compute_default_volumes:
   - "{{ node_config_directory }}/ceilometer-compute/:{{ container_config_directory }}/:ro"
   - "/etc/localtime:/etc/localtime:ro"
   - "{{ '/etc/timezone:/etc/timezone:ro' if ansible_facts.os_family == 'Debian' else '' }}"
-  - "/run/:/run/:shared"
+  - "/run:/run{{ ':shared' if kolla_container_engine == 'docker' else '' }}"
   - "ceilometer:/var/lib/ceilometer/"
   - "kolla_logs:/var/log/kolla/"
   - "{{ ceilometer_libvirt_volume }}:/var/lib/libvirt"

ansible/roles/cinder/defaults/main.yml

@@ -40,7 +40,7 @@ cinder_services:
     privileged: True
     ipc_mode: "host"
     tmpfs: "{{ cinder_volume_tmpfs }}"
-    volumes: "{{ cinder_volume_default_volumes + cinder_volume_extra_volumes }}"
+    volumes: "{{ cinder_volume_default_volumes + cinder_volume_extra_volumes + lookup('vars', 'run_default_volumes_' + kolla_container_engine) }}"
     dimensions: "{{ cinder_volume_dimensions }}"
     healthcheck: "{{ cinder_volume_healthcheck }}"
   cinder-backup:
@@ -49,7 +49,7 @@ cinder_services:
     enabled: "{{ enable_cinder_backup | bool }}"
     image: "{{ cinder_backup_image_full }}"
     privileged: True
-    volumes: "{{ cinder_backup_default_volumes + cinder_backup_extra_volumes }}"
+    volumes: "{{ cinder_backup_default_volumes + cinder_backup_extra_volumes + lookup('vars', 'run_default_volumes_' + kolla_container_engine) }}"
     dimensions: "{{ cinder_backup_dimensions }}"
     healthcheck: "{{ cinder_backup_healthcheck }}"
 
@@ -171,7 +171,7 @@ cinder_backup_default_volumes:
   - "{{ '/etc/timezone:/etc/timezone:ro' if ansible_facts.os_family == 'Debian' else '' }}"
   - "/dev/:/dev/"
   - "/lib/modules:/lib/modules:ro"
-  - "/run/:/run/:shared"
+  - "/run:/run{{ ':shared' if kolla_container_engine == 'docker' else '' }}"
   - "cinder:/var/lib/cinder"
   - "{% if enable_iscsid | bool %}iscsi_info:/etc/iscsi{% endif %}"
   - "kolla_logs:/var/log/kolla/"
@@ -188,7 +188,7 @@ cinder_volume_default_volumes:
   - "{{ '/etc/timezone:/etc/timezone:ro' if ansible_facts.os_family == 'Debian' else '' }}"
   - "/dev/:/dev/"
   - "/lib/modules:/lib/modules:ro"
-  - "/run/:/run/:shared"
+  - "/run:/run{{ ':shared' if kolla_container_engine == 'docker' else '' }}"
   - "cinder:/var/lib/cinder"
   - "{% if enable_iscsid | bool %}iscsi_info:/etc/iscsi{% endif %}"
   - "{% if enable_cinder_backend_lvm | bool and cinder_target_helper == 'lioadm' %}target_config:/etc/target{% endif %}"

ansible/roles/common/defaults/main.yml

@@ -18,7 +18,7 @@ common_services:
       ANSIBLE_NOCOLOR: "1"
       ANSIBLE_LIBRARY: "/usr/share/ansible"
     privileged: True
-    volumes: "{{ kolla_toolbox_default_volumes + kolla_toolbox_extra_volumes }}"
+    volumes: "{{ kolla_toolbox_default_volumes + kolla_toolbox_extra_volumes + lookup('vars', 'run_default_volumes_' + kolla_container_engine) }}"
     dimensions: "{{ kolla_toolbox_dimensions }}"
   cron:
     container_name: cron
@@ -107,7 +107,7 @@ kolla_toolbox_default_volumes:
   - "/etc/localtime:/etc/localtime:ro"
   - "{{ '/etc/timezone:/etc/timezone:ro' if ansible_facts.os_family == 'Debian' else '' }}"
   - "/dev/:/dev/"
-  - "/run/:/run/:shared"
+  - "/run/:/run/{{ ':shared' if kolla_container_engine == 'docker' else '' }}"   # see: https://github.com/containers/podman/issues/16305
   - "kolla_logs:/var/log/kolla/"
 cron_default_volumes:
   - "{{ node_config_directory }}/cron/:{{ container_config_directory }}/:ro"

ansible/roles/common/tasks/bootstrap.yml

@@ -10,7 +10,7 @@
 - name: Link kolla_logs volume to /var/log/kolla
   become: true
   file:
-    src: "{{ docker_runtime_directory or '/var/lib/docker' }}/volumes/kolla_logs/_data"
+    src: "{{ container_engine_volumes_path }}/kolla_logs/_data"
     path: /var/log/kolla
     state: link
   when: inventory_hostname in groups['kolla-logs']

ansible/roles/common/tasks/config.yml

@@ -27,6 +27,18 @@
   when:
     - kolla_copy_ca_into_containers | bool
 
+- name: Copying over /run subdirectories conf
+  become: true
+  template:
+    src: kolla-directories.conf.j2
+    dest: /etc/tmpfiles.d/kolla.conf
+  when: kolla_container_engine == 'podman'
+
+- name: Restart systemd-tmpfiles
+  become: true
+  command: systemd-tmpfiles --create
+  when: kolla_container_engine == 'podman'
+
 - name: Copying over config.json files for services
   template:
     src: "{{ item.key }}.json.j2"

ansible/roles/common/templates/kolla-directories.conf.j2

@@ -0,0 +1,3 @@
+{% for path in run_default_subdirectories %}
+d {{ path }} 0755 root root - -
+{% endfor %}

ansible/roles/ironic/defaults/main.yml

@@ -30,7 +30,7 @@ ironic_services:
     enabled: true
     image: "{{ ironic_conductor_image_full }}"
     privileged: True
-    volumes: "{{ ironic_conductor_default_volumes + ironic_conductor_extra_volumes }}"
+    volumes: "{{ ironic_conductor_default_volumes + ironic_conductor_extra_volumes + lookup('vars', 'run_default_volumes_' + kolla_container_engine) }}"
     dimensions: "{{ ironic_conductor_dimensions }}"
     healthcheck: "{{ ironic_conductor_healthcheck }}"
   ironic-inspector:
@@ -230,7 +230,7 @@ ironic_conductor_default_volumes:
   - "/lib/modules:/lib/modules:ro"
   - "/sys:/sys"
   - "/dev:/dev"
-  - "/run:/run:shared"
+  - "/run:/run{{ ':shared' if kolla_container_engine == 'docker' else '' }}"
   - "kolla_logs:/var/log/kolla"
   - "ironic:/var/lib/ironic"
   - "{{ kolla_dev_repos_directory ~ '/ironic/ironic:/var/lib/kolla/venv/lib/python' ~ distro_python_version ~ '/site-packages/ironic' if ironic_dev_mode | bool else '' }}"

ansible/roles/iscsi/defaults/main.yml

@@ -7,7 +7,7 @@ iscsi_services:
     image: "{{ iscsid_image_full }}"
     ipc_mode: "host"
     privileged: True
-    volumes: "{{ iscsid_default_volumes + iscsid_extra_volumes }}"
+    volumes: "{{ iscsid_default_volumes + iscsid_extra_volumes + lookup('vars', 'run_default_volumes_' + kolla_container_engine) }}"
     dimensions: "{{ iscsid_dimensions }}"
   tgtd:
     container_name: tgtd
@@ -16,7 +16,7 @@ iscsi_services:
     image: "{{ tgtd_image_full }}"
     ipc_mode: "host"
     privileged: True
-    volumes: "{{ tgtd_default_volumes + tgtd_extra_volumes }}"
+    volumes: "{{ tgtd_default_volumes + tgtd_extra_volumes + lookup('vars', 'run_default_volumes_' + kolla_container_engine) }}"
     dimensions: "{{ tgtd_dimensions }}"
 
 
@@ -42,7 +42,7 @@ iscsid_default_volumes:
   - "/etc/localtime:/etc/localtime:ro"
   - "{{ '/etc/timezone:/etc/timezone:ro' if ansible_facts.os_family == 'Debian' else '' }}"
   - "/dev/:/dev/"
-  - "/run/:/run/:shared"
+  - "/run:/run{{ ':shared' if kolla_container_engine == 'docker' else '' }}"
   - "/lib/modules:/lib/modules:ro"
   - "/sys/kernel/config:/configfs"
   - "iscsi_info:/etc/iscsi"
@@ -52,7 +52,7 @@ tgtd_default_volumes:
   - "/etc/localtime:/etc/localtime:ro"
   - "{{ '/etc/timezone:/etc/timezone:ro' if ansible_facts.os_family == 'Debian' else '' }}"
   - "/dev/:/dev/"
-  - "/run/:/run/:shared"
+  - "/run:/run{{ ':shared' if kolla_container_engine == 'docker' else '' }}"
   - "/lib/modules:/lib/modules:ro"
   - "/sys/kernel/config:/configfs"
 iscsid_extra_volumes: "{{ default_extra_volumes }}"

ansible/roles/kuryr/defaults/main.yml

@@ -16,7 +16,7 @@ kuryr_services:
     privileged: True
     cap_add:
       - NET_ADMIN
-    volumes: "{{ kuryr_default_volumes + kuryr_extra_volumes }}"
+    volumes: "{{ kuryr_default_volumes + kuryr_extra_volumes + lookup('vars', 'run_default_volumes_' + kolla_container_engine) }}"
     dimensions: "{{ kuryr_dimensions }}"
     healthcheck: "{{ kuryr_healthcheck }}"
 
@@ -52,7 +52,7 @@ kuryr_default_volumes:
   - "/etc/localtime:/etc/localtime:ro"
   - "{{ '/etc/timezone:/etc/timezone:ro' if ansible_facts.os_family == 'Debian' else '' }}"
   - "/lib/modules:/lib/modules:ro"
-  - "/run:/run:shared"
+  - "/run:/run{{ ':shared' if kolla_container_engine == 'docker' else '' }}"
   - "/usr/lib/docker:/usr/lib/docker"
   - "{{ kolla_dev_repos_directory ~ '/kuryr/kuryr:/var/lib/kolla/venv/lib/python' ~ distro_python_version ~ '/site-packages/kuryr' if kuryr_dev_mode | bool else '' }}"
   - "{{ kolla_dev_repos_directory ~ '/kuryr-libnetwork/kuryr_libnetwork:/var/lib/kolla/venv/lib/python' ~ distro_python_version ~ '/site-packages/kuryr_libnetwork' if kuryr_dev_mode | bool else '' }}"

ansible/roles/manila/defaults/main.yml

@@ -36,7 +36,7 @@ manila_services:
     image: "{{ manila_share_image_full }}"
     enabled: True
     privileged: True
-    volumes: "{{ manila_share_default_volumes + manila_share_extra_volumes }}"
+    volumes: "{{ manila_share_default_volumes + manila_share_extra_volumes + lookup('vars', 'run_default_volumes_' + kolla_container_engine) }}"
     dimensions: "{{ manila_share_dimensions }}"
     healthcheck: "{{ manila_share_healthcheck }}"
   manila-data:
@@ -45,7 +45,7 @@ manila_services:
     image: "{{ manila_data_image_full }}"
     enabled: True
     privileged: True
-    volumes: "{{ manila_data_default_volumes + manila_data_extra_volumes }}"
+    volumes: "{{ manila_data_default_volumes + manila_data_extra_volumes + lookup('vars', 'run_default_volumes_' + kolla_container_engine) }}"
     dimensions: "{{ manila_data_dimensions }}"
     healthcheck: "{{ manila_data_healthcheck }}"
 
@@ -159,7 +159,7 @@ manila_share_default_volumes:
   - "{{ node_config_directory }}/manila-share/:{{ container_config_directory }}/:ro"
   - "/etc/localtime:/etc/localtime:ro"
   - "{{ '/etc/timezone:/etc/timezone:ro' if ansible_facts.os_family == 'Debian' else '' }}"
-  - "/run/:/run/:shared"
+  - "/run:/run{{ ':shared' if kolla_container_engine == 'docker' else '' }}"
   - "kolla_logs:/var/log/kolla/"
   - "/lib/modules:/lib/modules:ro"
   - "{{ kolla_dev_repos_directory ~ '/manila/manila:/var/lib/kolla/venv/lib/python' ~ distro_python_version ~ '/site-packages/manila' if manila_dev_mode | bool else '' }}"
@@ -180,7 +180,7 @@ manila_data_default_volumes:
   - "/etc/localtime:/etc/localtime:ro"
   - "{{ '/etc/timezone:/etc/timezone:ro' if ansible_facts.os_family == 'Debian' else '' }}"
   - "/dev/:/dev/"
-  - "/run/:/run/:shared"
+  - "/run:/run{{ ':shared' if kolla_container_engine == 'docker' else '' }}"
   - "kolla_logs:/var/log/kolla/"
   - "{{ kolla_dev_repos_directory ~ '/manila/manila:/var/lib/kolla/venv/lib/python' ~ distro_python_version ~ '/site-packages/manila' if manila_dev_mode | bool else '' }}"
 

ansible/roles/mariadb/tasks/recover_cluster.yml

@@ -20,6 +20,7 @@
       kolla_docker:
         name: "{{ mariadb_service.container_name }}"
         action: "stop_container"
+        common_options: "{{ docker_common_options }}"
         ignore_missing: true
 
     # Run wsrep recovery with detach=false to block until completion. Use a
@@ -42,7 +43,7 @@
 
     - name: Copying MariaDB log file to /tmp
       become: true
-      command: "cp {{ docker_runtime_directory or '/var/lib/docker' }}/volumes/kolla_logs/_data/mariadb/mariadb.log /tmp/mariadb_tmp.log"
+      command: "cp {{ container_engine_volumes_path }}/kolla_logs/_data/mariadb/mariadb.log /tmp/mariadb_tmp.log"
 
     # Look for sequence number in logs. Format is:
     # WSREP: Recovered position: <UUID>:<seqno>.
@@ -99,7 +100,7 @@
   become: true
   lineinfile:
     create: yes
-    dest: "{{ docker_runtime_directory or '/var/lib/docker' }}/volumes/mariadb/_data/grastate.dat"
+    dest: "{{ container_engine_volumes_path }}/mariadb/_data/grastate.dat"
     regexp: 'safe_to_bootstrap:(.*)$'
     line: 'safe_to_bootstrap: 1'
     state: present

ansible/roles/multipathd/defaults/main.yml

@@ -7,7 +7,7 @@ multipathd_services:
     ipc_mode: "host"
     privileged: True
     image: "{{ multipathd_image_full }}"
-    volumes: "{{ multipathd_default_volumes + multipathd_extra_volumes }}"
+    volumes: "{{ multipathd_default_volumes + multipathd_extra_volumes + lookup('vars', 'run_default_volumes_' + kolla_container_engine) }}"
 
 
 ####################
@@ -23,7 +23,7 @@ multipathd_default_volumes:
   - "/etc/localtime:/etc/localtime:ro"
   - "{{ '/etc/timezone:/etc/timezone:ro' if ansible_facts.os_family == 'Debian' else '' }}"
   - "/dev/:/dev/"
-  - "/run/:/run/:shared"
+  - "/run:/run{{ ':shared' if kolla_container_engine == 'docker' else '' }}"
   - "/lib/modules:/lib/modules:ro"
   - "/sys/kernel/config:/configfs"
 multipathd_extra_volumes: "{{ default_extra_volumes }}"

ansible/roles/nova-cell/defaults/main.yml

@@ -8,7 +8,7 @@ nova_cell_services:
     pid_mode: "host"
     cgroupns_mode: "host"
     privileged: True
-    volumes: "{{ nova_libvirt_default_volumes + nova_libvirt_extra_volumes }}"
+    volumes: "{{ nova_libvirt_default_volumes + nova_libvirt_extra_volumes + lookup('vars', 'run_default_volumes_' + kolla_container_engine) }}"
     dimensions: "{{ nova_libvirt_dimensions }}"
     healthcheck: "{{ nova_libvirt_healthcheck }}"
   nova-ssh:
@@ -59,7 +59,7 @@ nova_cell_services:
     privileged: True
     enabled: "{{ not enable_nova_fake | bool }}"
     ipc_mode: "host"
-    volumes: "{{ nova_compute_default_volumes + nova_compute_extra_volumes }}"
+    volumes: "{{ nova_compute_default_volumes + nova_compute_extra_volumes + lookup('vars', 'run_default_volumes_' + kolla_container_engine) }}"
     dimensions: "{{ nova_compute_dimensions }}"
     healthcheck: "{{ nova_compute_healthcheck }}"
   nova-compute-ironic:
@@ -371,8 +371,9 @@ nova_libvirt_default_volumes:
   - "/etc/localtime:/etc/localtime:ro"
   - "{{ '/etc/timezone:/etc/timezone:ro' if ansible_facts.os_family == 'Debian' else '' }}"
   - "/lib/modules:/lib/modules:ro"
-  - "/run/:/run/:shared"
+  - "/run:/run{{ ':shared' if kolla_container_engine == 'docker' else '' }}"
   - "/dev:/dev"
+  - "{{ 'devpts:/dev/pts' if kolla_container_engine == 'podman' else '' }}"
   - "/sys/fs/cgroup:/sys/fs/cgroup"
   - "kolla_logs:/var/log/kolla/"
   - "libvirtd:/var/lib/libvirt"
@@ -418,7 +419,7 @@ nova_compute_default_volumes:
   - "/etc/localtime:/etc/localtime:ro"
   - "{{ '/etc/timezone:/etc/timezone:ro' if ansible_facts.os_family == 'Debian' else '' }}"
   - "/lib/modules:/lib/modules:ro"
-  - "/run:/run:shared"
+  - "/run:/run{{ ':shared' if kolla_container_engine == 'docker' else '' }}"
   - "/dev:/dev"
   - "kolla_logs:/var/log/kolla/"
   - "{% if enable_iscsid | bool %}iscsi_info:/etc/iscsi{% endif %}"
@@ -439,6 +440,7 @@ nova_cell_bootstrap_default_volumes:
   - "{{ '/etc/timezone:/etc/timezone:ro' if ansible_facts.os_family == 'Debian' else '' }}"
   - "kolla_logs:/var/log/kolla/"
   - "{{ kolla_dev_repos_directory ~ '/nova/nova:/var/lib/kolla/venv/lib/python' ~ distro_python_version ~ '/site-packages/nova' if nova_dev_mode | bool else '' }}"
+  - "{{ 'nova-cell:/var/lib/script/' if kolla_container_engine == 'podman' else '' }}"
 
 nova_extra_volumes: "{{ default_extra_volumes }}"
 nova_libvirt_extra_volumes: "{{ nova_extra_volumes }}"

ansible/roles/nova-cell/handlers/main.yml

@@ -198,7 +198,12 @@
       - "{{ node_config_directory }}/nova-compute-fake-{{ item }}/:{{ container_config_directory }}/:ro"
       - "/etc/localtime:/etc/localtime:ro"
       - "/lib/modules:/lib/modules:ro"
-      - "/run:/run:shared"
+      - "/run:/run{{ ':shared' if kolla_container_engine == 'docker' else '' }}"
+      - "/run/netns:/run/netns:shared"
+      - "/run/lock/nova:/run/lock/nova:shared"
+      - "/run/libvirt:/run/libvirt:shared"
+      - "/run/nova:/run/nova:shared"
+      - "/run/openvswitch:/run/openvswitch:shared"
       - "kolla_logs:/var/log/kolla/"
   with_sequence: start=1 end={{ num_nova_fake_per_node }}
   when:

ansible/roles/nova-cell/tasks/config-nova-fake.yml

@@ -57,7 +57,12 @@
       - "{{ node_config_directory }}/nova-compute-fake-{{ item }}/:{{ container_config_directory }}/:ro"
       - "/etc/localtime:/etc/localtime:ro"
       - "/lib/modules:/lib/modules:ro"
-      - "/run:/run:shared"
+      - "/run:/run{{ ':shared' if kolla_container_engine == 'docker' else '' }}"
+      - "/run/netns:/run/netns:shared"
+      - "/run/lock/nova:/run/lock/nova:shared"
+      - "/run/libvirt:/run/libvirt:shared"
+      - "/run/nova:/run/nova:shared"
+      - "/run/openvswitch:/run/openvswitch:shared"
       - "kolla_logs:/var/log/kolla/"
   with_sequence: start=1 end={{ num_nova_fake_per_node }}
   when:

ansible/roles/openvswitch/tasks/config-host.yml

@@ -5,3 +5,16 @@
   vars:
     modules:
       - {'name': openvswitch}
+
+# NOTE(m.hiner): Podman considers non-existent mount directory
+# as a error, so it has to be created beforehand.
+# See: https://github.com/containers/podman/issues/14781
+- name: Create /run/openvswitch directory on host
+  become: True
+  file:
+    path: /run/openvswitch
+    state: directory
+    mode: "0770"
+    owner: "{{ config_owner_user }}"
+    group: "{{ config_owner_group }}"
+  when: kolla_container_engine == 'podman'

ansible/roles/prechecks/tasks/service_checks.yml

@@ -12,7 +12,9 @@
   register: result
   changed_when: false
   check_mode: false
-  when: inventory_hostname in groups['baremetal']
+  when:
+    - kolla_container_engine == 'docker'
+    - inventory_hostname in groups['baremetal']
   failed_when: result is failed
                or result.stdout | regex_replace('.*\\b(\\d+\\.\\d+\\.\\d+)\\b.*', '\\1') is version(docker_version_min, '<')
 

ansible/roles/prune-images/tasks/prune_images.yml

@@ -6,3 +6,11 @@
     images_filters:
       label: kolla_version
     timeout: "{{ docker_image_prune_timeout }}"
+  when: kolla_container_engine == 'docker'
+
+# NOTE(m.hiner): Podman does not (yet?) have equivalent of docker_prune
+# and generic module podman_image does not support label filters
+- name: Pruning Kolla images
+  become: true
+  command: podman image prune --force --filter 'label=kolla_version'
+  when: kolla_container_engine == 'podman'

ansible/roles/sahara/defaults/main.yml

@@ -28,7 +28,7 @@ sahara_services:
     enabled: true
     image: "{{ sahara_engine_image_full }}"
     privileged: True
-    volumes: "{{ sahara_engine_default_volumes + sahara_engine_extra_volumes }}"
+    volumes: "{{ sahara_engine_default_volumes + sahara_engine_extra_volumes + lookup('vars', 'run_default_volumes_' + kolla_container_engine) }}"
     dimensions: "{{ sahara_engine_dimensions }}"
     healthcheck: "{{ sahara_engine_healthcheck }}"
 
@@ -115,7 +115,7 @@ sahara_engine_default_volumes:
   - "{{ '/etc/timezone:/etc/timezone:ro' if ansible_facts.os_family == 'Debian' else '' }}"
   - "sahara:/var/lib/sahara/"
   - "kolla_logs:/var/log/kolla/"
-  - "/run:/run:shared"
+  - "/run:/run{{ ':shared' if kolla_container_engine == 'docker' else '' }}"
   - "{{ kolla_dev_repos_directory ~ '/sahara/sahara:/var/lib/kolla/venv/lib/python' ~ distro_python_version ~ '/site-packages/sahara' if sahara_dev_mode | bool else '' }}"
 
 sahara_extra_volumes: "{{ default_extra_volumes }}"

ansible/roles/zun/defaults/main.yml

@@ -47,7 +47,7 @@ zun_services:
     enabled: true
     image: "{{ zun_compute_image_full }}"
     privileged: True
-    volumes: "{{ zun_compute_default_volumes + zun_compute_extra_volumes }}"
+    volumes: "{{ zun_compute_default_volumes + zun_compute_extra_volumes + lookup('vars', 'run_default_volumes_' + kolla_container_engine) }}"
     dimensions: "{{ zun_compute_dimensions }}"
     healthcheck: "{{ zun_compute_healthcheck }}"
   zun-cni-daemon:
@@ -56,7 +56,7 @@ zun_services:
     enabled: true
     image: "{{ zun_cni_daemon_image_full }}"
     privileged: True
-    volumes: "{{ zun_cni_daemon_default_volumes + zun_cni_daemon_extra_volumes }}"
+    volumes: "{{ zun_cni_daemon_default_volumes + zun_cni_daemon_extra_volumes + lookup('vars', 'run_default_volumes_' + kolla_container_engine) }}"
     dimensions: "{{ zun_cni_daemon_dimensions }}"
     healthcheck: "{{ zun_cni_daemon_healthcheck }}"
 
@@ -185,7 +185,7 @@ zun_compute_default_volumes:
   - "{{ '/etc/timezone:/etc/timezone:ro' if ansible_facts.os_family == 'Debian' else '' }}"
   - "kolla_logs:/var/log/kolla/"
   - "{{ kolla_dev_repos_directory ~ '/zun/zun:/var/lib/kolla/venv/lib/python' ~ distro_python_version ~ '/site-packages/zun' if zun_dev_mode | bool else '' }}"
-  - "/run:/run:shared"
+  - "/run:/run{{ ':shared' if kolla_container_engine == 'docker' else '' }}"
   - "/usr/lib/docker:/usr/lib/docker"
   - "/var/lib/docker:/var/lib/docker"
   - "/lib/modules:/lib/modules:ro"
@@ -198,7 +198,7 @@ zun_cni_daemon_default_volumes:
   - "{{ '/etc/timezone:/etc/timezone:ro' if ansible_facts.os_family == 'Debian' else '' }}"
   - "kolla_logs:/var/log/kolla/"
   - "{{ kolla_dev_repos_directory ~ '/zun/zun:/var/lib/kolla/venv/lib/python' ~ distro_python_version ~ '/site-packages/zun' if zun_dev_mode | bool else '' }}"
-  - "/run:/run:shared"
+  - "/run:/run{{ ':shared' if kolla_container_engine == 'docker' else '' }}"
 
 zun_extra_volumes: "{{ default_extra_volumes }}"
 zun_api_extra_volumes: "{{ zun_extra_volumes }}"

etc/kolla/globals.yml

@@ -79,12 +79,13 @@ workaround_ansible_issue_8743: yes
 # Optionally change the path to sysctl.conf modified by Kolla Ansible plays.
 #kolla_sysctl_conf_path: /etc/sysctl.conf
 
-################
+##################
 # Container engine
-################
+##################
+
+# Valid options are [ docker, podman ]
+#kolla_container_engine: docker
 
-# Valid options are [ docker ]
-# kolla_container_engine: docker
 
 ################
 # Docker options

releasenotes/notes/feature-podman-support-c4fa7c367b4ceac4.yaml

@@ -0,0 +1,7 @@
+---
+features:
+  - |
+    Implements support for Podman deployment as an alternative to Docker.
+    To perform deployment using Podman, set the variable
+    ``kolla_container_engine`` to value ``podman``
+    inside of the ``globals.yml`` file.

tests/kolla-toolbox-testsuite.yml

@@ -0,0 +1,89 @@
+---
+- name: Test successful & unchanged
+  kolla_toolbox:
+    common_options:
+      container_engine: "{{ item }}"
+    module_name: debug
+    module_args:
+      msg: hi
+  register: result
+
+- name: Assert result is successful
+  assert:
+    that: result is successful
+
+- name: Assert result is not changed
+  assert:
+    that: result is not changed
+
+- name: Test successful & changed
+  kolla_toolbox:
+    common_options:
+      container_engine: "{{ item }}"
+    module_name: command
+    module_args:
+      echo hi
+  register: result
+
+- name: Assert result is successful
+  assert:
+    that: result is successful
+
+- name: Assert result is changed
+  assert:
+    that: result is changed
+
+- name: Test unsuccessful
+  kolla_toolbox:
+    common_options:
+      container_engine: "{{ item }}"
+    module_name: command
+    module_args:
+      foo
+  register: result
+  ignore_errors: true
+
+- name: Assert result is failed
+  assert:
+    that: result is failed
+
+- name: Test invalid module parameters
+  kolla_toolbox:
+    common_options:
+      container_engine: "{{ item }}"
+    module_name: debug
+    module_args:
+      foo: bar
+  register: result
+  ignore_errors: true
+
+- name: Assert result is failed
+  assert:
+    that: result is failed
+
+- name: Setup for Test successful & changed (JSON format)
+  kolla_toolbox:
+    common_options:
+      container_engine: "{{ item }}"
+    module_name: file
+    module_args:
+      path: /tmp/foo
+      state: absent
+
+- name: Test successful & changed (JSON format)
+  kolla_toolbox:
+    common_options:
+      container_engine: "{{ item }}"
+    module_name: file
+    module_args:
+      path: /tmp/foo
+      state: directory
+  register: result
+
+- name: Assert result is successful
+  assert:
+    that: result is successful
+
+- name: Assert result is changed
+  assert:
+    that: result is changed

tests/kolla_docker_tests/test_docker_worker.py

@@ -39,7 +39,7 @@ dwm = imp.load_source('kolla_docker_worker', docker_worker_file)
 FAKE_DATA = {
 
     'params': {
-        'common_options': None,
+        'container_engine': 'docker',
         'api_version': None,
         'auth_username': None,
         'auth_password': None,
@@ -224,19 +224,21 @@ class TestMainModule(base.BaseTestCase):
         module_mock.fail_json.assert_called_once_with(
             changed=True, msg=repr("Some very ugly traceback"))
 
-    @mock.patch("kolla_docker.DockerWorker")
     @mock.patch("kolla_docker.generate_module")
-    def test_execute_module(self, mock_generate_module, mock_dw):
+    def test_execute_module(self, mock_generate_module):
-        mock_dw.return_value.check_image.return_value = False
-        mock_dw.return_value.changed = False
-        mock_dw.return_value.result = {"some_key": "some_value"}
         module_mock = mock.MagicMock()
         module_mock.params = self.fake_data['params']
         module_mock.params["action"] = "check_image"
         mock_generate_module.return_value = module_mock
-        kd.main()
+        with mock.patch(
-        mock_dw.assert_called_once_with(module_mock)
+            "ansible.module_utils.kolla_docker_worker.DockerWorker"
-        mock_dw.return_value.check_image.assert_called_once_with()
+        ) as mock_dw:
+            mock_dw.return_value.check_image.return_value = False
+            mock_dw.return_value.changed = False
+            mock_dw.return_value.result = {"some_key": "some_value"}
+            kd.main()
+            mock_dw.assert_called_once_with(module_mock)
+            mock_dw.return_value.check_image.assert_called_once_with()
         module_mock.exit_json.assert_called_once_with(changed=False,
                                                       result=False,
                                                       some_key="some_value")

tests/setup_gate.sh

@@ -103,11 +103,15 @@ function prepare_images {
     fi
 
     sudo tee -a /etc/kolla/kolla-build.conf <<EOF
+[DEFAULT]
+engine = ${CONTAINER_ENGINE}
+
 [profiles]
 gate = ${GATE_IMAGES}
 EOF
 
-    mkdir -p /tmp/logs/build
+    sudo mkdir -p /tmp/logs/build
+    sudo mkdir -p /opt/kolla_registry
 
     sudo $CONTAINER_ENGINE run -d --net=host -e REGISTRY_HTTP_ADDR=0.0.0.0:4000 --restart=always -v /opt/kolla_registry/:/var/lib/registry --name registry registry:2
 

tests/test-kolla-toolbox.yml

@@ -2,80 +2,11 @@
 - name: Test the kolla_toolbox module
   hosts: localhost
   gather_facts: false
+  vars:
+    container_engines:
+      - "docker"
+      - "podman"
   tasks:
-    - name: Test successful & unchanged
+    - name: Test kolla-toolbox for each container engine
-      kolla_toolbox:
+      include_tasks: kolla-toolbox-testsuite.yml
-        module_name: debug
+      with_items: "{{ container_engines }}"
-        module_args:
-          msg: hi
-      register: result
-
-    - name: Assert result is successful
-      assert:
-        that: result is successful
-
-    - name: Assert result is not changed
-      assert:
-        that: result is not changed
-
-    - name: Test successful & changed
-      kolla_toolbox:
-        module_name: command
-        module_args:
-          echo hi
-      register: result
-
-    - name: Assert result is successful
-      assert:
-        that: result is successful
-
-    - name: Assert result is changed
-      assert:
-        that: result is changed
-
-    - name: Test unsuccessful
-      kolla_toolbox:
-        module_name: command
-        module_args:
-          foo
-      register: result
-      ignore_errors: true
-
-    - name: Assert result is failed
-      assert:
-        that: result is failed
-
-    - name: Test invalid module parameters
-      kolla_toolbox:
-        module_name: debug
-        module_args:
-          foo: bar
-      register: result
-      ignore_errors: true
-
-    - name: Assert result is failed
-      assert:
-        that: result is failed
-
-    - name: Setup for Test successful & changed (JSON format)
-      kolla_toolbox:
-        module_name: file
-        module_args:
-          path: /tmp/foo
-          state: absent
-
-    - name: Test successful & changed (JSON format)
-      kolla_toolbox:
-        module_name: file
-        module_args:
-          path: /tmp/foo
-          state: directory
-      register: result
-
-    - name: Assert result is successful
-      assert:
-        that: result is successful
-
-    - name: Assert result is changed
-      assert:
-        that: result is changed

tox.ini

@@ -12,6 +12,7 @@ allowlist_externals = bash
 deps = -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master}
        -r{toxinidir}/requirements.txt
        -r{toxinidir}/test-requirements.txt
+       podman>=4.3.0,<5
 passenv = http_proxy,HTTP_PROXY,https_proxy,HTTPS_PROXY,no_proxy,NO_PROXY, \
           OS_STDOUT_CAPTURE,OS_STDERR_CAPTURE,OS_LOG_CAPTURE,OS_TEST_TIMEOUT, \
           PYTHON,OS_TEST_PATH,LISTOPT,IDOPTION
@@ -93,6 +94,7 @@ setenv =
   ANSIBLE_LIBRARY = {toxinidir}/ansible/library
   ANSIBLE_ACTION_PLUGINS = {toxinidir}/ansible/action_plugins
   ANSIBLE_FILTER_PLUGINS = {toxinidir}/ansible/filter_plugins
+
 deps =
   -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master}
   -r{toxinidir}/requirements.txt

zuul.d/jobs.yaml

@@ -17,6 +17,16 @@
     required-projects:
       - openstack/kolla
 
+- job:
+    name: kolla-ansible-debian-aarch64-podman
+    parent: kolla-ansible-debian
+    nodeset: kolla-ansible-debian-bookworm-aarch64
+    timeout: 10800
+    vars:
+      container_engine: podman
+    required-projects:
+      - openstack/kolla
+
 - job:
     name: kolla-ansible-debian
     parent: kolla-ansible-base
@@ -25,6 +35,15 @@
       base_distro: debian
       tls_enabled: true
 
+- job:
+    name: kolla-ansible-debian-podman
+    parent: kolla-ansible-base
+    nodeset: kolla-ansible-debian-bookworm
+    vars:
+      base_distro: debian
+      tls_enabled: true
+      container_engine: podman
+
 - job:
     name: kolla-ansible-openeuler
     parent: kolla-ansible-base
@@ -42,6 +61,15 @@
       base_distro: rocky
       tls_enabled: true
 
+- job:
+    name: kolla-ansible-rocky9-podman
+    parent: kolla-ansible-base
+    nodeset: kolla-ansible-rocky9
+    vars:
+      base_distro: rocky
+      tls_enabled: true
+      container_engine: podman
+
 - job:
     name: kolla-ansible-ubuntu
     parent: kolla-ansible-base
@@ -50,6 +78,15 @@
       base_distro: ubuntu
       tls_enabled: true
 
+- job:
+    name: kolla-ansible-ubuntu-podman
+    parent: kolla-ansible-base
+    nodeset: kolla-ansible-jammy
+    vars:
+      base_distro: ubuntu
+      tls_enabled: true
+      container_engine: podman
+
 - job:
     name: kolla-ansible-rocky9-kvm
     parent: kolla-ansible-kvm-base

zuul.d/project.yaml

@@ -13,9 +13,12 @@
       jobs:
         - kolla-ansible-centos9s
         - kolla-ansible-debian
+        - kolla-ansible-debian-podman
         - kolla-ansible-openeuler
         - kolla-ansible-rocky9
+        - kolla-ansible-rocky9-podman
         - kolla-ansible-ubuntu
+        - kolla-ansible-ubuntu-podman
         - kolla-ansible-rocky9-kvm
         - kolla-ansible-ubuntu-kvm
         - kolla-ansible-rocky9-multinode-ipv6
@@ -62,15 +65,19 @@
     check-arm64:
       jobs:
         - kolla-ansible-debian-aarch64
+        - kolla-ansible-debian-aarch64-podman
         - kolla-ansible-debian-upgrade-aarch64
     gate:
       jobs:
         - kolla-ansible-debian
         - kolla-ansible-debian-upgrade
+        - kolla-ansible-debian-podman
         - kolla-ansible-rocky9
         - kolla-ansible-rocky9-upgrade
+        - kolla-ansible-rocky9-podman
         - kolla-ansible-ubuntu
         - kolla-ansible-ubuntu-upgrade
+        - kolla-ansible-ubuntu-podman
     experimental:
       jobs:
         - kolla-ansible-rocky9-swift-upgrade