Add resource Constraints to Nova containers.

This commit will constrain the dimensions of service `Nova`
and sub-containers deployed along with it.

A user can give the dimension values in `/etc/kolla/globals.yml`
the data-types just like stated in this commit.

Reference-Docs:
https://docs.docker.com/config/containers/resource_constraints/

Added Test-cases for the same.

Partially-Implements: blueprint resource-constraints
Change-Id: I6458d8fb7b26a6e7c3a9fd0d674d9cf129b0bf5d

ansible/group_vars/all.yml

@@ -108,6 +108,11 @@ docker_common_options:
   restart_policy: "{{ docker_restart_policy }}"
   restart_retries: "{{ docker_restart_policy_retry }}"
 
+####################
+# Dimensions options
+####################
+# Dimension options for Docker Containers
+default_container_dimensions: {}
 
 ####################
 # keepalived options

ansible/library/kolla_docker.py

@@ -552,6 +552,18 @@ class DockerWorker(object):
             'volumes_from': self.params.get('volumes_from')
         }
 
+        if self.params.get('dimensions'):
+            supported = {'cpu_period', 'cpu_quota', 'cpu_shares',
+                         'cpuset_cpus', 'cpuset_mems', 'mem_limit',
+                         'mem_reservation', 'memswap_limit',
+                         'kernel_memory', 'blkio_weight'}
+            unsupported = set(self.params.get('dimensions')) - supported
+            if unsupported:
+                self.module.exit_json(failed=True,
+                                      msg=repr("Unsupported dimensions"),
+                                      unsupported_dimensions=unsupported)
+            options.update(self.params.get('dimensions'))
+
         if self.params.get('restart_policy') in ['on-failure',
                                                  'always',
                                                  'unless-stopped']:
@@ -772,7 +784,8 @@ def generate_module():
         tls_key=dict(required=False, type='str'),
         tls_cacert=dict(required=False, type='str'),
         volumes=dict(required=False, type='list'),
-        volumes_from=dict(required=False, type='list')
+        volumes_from=dict(required=False, type='list'),
+        dimensions=dict(required=False, type='dict', default=dict())
     )
     required_if = [
         ['action', 'pull_image', ['image']],

ansible/roles/nova/defaults/main.yml

@@ -21,6 +21,7 @@ nova_services:
       - "nova_compute:/var/lib/nova/"
       - "{% if enable_cinder_backend_nfs | bool %}/var/lib/nova/mnt:/var/lib/nova/mnt:shared{% endif %}"
       - "nova_libvirt_qemu:/etc/libvirt/qemu"
+    dimensions: "{{ nova_libvirt_dimensions }}"
   nova-ssh:
     container_name: "nova_ssh"
     group: "compute"
@@ -32,6 +33,7 @@ nova_services:
       - "kolla_logs:/var/log/kolla"
       - "nova_compute:/var/lib/nova"
       - "{% if enable_cinder_backend_nfs | bool %}/var/lib/nova/mnt:/var/lib/nova/mnt:shared{% endif %}"
+    dimensions: "{{ nova_ssh_dimensions }}"
   placement-api:
     container_name: "placement_api"
     group: "placement-api"
@@ -41,6 +43,7 @@ nova_services:
       - "{{ node_config_directory }}/placement-api/:{{ container_config_directory }}/:ro"
       - "/etc/localtime:/etc/localtime:ro"
       - "kolla_logs:/var/log/kolla/"
+    dimensions: "{{ placement_api_dimensions }}"
   nova-api:
     container_name: "nova_api"
     group: "nova-api"
@@ -52,6 +55,7 @@ nova_services:
       - "/etc/localtime:/etc/localtime:ro"
       - "/lib/modules:/lib/modules:ro"
       - "kolla_logs:/var/log/kolla/"
+    dimensions: "{{ nova_api_dimensions }}"
   nova-consoleauth:
     container_name: "nova_consoleauth"
     group: "nova-consoleauth"
@@ -61,6 +65,7 @@ nova_services:
       - "{{ node_config_directory }}/nova-consoleauth/:{{ container_config_directory }}/:ro"
       - "/etc/localtime:/etc/localtime:ro"
       - "kolla_logs:/var/log/kolla/"
+    dimensions: "{{ nova_consoleauth_dimensions }}"
   nova-novncproxy:
     container_name: "nova_novncproxy"
     group: "nova-novncproxy"
@@ -70,6 +75,7 @@ nova_services:
       - "{{ node_config_directory }}/nova-novncproxy/:{{ container_config_directory }}/:ro"
       - "/etc/localtime:/etc/localtime:ro"
       - "kolla_logs:/var/log/kolla/"
+    dimensions: "{{ nova_novncproxy_dimensions }}"
   nova-scheduler:
     container_name: "nova_scheduler"
     group: "nova-scheduler"
@@ -79,6 +85,7 @@ nova_services:
       - "{{ node_config_directory }}/nova-scheduler/:{{ container_config_directory }}/:ro"
       - "/etc/localtime:/etc/localtime:ro"
       - "kolla_logs:/var/log/kolla/"
+    dimensions: "{{ nova_scheduler_dimensions }}"
   nova-spicehtml5proxy:
     container_name: "nova_spicehtml5proxy"
     group: "nova-spicehtml5proxy"
@@ -88,6 +95,7 @@ nova_services:
       - "{{ node_config_directory }}/nova-spicehtml5proxy/:{{ container_config_directory }}/:ro"
       - "/etc/localtime:/etc/localtime:ro"
       - "kolla_logs:/var/log/kolla/"
+    dimensions: "{{ nova_spicehtml5proxy_dimensions }}"
   nova-serialproxy:
     container_name: "nova_serialproxy"
     group: "nova-serialproxy"
@@ -97,6 +105,7 @@ nova_services:
       - "{{ node_config_directory }}/nova-serialproxy/:{{ container_config_directory }}/:ro"
       - "/etc/localtime:/etc/localtime:ro"
       - "kolla_logs:/var/log/kolla/"
+    dimensions: "{{ nova_serialproxy_dimensions }}"
   nova-conductor:
     container_name: "nova_conductor"
     group: "nova-conductor"
@@ -106,6 +115,7 @@ nova_services:
       - "{{ node_config_directory }}/nova-conductor/:{{ container_config_directory }}/:ro"
       - "/etc/localtime:/etc/localtime:ro"
       - "kolla_logs:/var/log/kolla/"
+    dimensions: "{{ nova_conductor_dimensions }}"
   nova-compute:
     container_name: "nova_compute"
     group: "compute"
@@ -126,6 +136,7 @@ nova_services:
       - "libvirtd:/var/lib/libvirt"
       - "nova_compute:/var/lib/nova/"
       - "{% if enable_cinder_backend_nfs | bool %}/var/lib/nova/mnt:/var/lib/nova/mnt:shared{% endif %}"
+    dimensions: "{{ nova_compute_dimensions }}"
   nova-compute-ironic:
     container_name: "nova_compute_ironic"
     group: "nova-compute-ironic"
@@ -135,6 +146,7 @@ nova_services:
       - "{{ node_config_directory }}/nova-compute-ironic/:{{ container_config_directory }}/:ro"
       - "/etc/localtime:/etc/localtime:ro"
       - "kolla_logs:/var/log/kolla/"
+    dimensions: "{{ nova_compute_ironic_dimensions }}"
 
 ####################
 # Ceph
@@ -232,6 +244,19 @@ placement_api_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ d
 placement_api_tag: "{{ nova_tag }}"
 placement_api_image_full: "{{ placement_api_image }}:{{ placement_api_tag }}"
 
+nova_libvirt_dimensions: "{{ default_container_dimensions }}"
+nova_ssh_dimensions: "{{ default_container_dimensions }}"
+placement_api_dimensions: "{{ default_container_dimensions }}"
+nova_api_dimensions: "{{ default_container_dimensions }}"
+nova_consoleauth_dimensions: "{{ default_container_dimensions }}"
+nova_novncproxy_dimensions: "{{ default_container_dimensions }}"
+nova_scheduler_dimensions: "{{ default_container_dimensions }}"
+nova_spicehtml5proxy_dimensions: "{{ default_container_dimensions }}"
+nova_serialproxy_dimensions: "{{ default_container_dimensions }}"
+nova_conductor_dimensions: "{{ default_container_dimensions }}"
+nova_compute_dimensions: "{{ default_container_dimensions }}"
+nova_compute_ironic_dimensions: "{{ default_container_dimensions }}"
+
 ####################
 # OpenStack
 ####################

ansible/roles/nova/handlers/main.yml

@@ -14,6 +14,7 @@
     pid_mode: "{{ service.pid_mode | default('') }}"
     privileged: "{{ service.privileged | default(False) }}"
     volumes: "{{ service.volumes|reject('equalto', '')|list }}"
+    dimensions: "{{ service.dimensions }}"
   when:
     - kolla_action != "config"
     - inventory_hostname in groups[service.group]
@@ -37,6 +38,7 @@
     pid_mode: "{{ service.pid_mode | default('') }}"
     privileged: "{{ service.privileged | default(False) }}"
     volumes: "{{ service.volumes|reject('equalto', '')|list }}"
+    dimensions: "{{ service.dimensions }}"
   register: restart_nova_libvirt
   # NOTE(Jeffrey4l): retry 5 to remove nova_libvirt container because when
   # guests running, nova_libvirt will raise error even though it is removed.
@@ -65,6 +67,7 @@
     name: "{{ service.container_name }}"
     image: "{{ service.image }}"
     volumes: "{{ service.volumes|reject('equalto', '')|list }}"
+    dimensions: "{{ service.dimensions }}"
   when:
     - kolla_action != "config"
     - inventory_hostname in groups[service.group]
@@ -91,6 +94,7 @@
     image: "{{ service.image }}"
     privileged: "{{ service.privileged | default(False) }}"
     volumes: "{{ service.volumes|reject('equalto', '')|list }}"
+    dimensions: "{{ service.dimensions }}"
   when:
     - kolla_action != "config"
     - inventory_hostname in groups[service.group]
@@ -116,6 +120,7 @@
     image: "{{ service.image }}"
     privileged: "{{ service.privileged | default(False) }}"
     volumes: "{{ service.volumes|reject('equalto', '')|list }}"
+    dimensions: "{{ service.dimensions }}"
   when:
     - kolla_action != "config"
     - inventory_hostname in groups[service.group]
@@ -141,6 +146,7 @@
     image: "{{ service.image }}"
     privileged: "{{ service.privileged | default(False) }}"
     volumes: "{{ service.volumes|reject('equalto', '')|list }}"
+    dimensions: "{{ service.dimensions }}"
   when:
     - kolla_action != "config"
     - inventory_hostname in groups[service.group]
@@ -167,6 +173,7 @@
     image: "{{ service.image }}"
     privileged: "{{ service.privileged | default(False) }}"
     volumes: "{{ service.volumes|reject('equalto', '')|list }}"
+    dimensions: "{{ service.dimensions }}"
   when:
     - kolla_action != "config"
     - inventory_hostname in groups[service.group]
@@ -192,6 +199,7 @@
     image: "{{ service.image }}"
     privileged: "{{ service.privileged | default(False) }}"
     volumes: "{{ service.volumes|reject('equalto', '')|list }}"
+    dimensions: "{{ service.dimensions }}"
   when:
     - kolla_action != "config"
     - inventory_hostname in groups[service.group]
@@ -217,6 +225,7 @@
     image: "{{ service.image }}"
     privileged: "{{ service.privileged | default(False) }}"
     volumes: "{{ service.volumes|reject('equalto', '')|list }}"
+    dimensions: "{{ service.dimensions }}"
   when:
     - kolla_action != "config"
     - inventory_hostname in groups[service.group]
@@ -242,6 +251,7 @@
     image: "{{ service.image }}"
     privileged: "{{ service.privileged | default(False) }}"
     volumes: "{{ service.volumes|reject('equalto', '')|list }}"
+    dimensions: "{{ service.dimensions }}"
   when:
     - kolla_action != "config"
     - inventory_hostname in groups[service.group]
@@ -269,6 +279,7 @@
     privileged: "{{ service.privileged | default(False) }}"
     ipc_mode: "{{ service.ipc_mode | default(omit) }}"
     volumes: "{{ service.volumes|reject('equalto', '')|list }}"
+    dimensions: "{{ service.dimensions }}"
   when:
     - kolla_action != "config"
     - inventory_hostname in groups[service.group]
@@ -295,6 +306,7 @@
     image: "{{ service.image }}"
     privileged: "{{ service.privileged | default(False) }}"
     volumes: "{{ service.volumes|reject('equalto', '')|list }}"
+    dimensions: "{{ service.dimensions }}"
   when:
     - kolla_action != "config"
     - inventory_hostname in groups[service.group]

doc/source/reference/index.rst

@@ -27,3 +27,4 @@ Projects Deployment References
    tacker-guide
    xenserver-guide
    horizon-guide
+   resource-constraints

doc/source/reference/resource-constraints.rst

@@ -0,0 +1,84 @@
+.. _resource-constraints:
+
+=====================================
+Resource Constraints in Kolla Ansible
+=====================================
+
+Overview
+~~~~~~~~
+
+Since the Rocky release it is possible to restrict
+the resource usage of deployed containers.
+The following components support this feature:
+
+* Nova
+
+In Kolla Ansible,
+container resources to be constrained are referred to as dimensions.
+
+The `Docker documentation <https://docs.docker.com/config/containers/resource_constraints/>`__
+provides information on container resource constraints.
+The resources currently supported by Kolla Ansible are:
+
+.. code-block:: console
+
+    cpu_period
+    cpu_quota
+    cpu_shares
+    cpuset_cpus
+    cpuset_mems
+    mem_limit
+    mem_reservation
+    memswap_limit
+    kernel_memory
+    blkio_weight
+
+.. end
+
+Pre-deployment Configuration
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Dimensions are defined as a mapping from a Docker resource name
+(e.g. ``cpu_period``) to a string constraint value.
+
+The variable ``default_container_dimensions`` sets the default dimensions
+for all supported containers, and by default these are unconstrained.
+
+Each supported container has an associated variable,
+``<container name>_dimensions``, that can be used to set the resources
+for the container. For example, dimensions for the ``nova_libvirt``
+container are set via the variable ``nova_libvirt_dimensions``.
+
+For example,
+to constrain the number of CPUs that may be used by all supported containers,
+add the following to the dimensions options section in
+``/etc/kolla/globals.yml``:
+
+.. code-block:: yaml
+
+   default_container_dimensions:
+     cpuset_cpus: "1"
+
+.. end
+
+For example, to constrain the number of CPUs that may be used by
+the ``nova_libvirt`` container, add the following to the dimensions
+options section in ``/etc/kolla/globals.yml``:
+
+.. code-block:: yaml
+
+   nova_libvirt_dimensions:
+     cpuset_cpus: "2"
+
+.. end
+
+Deployment
+~~~~~~~~~~
+
+To deploy resource constrained containers, run the deployment as usual:
+
+.. code-block:: console
+
+  $ kolla-ansible deploy -i /path/to/inventory
+
+.. end

etc/kolla/globals.yml

@@ -113,6 +113,28 @@ kolla_internal_vip_address: "10.10.10.254"
 # Arbitrary unique number from 0..255
 #keepalived_virtual_router_id: "51"
 
+###################
+# Dimension options
+###################
+# This is to provide an extra option to deploy containers with Resource constraints.
+# We call it dimensions here.
+# The dimensions for each container are defined by a mapping, where each dimension value should be a
+# string.
+# Reference_Docs
+# https://docs.docker.com/config/containers/resource_constraints/
+# eg:
+# <container_name>_dimensions:
+#    blkio_weight:
+#    cpu_period:
+#    cpu_quota:
+#    cpu_shares:
+#    cpuset_cpus:
+#    cpuset_mems:
+#    mem_limit:
+#    mem_reservation:
+#    memswap_limit:
+#    kernel_memory:
+
 
 #############
 # TLS options

tests/test_kolla_docker.py

@@ -83,7 +83,8 @@ class ModuleArgsTest(base.BaseTestCase):
             tls_key=dict(required=False, type='str'),
             tls_cacert=dict(required=False, type='str'),
             volumes=dict(required=False, type='list'),
-            volumes_from=dict(required=False, type='list')
+            volumes_from=dict(required=False, type='list'),
+            dimensions=dict(required=False, type='dict', default=dict())
             )
         required_if = [
             ['action', 'pull_image', ['image']],
@@ -191,14 +192,38 @@ class TestContainer(base.BaseTestCase):
         super(TestContainer, self).setUp()
         self.fake_data = copy.deepcopy(FAKE_DATA)
 
-    def test_create_container(self):
+    def test_create_container_without_dimensions(self):
         self.dw = get_DockerWorker(self.fake_data['params'])
         self.dw.dc.create_host_config = mock.MagicMock(
             return_value=self.fake_data['params']['host_config'])
         self.dw.create_container()
         self.assertTrue(self.dw.changed)
+
+    def test_create_container_with_dimensions(self):
+        self.fake_data['params']['dimensions'] = {'blkio_weight': 10}
+        self.dw = get_DockerWorker(self.fake_data['params'])
+        self.dw.dc.create_host_config = mock.MagicMock(
+            return_value=self.fake_data['params']['host_config'])
+        self.dw.create_container()
+        self.assertTrue(self.dw.changed)
+        self.fake_data['params'].pop('dimensions')
+        self.fake_data['params']['host_config']['blkio_weight'] = '10'
         self.dw.dc.create_container.assert_called_once_with(
             **self.fake_data['params'])
+        self.dw.dc.create_host_config.assert_called_with(
+            cap_add=None, network_mode='host', ipc_mode=None,
+            pid_mode=None, volumes_from=None, blkio_weight=10,
+            security_opt=None, privileged=None)
+
+    def test_create_container_wrong_dimensions(self):
+        self.fake_data['params']['dimensions'] = {'random': 10}
+        self.dw = get_DockerWorker(self.fake_data['params'])
+        self.dw.dc.create_host_config = mock.MagicMock(
+            return_value=self.fake_data['params']['host_config'])
+        self.dw.create_container()
+        self.dw.module.exit_json.assert_called_once_with(
+            failed=True, msg=repr("Unsupported dimensions"),
+            unsupported_dimensions=set(['random']))
 
     def test_start_container_without_pull(self):
         self.fake_data['params'].update({'auth_username': 'fake_user',