Deploy neutron-mlnx-agent and neutron-eswitchd containers

Change-Id: I173669bdf92b1f2ea98907ba16808ca3c914944c

ansible/group_vars/all.yml

@@ -625,6 +625,7 @@ enable_multipathd: "no"
 enable_murano: "no"
 enable_neutron_vpnaas: "no"
 enable_neutron_sriov: "no"
+enable_neutron_mlnx: "no"
 enable_neutron_dvr: "no"
 enable_neutron_qos: "no"
 enable_neutron_agent_ha: "no"

ansible/roles/neutron/defaults/main.yml

@@ -97,6 +97,21 @@ neutron_services:
     host_in_groups: "{{ inventory_hostname in groups['compute'] }}"
     volumes: "{{ neutron_sriov_agent_default_volumes + neutron_sriov_agent_extra_volumes }}"
     dimensions: "{{ neutron_sriov_agent_dimensions }}"
+  neutron-mlnx-agent:
+    container_name: "neutron_mlnx_agent"
+    image: "{{ neutron_mlnx_agent_image_full }}"
+    enabled: "{{ enable_neutron_mlnx | bool and neutron_plugin_agent not in ['vmware_nsxv', 'vmware_nsxv3' ] }}"
+    host_in_groups: "{{ inventory_hostname in groups['compute'] }}"
+    volumes: "{{ neutron_mlnx_agent_default_volumes + neutron_mlnx_agent_extra_volumes }}"
+    dimensions: "{{ neutron_mlnx_agent_dimensions }}"
+  neutron-eswitchd:
+    container_name: "neutron_eswitchd"
+    image: "{{ neutron_eswitchd_image_full }}"
+    privileged: True
+    enabled: "{{ enable_neutron_mlnx | bool and neutron_plugin_agent not in ['vmware_nsxv', 'vmware_nsxv3' ] }}"
+    host_in_groups: "{{ inventory_hostname in groups['compute'] }}"
+    volumes: "{{ neutron_eswitchd_default_volumes + neutron_eswitchd_extra_volumes }}"
+    dimensions: "{{ neutron_eswitchd_dimensions }}"
   neutron-metadata-agent:
     container_name: "neutron_metadata_agent"
     image: "{{ neutron_metadata_agent_image_full }}"
@@ -180,6 +195,14 @@ neutron_sriov_agent_image: "{{ docker_registry ~ '/' if docker_registry else ''
 neutron_sriov_agent_tag: "{{ neutron_tag }}"
 neutron_sriov_agent_image_full: "{{ neutron_sriov_agent_image }}:{{ neutron_sriov_agent_tag }}"
 
+neutron_mlnx_agent_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ neutron_install_type }}-neutron-mlnx-agent"
+neutron_mlnx_agent_tag: "{{ neutron_tag }}"
+neutron_mlnx_agent_image_full: "{{ neutron_mlnx_agent_image }}:{{ neutron_mlnx_agent_tag }}"
+
+neutron_eswitchd_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ neutron_install_type }}-neutron-mlnx-agent"
+neutron_eswitchd_tag: "{{ neutron_mlnx_agent_tag }}"
+neutron_eswitchd_image_full: "{{ neutron_eswitchd_image }}:{{ neutron_eswitchd_tag }}"
+
 neutron_linuxbridge_agent_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ neutron_install_type }}-neutron-linuxbridge-agent"
 neutron_linuxbridge_agent_tag: "{{ neutron_tag }}"
 neutron_linuxbridge_agent_image_full: "{{ neutron_linuxbridge_agent_image }}:{{ neutron_linuxbridge_agent_tag }}"
@@ -221,6 +244,8 @@ neutron_agent_dimensions: "{{ default_container_dimensions }}"
 neutron_dhcp_agent_dimensions: "{{ neutron_agent_dimensions }}"
 neutron_l3_agent_dimensions: "{{ neutron_agent_dimensions }}"
 neutron_sriov_agent_dimensions: "{{ neutron_agent_dimensions }}"
+neutron_mlnx_agent_dimensions: "{{ neutron_agent_dimensions }}"
+neutron_eswitchd_dimensions: "{{ neutron_agent_dimensions }}"
 neutron_linuxbridge_agent_dimensions: "{{ neutron_agent_dimensions }}"
 neutron_metadata_agent_dimensions: "{{ neutron_agent_dimensions }}"
 neutron_ovn_metadata_agent_dimensions: "{{ neutron_agent_dimensions }}"
@@ -255,6 +280,19 @@ neutron_sriov_agent_default_volumes:
   - "{{ '/etc/timezone:/etc/timezone:ro' if ansible_os_family == 'Debian' else '' }}"
   - "kolla_logs:/var/log/kolla/"
   - "{{ kolla_dev_repos_directory ~ '/neutron/neutron:/var/lib/kolla/venv/lib/python' ~ distro_python_version ~ '/site-packages/neutron' if neutron_dev_mode | bool else '' }}"
+neutron_mlnx_agent_default_volumes:
+  - "{{ node_config_directory }}/neutron-mlnx-agent/:{{ container_config_directory }}/:ro"
+  - "/etc/localtime:/etc/localtime:ro"
+  - "{{ '/etc/timezone:/etc/timezone:ro' if kolla_base_distro in ['debian', 'ubuntu'] else '' }}"
+  - "kolla_logs:/var/log/kolla/"
+  - "{{ kolla_dev_repos_directory ~ '/neutron/neutron:/var/lib/kolla/venv/lib/python' ~ distro_python_version ~ '/site-packages/neutron' if neutron_dev_mode | bool else '' }}"
+neutron_eswitchd_default_volumes:
+  - "{{ node_config_directory }}/neutron-eswitchd/:{{ container_config_directory }}/:ro"
+  - "/etc/localtime:/etc/localtime:ro"
+  - "/run/libvirt:/run/libvirt:ro"
+  - "{{ '/etc/timezone:/etc/timezone:ro' if kolla_base_distro in ['debian', 'ubuntu'] else '' }}"
+  - "kolla_logs:/var/log/kolla/"
+  - "{{ kolla_dev_repos_directory ~ '/neutron/neutron:/var/lib/kolla/venv/lib/python' ~ distro_python_version ~ '/site-packages/neutron' if neutron_dev_mode | bool else '' }}"
 neutron_linuxbridge_agent_default_volumes:
   - "{{ node_config_directory }}/neutron-linuxbridge-agent/:{{ container_config_directory }}/:ro"
   - "/etc/localtime:/etc/localtime:ro"
@@ -317,6 +355,8 @@ neutron_extra_volumes: "{{ default_extra_volumes }}"
 neutron_dhcp_agent_extra_volumes: "{{ neutron_extra_volumes }}"
 neutron_l3_agent_extra_volumes: "{{ neutron_extra_volumes }}"
 neutron_sriov_agent_extra_volumes: "{{ neutron_extra_volumes }}"
+neutron_mlnx_agent_extra_volumes: "{{ neutron_extra_volumes }}"
+neutron_eswitchd_extra_volumes: "{{ neutron_extra_volumes }}"
 neutron_linuxbridge_agent_extra_volumes: "{{ neutron_extra_volumes }}"
 neutron_metadata_agent_extra_volumes: "{{ neutron_extra_volumes }}"
 neutron_ovn_metadata_agent_extra_volumes: "{{ neutron_extra_volumes }}"
@@ -357,6 +397,8 @@ mechanism_drivers:
     enabled: "{{ neutron_plugin_agent == 'linuxbridge' }}"
   - name: "openvswitch"
     enabled: "{{ neutron_plugin_agent == 'openvswitch' }}"
+  - name: "mlnx_infiniband"
+    enabled: "{{ enable_neutron_mlnx | bool }}"
   - name: "baremetal"
     enabled: "{{ enable_ironic_neutron_agent | bool }}"
   - name: "l2population"
@@ -498,6 +540,14 @@ infoblox_admin_user_name: "admin"
 infoblox_wapi_version: "2.3"
 infoblox_wapi_max_results: "-50000"
 
+######################
+# Mellanox Agent
+######################
+# "ibphysnet" and "ib0" used here as placeholders
+neutron_mlnx_physnet_mappings:
+  ibphysnet: "ib0"
+neutron_mlnx_physnets: "{{ neutron_mlnx_physnet_mappings.items() | map('join', ':') | join(',') }}"
+
 ######################
 # Notification Drivers
 ######################

ansible/roles/neutron/handlers/main.yml

@@ -115,6 +115,38 @@
   when:
     - kolla_action != "config"
 
+- name: Restart neutron-mlnx-agent container
+  vars:
+    service_name: "neutron-mlnx-agent"
+    service: "{{ neutron_services[service_name] }}"
+  become: true
+  kolla_docker:
+    action: "recreate_or_restart_container"
+    common_options: "{{ docker_common_options }}"
+    name: "{{ service.container_name }}"
+    image: "{{ service.image }}"
+    volumes: "{{ service.volumes|reject('equalto', '')|list }}"
+    dimensions: "{{ service.dimensions }}"
+    privileged: "{{ service.privileged | default(False) }}"
+  when:
+    - kolla_action != "config"
+
+- name: Restart neutron-eswitchd container
+  vars:
+    service_name: "neutron-eswitchd"
+    service: "{{ neutron_services[service_name] }}"
+  become: true
+  kolla_docker:
+    action: "recreate_or_restart_container"
+    common_options: "{{ docker_common_options }}"
+    name: "{{ service.container_name }}"
+    image: "{{ service.image }}"
+    volumes: "{{ service.volumes|reject('equalto', '')|list }}"
+    dimensions: "{{ service.dimensions }}"
+    privileged: "{{ service.privileged | default(False) }}"
+  when:
+    - kolla_action != "config"
+
 - name: Restart neutron-metadata-agent container
   vars:
     service_name: "neutron-metadata-agent"

ansible/roles/neutron/tasks/config.yml

@@ -87,6 +87,8 @@
       - "neutron-bgp-dragent"
       - "neutron-infoblox-ipam-agent"
       - "neutron-sriov-agent"
+      - "neutron-mlnx-agent"
+      - "neutron-eswitchd"
   merge_configs:
     sources:
       - "{{ role_path }}/templates/neutron.conf.j2"
@@ -183,6 +185,7 @@
     - "Restart {{ service_name }} container"
 
 - name: Copying over sriov_agent.ini
+  become: true
   vars:
     service_name: "neutron-sriov-agent"
     neutron_sriov_agent: "{{ neutron_services[service_name] }}"
@@ -199,6 +202,42 @@
   notify:
     - "Restart {{ service_name }} container"
 
+- name: Copying over mlnx_agent.ini
+  become: true
+  vars:
+    service_name: "neutron-mlnx-agent"
+    neutron_mlnx_agent: "{{ neutron_services[service_name] }}"
+  merge_configs:
+    sources:
+      - "{{ role_path }}/templates/mlnx_agent.ini.j2"
+      - "{{ node_custom_config }}/neutron/mlnx_agent.ini"
+      - "{{ node_custom_config }}/neutron/{{ inventory_hostname }}/mlnx_agent.ini"
+    dest: "{{ node_config_directory }}/{{ service_name }}/mlnx_agent.ini"
+    mode: "0660"
+  when:
+    - neutron_mlnx_agent.enabled | bool
+    - neutron_mlnx_agent.host_in_groups | bool
+  notify:
+    - "Restart {{ service_name }} container"
+
+- name: Copying over eswitchd.conf
+  become: true
+  vars:
+    service_name: "neutron-eswitchd"
+    neutron_eswitchd: "{{ neutron_services[service_name] }}"
+  merge_configs:
+    sources:
+      - "{{ role_path }}/templates/eswitchd.conf.j2"
+      - "{{ node_custom_config }}/neutron/eswitchd.conf"
+      - "{{ node_custom_config }}/neutron/{{ inventory_hostname }}/eswitchd.conf"
+    dest: "{{ node_config_directory }}/{{ service_name }}/eswitchd.conf"
+    mode: "0660"
+  when:
+    - neutron_eswitchd.enabled | bool
+    - neutron_eswitchd.host_in_groups | bool
+  notify:
+    - "Restart {{ service_name }} container"
+
 - name: Copying over dhcp_agent.ini
   become: true
   vars:

ansible/roles/neutron/templates/eswitchd.conf

@@ -0,0 +1,2 @@
+[DAEMON]
+fabrics = {{ neutron_mlnx_physnets }}

ansible/roles/neutron/templates/mlnx_agent.ini.j2

@@ -0,0 +1,2 @@
+[eswitch]
+physical_interface_mappings  = {{ neutron_mlnx_physnets }}

ansible/roles/neutron/templates/neutron-eswitchd.json.j2

@@ -0,0 +1,29 @@
+{
+    "command": "eswitchd --config-file /etc/neutron/neutron.conf --config-file /etc/neutron/plugins/ml2/eswitchd.conf",
+    "config_files": [
+        {
+            "source": "{{ container_config_directory }}/neutron.conf",
+            "dest": "/etc/neutron/neutron.conf",
+            "owner": "neutron",
+            "perm": "0600"
+        },
+        {
+            "source": "{{ container_config_directory }}/eswitchd.conf",
+            "dest": "/etc/neutron/plugins/ml2/eswitchd.conf",
+            "owner": "neutron",
+            "perm": "0600"
+        }
+    ],
+    "permissions": [
+        {
+            "path": "/var/log/kolla/neutron",
+            "owner": "neutron:neutron",
+            "recurse": true
+        },
+        {
+            "path": "/var/lib/neutron/kolla",
+            "owner": "neutron:neutron",
+            "recurse": true
+        }
+    ]
+}

ansible/roles/neutron/templates/neutron-mlnx-agent.json.j2

@@ -0,0 +1,29 @@
+{
+    "command": "neutron-mlnx-agent --config-file /etc/neutron/neutron.conf --config-file /etc/neutron/plugins/mlnx/mlnx_agent.ini",
+    "config_files": [
+        {
+            "source": "{{ container_config_directory }}/neutron.conf",
+            "dest": "/etc/neutron/neutron.conf",
+            "owner": "neutron",
+            "perm": "0600"
+        },
+        {
+            "source": "{{ container_config_directory }}/mlnx_agent.ini",
+            "dest": "/etc/neutron/plugins/mlnx/mlnx_agent.ini",
+            "owner": "neutron",
+            "perm": "0600"
+        }
+    ],
+    "permissions": [
+        {
+            "path": "/var/log/kolla/neutron",
+            "owner": "neutron:neutron",
+            "recurse": true
+        },
+        {
+            "path": "/var/lib/neutron/kolla",
+            "owner": "neutron:neutron",
+            "recurse": true
+        }
+    ]
+}

ansible/roles/nova-cell/templates/libvirtd.conf.j2

@@ -20,3 +20,7 @@ log_level = 3
 log_outputs = "3:file:/var/log/kolla/libvirt/libvirtd.log"
 {% endif %}
 listen_addr = "{{ migration_interface_address }}"
+{% if enable_neutron_mlnx | bool %}
+# Enable read-only access to libvirt socket
+auth_unix_ro = "none"
+{% endif %}

doc/source/reference/networking/neutron.rst

@@ -80,3 +80,26 @@ To change this behaviour you need to set the following:
 
    neutron_ovn_distributed_fip: "yes"
 
+Mellanox Infiniband (ml2/mlnx)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+In order to add ``mlnx_infiniband`` to the list of mechanism driver
+for ``neutron`` to support Infiniband virtual funtions, you need to
+set the following (assuming neutron SR-IOV agent is also enabled using
+``enable_neutron_sriov`` flag):
+
+.. path /etc/kolla/globals.yml
+.. code-block:: yaml
+
+   enable_neutron_mlnx: "yes"
+
+Additionally, you will also need to provide physnet:interface mappings
+via ``neutron_mlnx_physnet_mappings`` which is presented to
+``neutron_mlnx_agent`` container via ``mlnx_agent.ini`` and
+``neutron_eswitchd`` container via ``eswitchd.conf``:
+
+.. path /etc/kolla/globals.yml
+.. code-block:: yaml
+
+   neutron_mlnx_physnet_mappings:
+     ibphysnet: "ib0"

releasenotes/notes/neutron-mlnx-agent-3e7b3aecbb298811.yaml

@@ -0,0 +1,6 @@
+---
+features:
+  - |
+    Kolla started building neutron-mlnx-agent in Ussuri cycle. Now those
+    containers can be deployed too via Kolla-Ansible. Note that
+    neutron-mlnx-agent image is also used to deploy neutron-eswitchd container.