octavia: support tenant management network

- bind health manager port to ovs
- create a octavia-interface.service (used for creating health manager
  interface after reboot)
- used for ci, test or poc

Implements: blueprint implement-automatic-deploy-of-octavia

Change-Id: I8ac30dea6c8402d97189c6b05032c608936b0ff2
Co-Authored-By: wu.chunyang <wuchunyang@yovole.com>

ansible/group_vars/all.yml

@@ -223,7 +223,7 @@ swift_storage_interface: "{{ storage_interface }}"
 swift_replication_interface: "{{ swift_storage_interface }}"
 migration_interface: "{{ api_interface }}"
 tunnel_interface: "{{ network_interface }}"
-octavia_network_interface: "{{ api_interface }}"
+octavia_network_interface: "{{ 'o-hm0' if octavia_network_type == 'tenant' else api_interface }}"
 bifrost_network_interface: "{{ network_interface }}"
 dns_interface: "{{ network_interface }}"
 dpdk_tunnel_interface: "{{ neutron_external_interface }}"
@@ -1157,6 +1157,19 @@ swift_public_endpoint: "{{ public_protocol }}://{{ swift_external_fqdn | put_add
 ##########
 # Octavia
 ##########
+# Whether to run Kolla-Ansible's automatic configuration for Octavia.
+# NOTE: if you upgrade from Ussuri, you must set `octavia_auto_configure` to `no`
+# and keep your other Octavia config like before.
+octavia_auto_configure: yes
+
+# Octavia network type options are [ tenant, provider ]
+# * tenant indicates that we will create a tenant network and a network
+#   interface on the Octavia worker nodes for communication with amphorae.
+# * provider indicates that we will create a flat or vlan provider network.
+#   In this case octavia_network_interface should be set to a network interface
+#   on the Octavia woker nodes on the same provider network.
+octavia_network_type: "provider"
+
 octavia_admin_endpoint: "{{ admin_protocol }}://{{ octavia_internal_fqdn | put_address_in_context('url') }}:{{ octavia_api_port }}"
 octavia_internal_endpoint: "{{ internal_protocol }}://{{ octavia_internal_fqdn | put_address_in_context('url') }}:{{ octavia_api_port }}"
 octavia_public_endpoint: "{{ public_protocol }}://{{ octavia_external_fqdn | put_address_in_context('url') }}:{{ octavia_api_port }}"

ansible/roles/destroy/tasks/cleanup_host.yml

@@ -15,3 +15,26 @@
     kolla_external_vip_address: "{{ kolla_external_vip_address }}"
     kolla_dev_repos_directory: "{{ kolla_dev_repos_directory }}"
     destroy_include_dev: "{{ destroy_include_dev }}"
+
+- block:
+    - name: disable octavia-interface service
+      service:
+        name: octavia-interface
+        enabled: no
+        state: stopped
+      failed_when: false
+
+    - name: remove octavia-interface service file
+      file:
+        path: /etc/systemd/system/octavia-interface.service
+        state: absent
+
+    - name: remove dhclient.conf
+      file:
+        path: /etc/dhcp/octavia-dhclient.conf
+        state: absent
+  when:
+    - enable_octavia | bool
+    - octavia_auto_configure | bool
+    - octavia_network_type == 'tenant'
+    - inventory_hostname in groups['octavia-health-manager']

ansible/roles/octavia/defaults/main.yml

@@ -161,15 +161,11 @@ octavia_source_version: "{{ kolla_source_version }}"
 octavia_amp_ssh_key_name: "octavia_ssh_key"
 octavia_amp_listen_port: "9443"
 octavia_amp_image_tag: "amphora"
+octavia_network_type: "provider"
 
 # Load balancer topology options are [ SINGLE, ACTIVE_STANDBY ]
 octavia_loadbalancer_topology: "SINGLE"
 
-# Whether to run Kolla Ansible's automatic configuration for Octavia.
-# NOTE: if you upgrade from Ussuri, you must set `octavia_auto_configure` to `no`
-# and keep your other Octavia config like before.
-octavia_auto_configure: yes
-
 # OpenStack auth used when registering resources for Octavia.
 octavia_user_auth:
   auth_url: "{{ keystone_admin_url }}"
@@ -197,9 +193,11 @@ octavia_amp_flavor:
   disk: 5
 
 # Octavia security groups. lb-mgmt-sec-grp is for amphorae.
+# lb-health-mgr-sec-grp is used for health manager ports.
 octavia_amp_security_groups:
   mgmt-sec-grp:
     name: "lb-mgmt-sec-grp"
+    enabled: true
     rules:
       - protocol: icmp
       - protocol: tcp
@@ -208,6 +206,13 @@ octavia_amp_security_groups:
       - protocol: tcp
         src_port: "{{ octavia_amp_listen_port }}"
         dst_port: "{{ octavia_amp_listen_port }}"
+  health-mgr-sec-grp:
+    name: "lb-health-mgr-sec-grp"
+    enabled: "{{ true if octavia_network_type == 'tenant' else false }}"
+    rules:
+      - protocol: udp
+        src_port: "{{ octavia_health_manager_port }}"
+        dst_port: "{{ octavia_health_manager_port }}"
 
 # Octavia management network.
 # See os_network and os_subnet for details. Supported parameters:

ansible/roles/octavia/tasks/config-host.yml

@@ -0,0 +1,6 @@
+---
+- include_tasks: hm-interface.yml
+  when:
+    - octavia_auto_configure | bool
+    - octavia_network_type == "tenant"
+    - inventory_hostname in groups[octavia_services['octavia-health-manager']['group']]

ansible/roles/octavia/tasks/deploy.yml

@@ -4,6 +4,8 @@
 - include_tasks: prepare.yml
   when: octavia_auto_configure | bool
 
+- import_tasks: config-host.yml
+
 - import_tasks: config.yml
 
 - import_tasks: check-containers.yml

ansible/roles/octavia/tasks/hm-interface.yml

@@ -0,0 +1,98 @@
+---
+- name: Create ports for Octavia health-manager nodes
+  become: true
+  kolla_toolbox:
+    module_name: os_port
+    module_args:
+      auth: "{{ octavia_user_auth }}"
+      cacert: "{{ openstack_cacert }}"
+      endpoint_type: "{{ openstack_interface }}"
+      region_name: "{{ openstack_region_name }}"
+      state: present
+      network: "{{ octavia_amp_network['name'] }}"
+      security_groups: "{{ octavia_amp_security_groups['health-mgr-sec-grp']['name'] }}"
+      device_owner: 'Octavia:health-mgr'
+      name: "octavia-listen-port-{{ ansible_nodename }}"
+  register: port_info
+
+# ansible os_port module does not support 'host' parameter, but we need set the port's host
+# value to {{ ansible_nodename }}, once os_port support this parameter, remove the task below
+# https://docs.ansible.com/ansible/latest/modules/os_port_module.html#parameters
+- name: Update Octavia health manager port host_id
+  become: True
+  vars:
+    port_id: "{{ port_info.id }}"
+  command: >
+    docker exec kolla_toolbox openstack
+    --os-interface {{ openstack_interface }}
+    --os-auth-url {{ octavia_user_auth.auth_url }}
+    --os-identity-api-version 3
+    --os-project-domain-name {{ octavia_user_auth.domain_name }}
+    --os-project-name {{ octavia_user_auth.project_name }}
+    --os-region-name {{ openstack_region_name }}
+    --os-username {{ octavia_user_auth.username }}
+    --os-password {{ octavia_user_auth.password }}
+    {% if openstack_cacert != '' %}--os-cacert {{ openstack_cacert }} {% endif %}
+    port set --host {{ ansible_nodename }} {{ port_id }}
+  when:
+    - port_info.changed
+
+- name: Add Octavia port to openvswitch br-int
+  vars:
+    port_mac: "{{ port_info.port.mac_address }}"
+    port_id: "{{ port_info.id }}"
+  become: True
+  command: >
+    docker exec openvswitch_vswitchd ovs-vsctl --may-exist \
+    add-port br-int {{ octavia_network_interface }} \
+    -- set Interface {{ octavia_network_interface }} type=internal \
+    -- set Interface {{ octavia_network_interface }} external-ids:iface-status=active \
+    -- set Interface {{ octavia_network_interface }} external-ids:attached-mac={{ port_mac }} \
+    -- set Interface {{ octavia_network_interface }} external-ids:iface-id={{ port_id }} \
+    -- set Interface {{ octavia_network_interface }} external-ids:skip_cleanup=true
+
+- name: Create octavia dhclient conf
+  become: true
+  copy:
+    content: |
+      request subnet-mask,broadcast-address,interface-mtu;
+      do-forward-updates false;
+    dest: /etc/dhcp/octavia-dhclient.conf
+    mode: 0664
+
+- name: Create octavia-interface service
+  become: True
+  template:
+    src: octavia-interface.service.j2
+    dest: /etc/systemd/system/octavia-interface.service
+  register: octavia_interface
+
+- name: Reload octavia-interface.service if required
+  become: True
+  systemd:
+    name: octavia-interface
+    daemon_reload: yes
+  when: octavia_interface.changed
+
+- name: Enable and start octavia-interface.service
+  become: True
+  service:
+    name: octavia-interface
+    enabled: yes
+    state: started
+
+- name: Wait for interface {{ octavia_network_interface }} ip appear
+  vars:
+    port_ip: "{{ port_info.port.fixed_ips[0].ip_address }}"
+  command: ip address show dev {{ octavia_network_interface }}
+  changed_when: false
+  register: ip_info
+  until: ip_info.stdout.find(port_ip) != -1
+  retries: 5
+  delay: 2
+
+# NOTE(wuchunyang): we have gathered facts at first, but here we add a new
+# network device(o-hm0) which is not collected, so we need gather facts again
+- name: Gather facts
+  setup:
+  when: octavia_interface.changed

ansible/roles/octavia/tasks/precheck.yml

@@ -55,3 +55,14 @@
     - client_ca.cert.pem
     - server_ca.cert.pem
     - server_ca.key.pem
+
+- name: Fail when Neutron plugin agent is not supported
+  fail:
+    msg: >
+      Neutron plugin agent {{ neutron_plugin_agent }} is not supported when
+      octavia_network_type is tenant, only openvswitch is supported currently
+  run_once: True
+  when:
+    - octavia_auto_configure | bool
+    - octavia_network_type == "tenant"
+    - neutron_plugin_agent != 'openvswitch'

ansible/roles/octavia/tasks/prepare.yml

@@ -65,6 +65,7 @@
   loop: "{{ octavia_amp_security_groups.values() | list }}"
   loop_control:
     label: "{{ item.name }}"
+  when: item.enabled | bool
   run_once: True
   delegate_to: "{{ groups['octavia-api'][0] }}"
   register: sec_grp_info
@@ -85,6 +86,7 @@
   with_subelements:
     - "{{ octavia_amp_security_groups }}"
     - rules
+  when: item.0.enabled | bool
   run_once: True
   delegate_to: "{{ groups['octavia-api'][0] }}"
 

ansible/roles/octavia/templates/octavia-interface.service.j2

@@ -0,0 +1,14 @@
+[Unit]
+Description=Octavia Interface Creator
+Requires=docker.service
+After=docker.service
+
+[Service]
+Type=oneshot
+RemainAfterExit=true
+ExecStartPre=/usr/bin/sudo /sbin/ip link set dev {{ octavia_network_interface }} address {{ port_info.port.mac_address }}
+ExecStart=/usr/bin/sudo /sbin/dhclient -v {{ octavia_network_interface }} -cf /etc/dhcp/octavia-dhclient.conf
+ExecStop=/usr/bin/sudo /sbin/dhclient -r {{ octavia_network_interface }}
+
+[Install]
+WantedBy=multi-user.target

ansible/roles/octavia/templates/octavia.conf.j2

@@ -88,7 +88,7 @@ amp_flavor_id = {{ octavia_amp_flavor_id }}
 {% else %}
 amp_image_owner_id = {{ project_info.openstack_projects.0.id }}
 amp_boot_network_list = {{ network_info.id }}
-amp_secgroup_list = {{ (sec_grp_info.results | selectattr('secgroup.name', 'equalto', octavia_amp_security_groups['mgmt-sec-grp'].name) | list).0.secgroup.id }}
+amp_secgroup_list = {{ (sec_grp_info.results | selectattr('item.name', 'equalto', octavia_amp_security_groups['mgmt-sec-grp'].name) | list).0.secgroup.id }}
 amp_flavor_id = {{ amphora_flavor_info.id }}
 {% endif %}