From b356762b942690e63ca9b99dcf76cc478284e976 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Piliszek?= Date: Mon, 17 Feb 2020 18:40:15 +0100 Subject: [PATCH] CI: Refactor VXLAN overlay setup This makes it cleaner, allows reuse and outsourcing to zuul jobs and enables us to create multiple of these overlay networks for testing of more advanced scenarios. Change-Id: Id557c81f68a7f34556854e7d6efc6eddfd2e7216 --- .../tasks/main.yml | 72 +++++++++++ .../defaults/main.yml | 5 + roles/multi-node-vxlan-overlay/tasks/main.yml | 50 ++++++++ tests/pre.yml | 119 ++---------------- 4 files changed, 137 insertions(+), 109 deletions(-) create mode 100644 roles/multi-node-managed-addressing/tasks/main.yml create mode 100644 roles/multi-node-vxlan-overlay/defaults/main.yml create mode 100644 roles/multi-node-vxlan-overlay/tasks/main.yml diff --git a/roles/multi-node-managed-addressing/tasks/main.yml b/roles/multi-node-managed-addressing/tasks/main.yml new file mode 100644 index 0000000000..9235fe2df6 --- /dev/null +++ b/roles/multi-node-managed-addressing/tasks/main.yml @@ -0,0 +1,72 @@ +--- +# Not all variables have sensible defaults, let's ensure these are set. +- name: Ensure mandatory variables are defined + assert: + that: + - managed_interface_name is defined + - managed_network_prefix is defined + - managed_network_prefix_length is defined + - managed_network_address_family is defined + +- name: Set managed interface facts + set_fact: + managed_interface_address: "{{ managed_network_prefix }}{{ groups['all'].index(inventory_hostname) + 1 }}" + +- name: Add IPv4 address for managed network + become: true + vars: + managed_network_cidr: "{{ managed_interface_address }}/{{ managed_network_prefix_length }}" + # NOTE(yoctozepto): we have to compute and explicitly set the broadcast address, + # otherwise bifrost fails its pre-bootstrap sanity checks due to missing + # broadcast address as ansible picks up scope ('global') as the interface's + # broadcast address which fails checks logic + managed_network_broadcast_address: "{{ managed_network_cidr | ipaddr('broadcast') }}" + command: ip address add {{ managed_network_cidr }} broadcast {{ managed_network_broadcast_address }} dev {{ managed_interface_name }} + when: managed_network_address_family == 'ipv4' + +# NOTE(yoctozepto): IPv6 has no broadcast address, let's not create confusion by setting it +- name: Add IPv6 address for managed network + become: true + command: ip address add {{ managed_interface_address }}/{{ managed_network_prefix_length }} dev {{ managed_interface_name }} + when: managed_network_address_family == 'ipv6' + +- name: Accept traffic on the managed network (IN) + become: true + iptables: + state: present + action: insert + chain: INPUT + ip_version: "{{ managed_network_address_family }}" + in_interface: "{{ managed_interface_name }}" + jump: ACCEPT + +# NOTE(yoctozepto): the default policy is ACCEPT but it is nicer to get statistics +- name: Accept traffic on the managed network (OUT) + become: true + iptables: + state: present + action: insert + chain: OUTPUT + ip_version: "{{ managed_network_address_family }}" + out_interface: "{{ managed_interface_name }}" + jump: ACCEPT + +# NOTE(yoctozepto): IPv6 DAD may delay proper address assignment +# this task will wait until DAD is done and addresses are no longer tentative +# we assign addresses uniquely so DAD can only move it to preferred +# hence we only check whether it's no longer tentative +- name: Ensure IPv6 addresses on the managed interface are no longer tentative + become: true + command: ip -o address show tentative dev {{ managed_interface_name }} + register: tentative_addresses + until: tentative_addresses.stdout == '' + retries: 30 + delay: 2 + when: + - managed_network_address_family == 'ipv6' + +- name: Ping across the managed network + vars: + ping_command: "{{ 'ping' if managed_network_address_family == 'ipv4' else 'ping6' }}" + command: "{{ ping_command }} -c1 {{ hostvars[item].managed_interface_address }}" + with_inventory_hostnames: all diff --git a/roles/multi-node-vxlan-overlay/defaults/main.yml b/roles/multi-node-vxlan-overlay/defaults/main.yml new file mode 100644 index 0000000000..bf80bf46ff --- /dev/null +++ b/roles/multi-node-vxlan-overlay/defaults/main.yml @@ -0,0 +1,5 @@ +--- +# NOTE(yoctozepto): CI VXLAN overlay must use a different port than +# neutron-openvswitch-agent which defaults to 4789. +# Hence using port 4790 by default. +vxlan_dstport: 4790 diff --git a/roles/multi-node-vxlan-overlay/tasks/main.yml b/roles/multi-node-vxlan-overlay/tasks/main.yml new file mode 100644 index 0000000000..bc77a01c55 --- /dev/null +++ b/roles/multi-node-vxlan-overlay/tasks/main.yml @@ -0,0 +1,50 @@ +--- +# Not all variables have sensible defaults, let's ensure these are set. +- name: Ensure mandatory variables are defined + assert: + that: + - vxlan_interface_name is defined + - vxlan_vni is defined + +# We have had cases where the nodepool private IP address is not assigned, +# which causes hard to diagnose errors later on. Catch it early. +- name: Assert that the nodepool private IPv4 address is assigned + assert: + that: nodepool.private_ipv4 in ansible_all_ipv4_addresses + fail_msg: >- + The nodepool private IP address {{ nodepool.private_ipv4 }} is not assigned + +- name: Set VXLAN interface facts + set_fact: + tunnel_local_address: "{{ nodepool.private_ipv4 }}" + +- name: Create VXLAN interface + become: true + command: ip link add {{ vxlan_interface_name }} type vxlan id {{ vxlan_vni }} local {{ tunnel_local_address }} dstport {{ vxlan_dstport }} + +- name: Set VXLAN interface MTU + become: true + vars: + # Find the parent interface + parent_interface: >- + {{ ansible_interfaces | + map('extract', ansible_facts) | + selectattr('ipv4.address', 'defined') | + selectattr('ipv4.address', 'equalto', tunnel_local_address) | + first }} + # Allow 50 bytes overhead for VXLAN headers. + mtu: "{{ parent_interface.mtu | int - 50 }}" + command: ip link set {{ vxlan_interface_name }} mtu {{ mtu }} + +# emulate BUM by multiplicating traffic to unicast targets +- name: Add fdb entries for BUM traffic + become: true + vars: + dest_ip: "{{ hostvars[item].tunnel_local_address }}" + command: bridge fdb append 00:00:00:00:00:00 dev {{ vxlan_interface_name }} dst {{ dest_ip }} + with_inventory_hostnames: all + when: item != inventory_hostname + +- name: Bring VXLAN interface up + become: true + command: ip link set {{ vxlan_interface_name }} up diff --git a/tests/pre.yml b/tests/pre.yml index ac7644f779..6400a4f0d0 100644 --- a/tests/pre.yml +++ b/tests/pre.yml @@ -6,20 +6,22 @@ roles: - bindep - multi-node-firewall + - role: multi-node-vxlan-overlay + vars: + vxlan_interface_name: "{{ api_interface_name }}" + vxlan_vni: 10001 + - role: multi-node-managed-addressing + vars: + managed_interface_name: "{{ api_interface_name }}" + managed_network_prefix: "{{ api_network_prefix }}" + managed_network_prefix_length: "{{ api_network_prefix_length }}" + managed_network_address_family: "{{ address_family }}" tasks: # TODO(mnasiadka): Remove once infra merges virtualenv fixes - name: Upgrade virtualenv package command: python3 -m pip install -U virtualenv become: True - # We have had cases where the nodepool private IP address is not assigned, - # which causes hard to diagnose errors later on. Catch it early. - - name: Assert that the nodepool private IPv4 address is assigned - assert: - that: nodepool.private_ipv4 in ansible_all_ipv4_addresses - fail_msg: >- - The nodepool private IP address {{ nodepool.private_ipv4 }} is not assigned - - name: Install dbus for debian system apt: name=dbus when: @@ -48,107 +50,6 @@ name: "{{ inventory_hostname }}" become: true - # NOTE(yoctozepto): start VXLAN interface config - - - name: Set VXLAN interface facts - set_fact: - api_interface_address: "{{ api_network_prefix }}{{ groups['all'].index(inventory_hostname) + 1 }}" - api_interface_tunnel_vni: 10001 - tunnel_local_address: "{{ nodepool.private_ipv4 }}" - - # NOTE(yoctozepto): CI VXLAN must use a different port than neutron-openvswitch-agent - # which defaults to 4789 (the default is used in CI) - # hence using port 4790 - - name: Create VXLAN interface - become: true - command: ip link add {{ api_interface_name }} type vxlan id {{ api_interface_tunnel_vni }} local {{ tunnel_local_address }} dstport 4790 - - - name: Set VXLAN interface MTU - become: true - vars: - # Find the parent interface - parent_interface: >- - {{ ansible_interfaces | - map('extract', ansible_facts) | - selectattr('ipv4.address', 'defined') | - selectattr('ipv4.address', 'equalto', tunnel_local_address) | - first }} - # Allow 50 bytes overhead for VXLAN headers. - mtu: "{{ parent_interface.mtu | int - 50 }}" - command: ip link set {{ api_interface_name }} mtu {{ mtu }} - - # emulate BUM by multiplicating traffic to unicast targets - - name: Add fdb entries for BUM traffic - become: true - vars: - dest_ip: "{{ hostvars[item].tunnel_local_address }}" - command: bridge fdb append 00:00:00:00:00:00 dev {{ api_interface_name }} dst {{ dest_ip }} - with_inventory_hostnames: all - when: item != inventory_hostname - - - name: Add IPv4 address for VXLAN network - become: true - vars: - api_network_cidr: "{{ api_interface_address }}/{{ api_network_prefix_length }}" - # NOTE(yoctozepto): we have to compute and explicitly set the broadcast address, - # otherwise bifrost fails its pre-bootstrap sanity checks due to missing - # broadcast address as ansible picks up scope ('global') as the interface's - # broadcast address which fails checks logic - api_network_broadcast_address: "{{ api_network_cidr | ipaddr('broadcast') }}" - command: ip address add {{ api_network_cidr }} broadcast {{ api_network_broadcast_address }} dev {{ api_interface_name }} - when: address_family == 'ipv4' - - # NOTE(yoctozepto): IPv6 has no broadcast address, let's not create confusion by setting it - - name: Add IPv6 address for VXLAN network - become: true - command: ip address add {{ api_interface_address }}/{{ api_network_prefix_length }} dev {{ api_interface_name }} - when: address_family == 'ipv6' - - - name: Accept traffic on the VXLAN network (IN) - become: true - iptables: - state: present - action: insert - chain: INPUT - ip_version: "{{ address_family }}" - in_interface: "{{ api_interface_name }}" - jump: ACCEPT - - # NOTE(yoctozepto): the default policy is ACCEPT but it is nicer to get statistics - - name: Accept traffic on the VXLAN network (OUT) - become: true - iptables: - state: present - action: insert - chain: OUTPUT - ip_version: "{{ address_family }}" - out_interface: "{{ api_interface_name }}" - jump: ACCEPT - - - name: Bring VXLAN interface up - become: true - command: ip link set {{ api_interface_name }} up - - # NOTE(yoctozepto): IPv6 DAD may delay proper address assignment - # this task will wait until DAD is done and addresses are no longer tentative - # we assign addresses uniquely so DAD can only move it to preferred - # hence we only check whether it's no longer tentative - - name: Ensure IPv6 addresses on VXLAN are no longer tentative - become: true - command: ip -o address show tentative dev {{ api_interface_name }} - register: tentative_addresses - until: tentative_addresses.stdout == '' - retries: 30 - delay: 2 - when: - - address_family == 'ipv6' - - - name: Ping across VXLAN - vars: - ping_command: "{{ 'ping' if address_family == 'ipv4' else 'ping6' }}" - command: "{{ ping_command }} -c1 {{ hostvars[item].api_interface_address }}" - with_inventory_hostnames: all - # NOTE(yoctozepto): CentOS 7 image uses myhostname plugin for NSS # which creates issues with IPv6-only deployment by providing # an IPv4 address for the current hostname (affects rabbitmq)