diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml index 992eb4b2d9..b517f17c21 100644 --- a/ansible/group_vars/all.yml +++ b/ansible/group_vars/all.yml @@ -390,6 +390,7 @@ enable_nova_serialconsole_proxy: "no" enable_octavia: "no" enable_opendaylight: "no" enable_openvswitch: "{{ neutron_plugin_agent != 'linuxbridge' | bool }}" +enable_ovs_dpdk: "no" enable_osprofiler: "no" enable_panko: "no" enable_qdrouterd: "no" diff --git a/ansible/roles/ovs-dpdk/defaults/main.yml b/ansible/roles/ovs-dpdk/defaults/main.yml new file mode 100644 index 0000000000..4447522ef7 --- /dev/null +++ b/ansible/roles/ovs-dpdk/defaults/main.yml @@ -0,0 +1,86 @@ +--- +#################### +# Services +#################### + +ovsdpdk_services: + ovsdpdk-db: + container_name: "ovsdpdk_db" + image: "{{ ovsdpdk_db_image_full }}" + enabled: "{{ neutron_plugin_agent == 'openvswitch' }}" + host_in_groups: >- + {{ + inventory_hostname in groups['compute'] + or (enable_manila | bool and inventory_hostname in groups['manila-share']) + or inventory_hostname in groups['neutron-dhcp-agent'] + or inventory_hostname in groups['neutron-l3-agent'] + or inventory_hostname in groups['neutron-metadata-agent'] + or inventory_hostname in groups['neutron-vpnaas-agent'] + }} + volumes: + - "{{ node_config_directory }}/ovsdpdk-db/:{{ container_config_directory }}/:ro" + - "/etc/localtime:/etc/localtime:ro" + - "/run:/run:shared" + - "kolla_logs:/var/log/kolla/" + - "ovsdpdk_db:/var/lib/openvswitch/" + ovsdpdk-vswitchd: + container_name: "ovsdpdk_vswitchd" + image: "{{ ovsdpdk_vswitchd_image_full }}" + enabled: "{{ neutron_plugin_agent == 'openvswitch' }}" + privileged: True + host_in_groups: >- + {{ + inventory_hostname in groups['compute'] + or (enable_manila | bool and inventory_hostname in groups['manila-share']) + or inventory_hostname in groups['neutron-dhcp-agent'] + or inventory_hostname in groups['neutron-l3-agent'] + or inventory_hostname in groups['neutron-metadata-agent'] + or inventory_hostname in groups['neutron-vpnaas-agent'] + }} + volumes: + - "{{ node_config_directory }}/ovsdpdk-vswitchd/:{{ container_config_directory }}/:ro" + - "/etc/localtime:/etc/localtime:ro" + - "/lib/modules:/lib/modules:ro" + - "/run:/run:shared" + - "/dev:/dev:shared" + - "kolla_logs:/var/log/kolla/" + +#################### +# OVS +#################### +ovs_datapath: "netdev" +ovs_bridge_mappings: "{% for bridge in neutron_bridge_name.split(',') %}physnet{{ loop.index0 + 1 }}:{{ bridge }}{% if not loop.last %},{% endif %}{% endfor %}" +ovs_port_mappings: "{% for bridge in neutron_bridge_name.split(',') %} {{ neutron_external_interface.split(',')[loop.index0] }}:{{ bridge }}{% if not loop.last %},{% endif %}{% endfor %}" +dpdk_tunnel_interface : "{{neutron_external_interface}}" +dpdk_tunnel_interface_address: "{{ hostvars[inventory_hostname]['ansible_' + dpdk_tunnel_interface]['ipv4']['address'] }}" +tunnel_interface_network: "{{ hostvars[inventory_hostname]['ansible_' + dpdk_tunnel_interface]['ipv4']['network']}}/{{hostvars[inventory_hostname]['ansible_' + dpdk_tunnel_interface]['ipv4']['netmask']}}" +tunnel_interface_cidr: "{{dpdk_tunnel_interface_address}}/{{ tunnel_interface_network | ipaddr('prefix') }}" +ovs_cidr_mappings: "{% if neutron_bridge_name.split(',')|length != 1 %} {neutron_bridge_name.split(',')[0]}:{{ tunnel_interface_cidr }} {% else %} {{ neutron_bridge_name }}:{{ tunnel_interface_cidr }} {% endif %}" +ovs_mem_channels: 4 +ovs_socket_mem: 1024 +ovs_hugepage_mountpoint: /dev/hugepages + +# ovs <2.7 required dpdk phyical port names to be index +# in pci adress order as dpdkX where X is the index +# ovs>=2.7 allows arbitray names but the pci adress +# must be set in a new dpdkdev-opt field +# valid values are indexed or named. +ovs_physical_port_policy: named + +project_name: "ovs" +neutron_bridge_name: "br_dpdk" + + +#################### +# Docker +#################### +ovsdpdk_install_type: "{{ kolla_install_type }}" +ovsdpdk_tag: "{{ openstack_release }}" + +ovsdpdk_db_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ ovsdpdk_install_type }}-ovsdpdk-db" +ovsdpdk_db_tag: "{{ ovsdpdk_tag }}" +ovsdpdk_db_image_full: "{{ ovsdpdk_db_image }}:{{ ovsdpdk_db_tag }}" + +ovsdpdk_vswitchd_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ ovsdpdk_install_type }}-ovsdpdk-vswitchd" +ovsdpdk_vswitchd_tag: "{{ ovsdpdk_tag }}" +ovsdpdk_vswitchd_image_full: "{{ ovsdpdk_vswitchd_image }}:{{ ovsdpdk_vswitchd_tag }}" diff --git a/ansible/roles/ovs-dpdk/handlers/main.yml b/ansible/roles/ovs-dpdk/handlers/main.yml new file mode 100644 index 0000000000..4401940c3b --- /dev/null +++ b/ansible/roles/ovs-dpdk/handlers/main.yml @@ -0,0 +1,108 @@ +--- +- name: Restart ovsdpdk-db container + vars: + service_name: "ovsdpdk-db" + service: "{{ ovsdpdk_services[service_name] }}" + config_json: "{{ ovsdpdk_config_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}" + ovsdpdk_db_container: "{{ check_ovsdpdk_containers.results|selectattr('item.key', 'equalto', service_name)|first }}" + kolla_docker: + action: "recreate_or_restart_container" + common_options: "{{ docker_common_options }}" + name: "{{ service.container_name }}" + image: "{{ service.image }}" + volumes: "{{ service.volumes }}" + when: + - action != "config" + - service.enabled | bool + - service.host_in_groups | bool + - config_json | changed + or ovsdpdk_db_container | changed + notify: + - Waiting the ovs db service to be ready + - Ensuring ovsdpdk bridges are properly setup indexed + - Restart ovsdpdk-vswitchd container + - Ensuring ovsdpdk bridges are properly setup named + - wait for dpdk tunnel ip + - ovs-dpdk gather facts + +- name: Waiting the ovs db service to be ready + vars: + service_name: "ovsdpdk-db" + service: "{{ ovsdpdk_services[service_name] }}" + command: docker exec "{{ service.container_name }}" ovs-vsctl --no-wait show + register: check_result + until: check_result | success + changed_when: False + retries: 30 + delay: 2 + +- name: Ensuring ovsdpdk bridges are properly setup indexed + vars: + service_name: "ovsdpdk-db" + service: "{{ ovsdpdk_services[service_name] }}" + command: "docker exec -e \"CONFIG_FILE={{ container_config_directory }}/ovs-dpdkctl.conf\" {{ service.container_name }} {{ container_config_directory }}/ovs-dpdkctl.sh init" + register: status + changed_when: status.stdout.find('changed') != -1 + when: + - (inventory_hostname in groups['compute'] + or (enable_manila | bool and inventory_hostname in groups['manila-share']) + or inventory_hostname in groups['neutron-dhcp-agent'] + or inventory_hostname in groups['neutron-l3-agent'] + or inventory_hostname in groups['neutron-metadata-agent'] + or inventory_hostname in groups['neutron-vpnaas-agent']) + - ovs_physical_port_policy == 'indexed' + +- name: Restart ovsdpdk-vswitchd container + vars: + service_name: "ovsdpdk-vswitchd" + service: "{{ ovsdpdk_services[service_name] }}" + config_json: "{{ ovsdpdk_config_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}" + ovsdpdk_vswitchd_container: "{{ check_ovsdpdk_containers.results|selectattr('item.key', 'equalto', service_name)|first }}" + kolla_docker: + action: "recreate_or_restart_container" + common_options: "{{ docker_common_options }}" + name: "{{ service.container_name }}" + image: "{{ service.image }}" + volumes: "{{ service.volumes }}" + privileged: "{{ service.privileged | default(True) }}" + when: + - action != "config" + - service.enabled | bool + - service.host_in_groups | bool + - config_json | changed + or ovsdpdk_vswitchd_container | changed + + +- name: Ensuring ovsdpdk bridges are properly setup named + vars: + service_name: "ovsdpdk-db" + service: "{{ ovsdpdk_services[service_name] }}" + command: "docker exec -e \"CONFIG_FILE={{ container_config_directory }}/ovs-dpdkctl.conf\" {{ service.container_name }} {{ container_config_directory }}/ovs-dpdkctl.sh init" + register: status + changed_when: status.stdout.find('changed') != -1 + when: + - (inventory_hostname in groups['compute'] + or (enable_manila | bool and inventory_hostname in groups['manila-share']) + or inventory_hostname in groups['neutron-dhcp-agent'] + or inventory_hostname in groups['neutron-l3-agent'] + or inventory_hostname in groups['neutron-metadata-agent'] + or inventory_hostname in groups['neutron-vpnaas-agent']) + - ovs_physical_port_policy == 'named' + + +- name: wait for dpdk tunnel ip + wait_for: + host: "{{ dpdk_tunnel_interface_address }}" + timeout: 300 + sleep: 5 + +- name: ovs-dpdk gather facts + setup: + filter: ansible_* + when: + - (inventory_hostname in groups['compute'] + or (enable_manila | bool and inventory_hostname in groups['manila-share']) + or inventory_hostname in groups['neutron-dhcp-agent'] + or inventory_hostname in groups['neutron-l3-agent'] + or inventory_hostname in groups['neutron-metadata-agent'] + or inventory_hostname in groups['neutron-vpnaas-agent']) diff --git a/ansible/roles/ovs-dpdk/tasks/check.yml b/ansible/roles/ovs-dpdk/tasks/check.yml new file mode 100644 index 0000000000..ed97d539c0 --- /dev/null +++ b/ansible/roles/ovs-dpdk/tasks/check.yml @@ -0,0 +1 @@ +--- diff --git a/ansible/roles/ovs-dpdk/tasks/config.yml b/ansible/roles/ovs-dpdk/tasks/config.yml new file mode 100644 index 0000000000..bfb7b15bff --- /dev/null +++ b/ansible/roles/ovs-dpdk/tasks/config.yml @@ -0,0 +1,60 @@ +--- +- name: Ensuring config directories exist + file: + path: "{{ node_config_directory }}/{{ item.key }}" + state: "directory" + recurse: yes + when: + - item.value.enabled | bool + - item.value.host_in_groups | bool + with_dict: "{{ ovsdpdk_services }}" + +- name: Copying over config.json files for services + template: + src: "{{ item.key }}.json.j2" + dest: "{{ node_config_directory }}/{{ item.key }}/config.json" + register: ovsdpdk_config_jsons + when: + - item.value.enabled | bool + - item.value.host_in_groups | bool + with_dict: "{{ ovsdpdk_services }}" + notify: + - "Restart {{ item.key }} container" + +- name: Copying ovs-dpdkctl tool + copy: + src: ../tools/ovs-dpdkctl.sh + dest: "{{ node_config_directory }}/ovsdpdk-db/ovs-dpdkctl.sh" + mode: 0777 + +- name: Install ovs-dpdkctl service and config + become: True + command: "{{ node_config_directory }}/ovsdpdk-db/ovs-dpdkctl.sh install" + environment: + CONFIG_FILE: "{{ node_config_directory }}/ovsdpdk-db/ovs-dpdkctl.conf" + bridge_mappings: "{{ ovs_bridge_mappings }}" + port_mappings: "{{ ovs_port_mappings }}" + cidr_mappings: "{{ ovs_cidr_mappings }}" + ovs_mem_channels: "{{ ovs_mem_channels }}" + ovs_socket_mem: "{{ ovs_socket_mem }}" + hugepage_mountpoint: "{{ ovs_hugepage_mountpoint }}" + ovs_physical_port_policy: "{{ ovs_physical_port_policy }}" + + +# TODO check the environment change +- name: Check ovs containers + kolla_docker: + action: "compare_container" + common_options: "{{ docker_common_options }}" + name: "{{ item.value.container_name }}" + image: "{{ item.value.image }}" + privileged: "{{ item.value.privileged | default(False) }}" + volumes: "{{ item.value.volumes }}" + register: check_ovsdpdk_containers + when: + - action != "config" + - item.value.enabled | bool + - item.value.host_in_groups | bool + with_dict: "{{ ovsdpdk_services }}" + notify: + - "Restart {{ item.key }} container" diff --git a/ansible/roles/ovs-dpdk/tasks/deploy.yml b/ansible/roles/ovs-dpdk/tasks/deploy.yml new file mode 100644 index 0000000000..5aac9f5a7f --- /dev/null +++ b/ansible/roles/ovs-dpdk/tasks/deploy.yml @@ -0,0 +1,5 @@ +--- +- include: config.yml + +- name: Flush Handlers + meta: flush_handlers diff --git a/ansible/roles/ovs-dpdk/tasks/main.yml b/ansible/roles/ovs-dpdk/tasks/main.yml new file mode 100644 index 0000000000..b017e8b4ad --- /dev/null +++ b/ansible/roles/ovs-dpdk/tasks/main.yml @@ -0,0 +1,2 @@ +--- +- include: "{{ action }}.yml" diff --git a/ansible/roles/ovs-dpdk/tasks/prechecks.yml b/ansible/roles/ovs-dpdk/tasks/prechecks.yml new file mode 100644 index 0000000000..ed97d539c0 --- /dev/null +++ b/ansible/roles/ovs-dpdk/tasks/prechecks.yml @@ -0,0 +1 @@ +--- diff --git a/ansible/roles/ovs-dpdk/tasks/pull.yml b/ansible/roles/ovs-dpdk/tasks/pull.yml new file mode 100644 index 0000000000..9341fcc4b6 --- /dev/null +++ b/ansible/roles/ovs-dpdk/tasks/pull.yml @@ -0,0 +1,11 @@ +--- +- name: Pulling ovs-dpdk images + kolla_docker: + action: "pull_image" + common_options: "{{ docker_common_options }}" + image: "{{ item.value.image }}" + when: + - item.value.enabled | bool + - item.value.host_in_groups | bool + with_dict: "{{ ovsdpdk_services }}" + diff --git a/ansible/roles/ovs-dpdk/tasks/reconfigure.yml b/ansible/roles/ovs-dpdk/tasks/reconfigure.yml new file mode 100644 index 0000000000..e078ef1318 --- /dev/null +++ b/ansible/roles/ovs-dpdk/tasks/reconfigure.yml @@ -0,0 +1,2 @@ +--- +- include: deploy.yml diff --git a/ansible/roles/ovs-dpdk/tasks/upgrade.yml b/ansible/roles/ovs-dpdk/tasks/upgrade.yml new file mode 100644 index 0000000000..e078ef1318 --- /dev/null +++ b/ansible/roles/ovs-dpdk/tasks/upgrade.yml @@ -0,0 +1,2 @@ +--- +- include: deploy.yml diff --git a/ansible/roles/ovs-dpdk/templates/ovsdpdk-db.json.j2 b/ansible/roles/ovs-dpdk/templates/ovsdpdk-db.json.j2 new file mode 100644 index 0000000000..fdeea72ecb --- /dev/null +++ b/ansible/roles/ovs-dpdk/templates/ovsdpdk-db.json.j2 @@ -0,0 +1,4 @@ +{ + "command": "start-ovsdb-server {{ api_interface_address }}", + "config_files": [] +} diff --git a/ansible/roles/ovs-dpdk/templates/ovsdpdk-vswitchd.json.j2 b/ansible/roles/ovs-dpdk/templates/ovsdpdk-vswitchd.json.j2 new file mode 100644 index 0000000000..97308d886b --- /dev/null +++ b/ansible/roles/ovs-dpdk/templates/ovsdpdk-vswitchd.json.j2 @@ -0,0 +1,4 @@ +{ + "command": "/usr/sbin/ovs-vswitchd unix:/run/openvswitch/db.sock -vconsole:emer -vsyslog:err -vfile:info --mlockall --log-file=/var/log/kolla/openvswitch/ovs-vswitchd.log", + "config_files": [] +} diff --git a/ansible/site.yml b/ansible/site.yml index 50cbce1fa0..05435f5926 100644 --- a/ansible/site.yml +++ b/ansible/site.yml @@ -322,7 +322,15 @@ roles: - { role: openvswitch, tags: openvswitch, - when: enable_openvswitch | bool } + when: "(enable_openvswitch | bool) and not (enable_ovs_dpdk | bool)"} + +- name: Apply role ovs-dpdk + hosts: + - openvswitch + roles: + - { role: ovs-dpdk, + tags: ovs-dpdk, + when: "(enable_openvswitch | bool) and (enable_ovs_dpdk | bool)"} - name: Apply nova-hyperv role gather_facts: false diff --git a/etc/kolla/globals.yml b/etc/kolla/globals.yml index f0a4b65963..279a6a1a97 100644 --- a/etc/kolla/globals.yml +++ b/etc/kolla/globals.yml @@ -192,6 +192,7 @@ kolla_internal_vip_address: "10.10.10.254" #enable_octavia: "no" #enable_opendaylight: "no" #enable_openvswitch: "{{ neutron_plugin_agent != 'linuxbridge' }}" +#enable_ovs_dpdk: "no" #enable_osprofiler: "no" #enable_panko: "no" #enable_qdrouterd: "no" diff --git a/releasenotes/notes/ovs-dpdk-a48404777d3836a3.yaml b/releasenotes/notes/ovs-dpdk-a48404777d3836a3.yaml new file mode 100644 index 0000000000..cb7a031423 --- /dev/null +++ b/releasenotes/notes/ovs-dpdk-a48404777d3836a3.yaml @@ -0,0 +1,43 @@ +--- +prelude: > + The Data Plane Development Kit (DPDK) is a collection of userspace + libraries and tools that facilitate the development of high-performance + userspace networking applications. During the Pike cycle, kolla has been + extended to support deploying OpenvSwitch with the dpdk netdev dataplane. +features: + - | + A new ovs-dpdk role has been introduced to support deploy OpenvSwitch with + the userspace DPDK dataplane. This playbook has been tested on Centos and + Ubuntu hosts and introduces a new enable_ovs_dpdk variable to control + deployment of OVS with DPDK instead of kernel ovs. + - | + The ovs-dpdkctl.sh tool has been enhanced to enable automatic loading of + DPDK compatible kernel modules. + - | + The ovs-dpdkctl.sh tool has been extended to configure per host + tunnel ip address to enable vxlan and other tunnel networking. +issues: + - | + The ovs-dpdk role introduced this release has been tested on centos 7 and + ubuntu 16.04 hosts. While both host operating systems can be used to deploy + ovs-dpdk containers a conflict exists between the cgroups created by docker + and systemd on centos. These conflicts prevent restarting the ovs-dpdk + containers once deployed. This issue is not present on Ubuntu and is fixed + in newer versions of systemd. Deploying the ovs-dpdk role on centos is not + advised in production on CentOS hosts unless you have validated that this + issue has been addressed. +upgrade: + - | + When upgrading ovs-dpdk it should be noted that this will alway invovle a + dataplane outage. Unlike kernel OVS the dataplane for ovs-dpdk executes in + the ovs-vswitchd process. As such it is recommended to always evacuate + all vm workloads from a node running ovs-dpdk prior to upgrading. +other: + - | + Reconfiguring form kernel based OVS to OVS with DPDK has not been tested + in this release. Changing ovs datapaths on a deployed node requires + neutron config changes and libvirt xml changes for all running instances + including a hard reboot of the vm. VMs deployed to an node running ovs-dpdk + require hugepage backed memory to function correctly. As such it is not + advised to convert an existing compute node to use ovs-dpdk + if it currently has VMs deployed. diff --git a/tools/cleanup-host b/tools/cleanup-host index 0cf4a69b10..b0917e1e30 100755 --- a/tools/cleanup-host +++ b/tools/cleanup-host @@ -56,8 +56,14 @@ if [[ "$glance_file_datadir_volume" != "glance" && -d "$glance_file_datadir_volu rm -rfv $glance_file_datadir_volume fi -echo "Getting folders name..." FOLDER_PATH="/etc/kolla/" + +if [[ -e "$FOLDER_PATH/ovsdpdk-db/ovs-dpdkctl.sh" ]]; then + echo uninstalling ovs-dpdk service files and tools. + sudo CONFIG_FILE="$FOLDER_PATH/ovsdpdk-db/ovs-dpdkctl.conf" $FOLDER_PATH/ovsdpdk-db/ovs-dpdkctl.sh uninstall +fi + +echo "Getting folders name..." for dir in $FOLDER_PATH*; do if [ "$dir" == "$FOLDER_PATH""passwords.yml" ] || \ [ "$dir" == "$FOLDER_PATH""globals.yml" ] || \ diff --git a/tools/kolla-ansible b/tools/kolla-ansible index 53174cae46..63692267a5 100755 --- a/tools/kolla-ansible +++ b/tools/kolla-ansible @@ -49,7 +49,7 @@ Commands: prechecks Do pre-deployment checks for hosts check Do post-deployment smoke tests mariadb_recovery Recover a completely stopped mariadb cluster - bootstrap-servers bootstrap servers with kolla deploy dependencies + bootstrap-servers Bootstrap servers with kolla deploy dependencies destroy Destroy Kolla containers, volumes and host configuration --include-images to also destroy Kolla images --include-dev to also destroy dev mode repos diff --git a/tools/ovs-dpdkctl.sh b/tools/ovs-dpdkctl.sh index 2b89ad4322..fc5cf2f781 100755 --- a/tools/ovs-dpdkctl.sh +++ b/tools/ovs-dpdkctl.sh @@ -8,6 +8,7 @@ fi FULL_PATH=$(realpath "${BASH_SOURCE[0]}") CONFIG_FILE=${CONFIG_FILE:-"/etc/default/ovs-dpdk.conf"} SERVICE_FILE="/etc/systemd/system/ovs-dpdkctl.service" +BRIDGE_SERVICE_FILE="/etc/systemd/system/ovs-dpdk-bridge.service" function get_value { crudini --get $CONFIG_FILE $@ @@ -37,8 +38,12 @@ function del_config { rm -f $CONFIG_FILE } +function is_redhat_family { + [[ -e /etc/redhat-release ]] ; echo $? +} + function generate_pciwhitelist { - local _Whitelist='' + _Whitelist='' for nic in $(list_dpdk_nics); do address="$(get_value $nic address)" if [ "$_Whitelist" == '' ]; then @@ -79,6 +84,7 @@ function gen_config { set_value ovs ovs_socket_mem ${ovs_socket_mem:-"512"} set_value ovs dpdk_interface_driver ${dpdk_interface_driver:-"uio_pci_generic"} set_value ovs hugepage_mountpoint ${hugepage_mountpoint:-"/dev/hugepages"} + set_value ovs physical_port_policy ${ovs_physical_port_policy:-"named"} ls -al /sys/class/net/* | awk '$0 ~ /pci/ {n=split($NF,a,"/"); print "\n[" a[n] "]\naddress = " a[n-2] "\ndriver ="}' >> $CONFIG_FILE @@ -93,8 +99,8 @@ function gen_config { } function bind_nic { - echo $1 > /sys/bus/pci/drivers/$2/bind echo $2 > /sys/bus/pci/devices/$1/driver_override + echo $1 > /sys/bus/pci/drivers/$2/bind } function unbind_nic { @@ -137,7 +143,6 @@ function unbind_nics { done } - function get_address_by_name { ls -al /sys/class/net/$1 | awk '$0 ~ /pci/ {n=split($NF,a,"/"); print a[n-2] }' } @@ -147,7 +152,7 @@ function get_driver_by_address { } function get_port_bridge { - for pair in $(get_value ovs port port_mappings); do + for pair in $(get_value ovs port_mappings); do nic=`echo $pair | cut -f 1 -d ":"` if [[ "$nic" == "$1" ]]; then bridge=`echo $pair | cut -f 2 -d ":"` @@ -178,11 +183,11 @@ function init_ovs_bridges { } function init_ovs_interfaces { - local pci_port_pairs =='' + pci_port_pairs='' for nic in $(list_dpdk_nics); do address="$(get_value $nic address)" if [ "$pci_port_pairs" == '' ]; then - pci_port_pairs ="$address,$nic" + pci_port_pairs="$address,$nic" else pci_port_pairs="$pci_port_pairs $address,$nic" fi @@ -194,10 +199,13 @@ function init_ovs_interfaces { nic="$(echo $pair | cut -f 2 -d ",")" bridge="$(get_port_bridge $nic)" # ovs 2.6 and older requires dpdkX names, ovs 2.7+ requires dpdk-devargs instead. - ovs-vsctl --no-wait --may-exist add-port $bridge "dpdk${dpdk_port_number}" \ - -- set Interface "dpdk${dpdk_port_number}" type=dpdk || \ - ovs-vsctl --no-wait --may-exist add-port $bridge $nic \ - -- set Interface $nic type=dpdk options:dpdk-devargs=$addr + if [ "$(get_value ovs physical_port_policy)" == "indexed" ]; then + ovs-vsctl --no-wait --may-exist add-port $bridge "dpdk${dpdk_port_number}" \ + -- set Interface "dpdk${dpdk_port_number}" type=dpdk + else + ovs-vsctl --may-exist add-port $bridge $nic \ + -- set Interface $nic type=dpdk options:dpdk-devargs=$addr + fi dpdk_port_number=$((dpdk_port_number+1)) done @@ -210,6 +218,57 @@ function init { init_ovs_interfaces } +function install_network_manager_conf { + pair=$(get_value ovs cidr_mappings) + bridge=`echo $pair | cut -f 1 -d ":"` + cidr=`echo $pair | cut -f 2 -d ":"` + ip=`echo $cidr | cut -f 1 -d "/"` + prefix=`echo $cidr | cut -f 2 -d "/"` + mask="" + full_octets=$(expr $prefix / 8) + partial_octet=$(expr $prefix % 8) + + for octet in 0 1 2 3 ; do + if [[ "$octet" < "$full_octets" ]]; then + mask+=255 + elif [[ "$octet" == "$full_octets" ]]; then + mask+=$((256 - 2**(8-$partial_octet))) + else + mask+=0 + fi + [[ "$octet" < 3 ]] && mask+=. + done + if [[ is_redhat_family == 0 ]]; then + cat << EOF | tee "/etc/sysconfig/network-scripts/ifcfg-$bridge" +DEVICE=$bridge +BOOTPROTO=static +IPADDR=$ip +NETMASK=$mask +HOTPLUG=yes +ONBOOT=yes +EOF +install_redhat_bridge_service $bridge + else + cat << EOF | tee "/etc/network/interfaces.d/$bridge.cfg" + auto $bridge + iface $bridge inet static + address $ip + netmask $mask +EOF + + fi +} + +function uninstall_network_manager_conf { + pair=$(get_value ovs cidr_mappings) + bridge=`echo $pair | cut -f 1 -d ":"` + if [[ is_redhat_family == 0 ]]; then + rm -f /etc/sysconfig/network-scripts/ifcfg-$bridge + else + rm -f /etc/network/interfaces.d/$bridge.cfg + fi +} + function install_service { cat << EOF | tee "$SERVICE_FILE" @@ -235,13 +294,63 @@ EOF systemctl enable ovs-dpdkctl } +function install_redhat_bridge_service { + cat << EOF | tee "$BRIDGE_SERVICE_FILE" +[Unit] +Description=configuration service for ovs-dpdk bridge. +After=docker.service +After=network.target +After=ovs-dpdkctl.service + +[Service] +Type=simple +RemainAfterExit=yes +ExecStartPre=/bin/bash -c "[[ -e /sys/class/net/$1 ]]" +ExecStart=/usr/sbin/ifup $1 +ExecStop=/usr/sbin/ifdown $1 +Restart=on-failure +RestartSec=5 + +[Install] +WantedBy=multi-user.target + +EOF + systemctl daemon-reload + systemctl enable ovs-dpdk-bridge +} + function uninstall_service { systemctl disable ovs-dpdkctl rm -f "$SERVICE_FILE" + if [ -e "$BRIDGE_SERVICE_FILE" ]; then + systemctl disable ovs-dpdk-bridge + rm -f "$BRIDGE_SERVICE_FILE" + fi systemctl daemon-reload } +function configure_kernel_modules { + driver="$(get_value ovs dpdk_interface_driver)" + lsmod | grep -ws $driver > /dev/null || modprobe $driver + if [[ is_redhat_family == 0 ]]; then + [[ ! -e /etc/modules-load.d/${driver}.conf ]] && echo $driver | tee /etc/modules-load.d/${driver}.conf + else + grep -ws $driver /etc/modules > /dev/null || echo $driver | tee -a /etc/modules + fi +} + +function unconfigure_kernel_modules { + driver="$(get_value ovs dpdk_interface_driver)" + lsmod | grep -ws $driver > /dev/null && rmmod $driver + if [[ is_redhat_family == 0 ]] ; then + [[ -e /etc/modules-load.d/${driver}.conf ]] && rm -f /etc/modules-load.d/${driver}.conf + else + grep -ws $driver /etc/modules > /dev/null && sed -e "s/$driver//" -i /etc/modules + fi +} + function install { + configure_kernel_modules if [ ! -e "$SERVICE_FILE" ]; then install_service fi @@ -253,6 +362,10 @@ function install { gen_config fi systemctl start ovs-dpdkctl + install_network_manager_conf + if [[ is_redhat_family == 0 ]]; then + systemctl start ovs-dpdk-bridge + fi } function uninstall { @@ -260,14 +373,16 @@ function uninstall { if [ -e "$SERVICE_FILE" ]; then uninstall_service fi + uninstall_network_manager_conf + unconfigure_kernel_modules if [ -e /bin/ovs-dpdkctl ]; then rm -f /bin/ovs-dpdkctl fi if [ -e "$CONFIG_FILE" ]; then rm -f "$CONFIG_FILE" fi -} +} function useage { cat << "EOF"