From 7454e6040c45b58bca46bcce0c1634df6cefd681 Mon Sep 17 00:00:00 2001 From: Bjoern Teipel Date: Tue, 3 Jan 2017 18:03:26 -0600 Subject: [PATCH] Implementation Neutron SR-IOV This change implements SR-IOV support inside Neutron via the new ``ml2.sriov`` plugin. The current implementation automatically configures the ML2 mechanism drivers via the enabled neutron_plugins and configured mechanisms attribute inside the dictionary. The implementation is based off the neutron-sriov-nic-agent and configures physical device mappings over a new attribute ``sriov_host_interface`` as part of the provider_networks configuration. Additionally the FDB agent inside the ``linuxbridge_agent.ini`` is enabled to support the linuxbridge and sriov-nic-agent on the same interface. Closes-Bug: #1653283 Co-Authored-By: James Denton Change-Id: I7b580db0496ff009a2f64c71447b2977f22d6bf6 Depends-On: Ia62725e2369f75000157e0ab2c3f858e61fef10d Implements: sriov support --- defaults/main.yml | 39 ++++- doc/source/configure-network-services.rst | 133 ++++++++++++++++++ .../notes/neutron-sriov-50c0099554574d01.yaml | 7 + tasks/neutron_init.yml | 13 ++ tasks/neutron_post_install.yml | 13 +- .../plugins/ml2/linuxbridge_agent.ini.j2 | 8 ++ templates/plugins/ml2/ml2_conf.ini.j2 | 2 +- templates/plugins/ml2/sriov_nic_agent.ini.j2 | 9 ++ 8 files changed, 218 insertions(+), 6 deletions(-) create mode 100644 releasenotes/notes/neutron-sriov-50c0099554574d01.yaml create mode 100644 templates/plugins/ml2/sriov_nic_agent.ini.j2 diff --git a/defaults/main.yml b/defaults/main.yml index 085477b2..3d58aa8d 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -131,7 +131,20 @@ neutron_plugin_base: neutron_plugin_loaded_base: "{% for plugin in neutron_plugin_base %}{{ plugin }}{% if not loop.last %},{% endif %}{% endfor %}" # Neutron Plugins -neutron_plugin_type: ml2.lxb +# The neutron core plugin (ML2) is defined with neutron_plugin_type, +# you can not load multiple ML2 plugins as core. +# Additional ML2 plugins can be loaded with neutron_plugin_types (as list) +neutron_plugin_type: 'ml2.lxb' +neutron_plugin_types: [] +neutron_ml2_mechanism_drivers: >- + {%- set _var = [] -%} + {%- for plugin in [neutron_plugin_type]|union(neutron_plugin_types) -%} + {%- if _var.append(neutron_plugins[plugin].mechanisms) -%}{%- endif -%} + {%- endfor -%} + {%- if neutron_l2_population | bool -%} + {%- if _var.append('l2population') -%}{%- endif -%} + {%- endif -%} + {{ _var | join(',') }} neutron_plugins: ml2.lxb: @@ -139,7 +152,7 @@ neutron_plugins: driver_interface: linuxbridge drivers_type: "{{ neutron_ml2_drivers_type }}" l2_population: "{{ neutron_l2_population }}" - mechanisms: "linuxbridge{% if neutron_l2_population | bool %},l2population{% endif %}" + mechanisms: "linuxbridge" l3_agent_mode: "legacy" plugin_conf_ini_overrides: "{{ neutron_ml2_conf_ini_overrides }}" plugin_core: ml2 @@ -149,7 +162,7 @@ neutron_plugins: driver_interface: openvswitch drivers_type: "{{ neutron_ml2_drivers_type }}" l2_population: "{{ neutron_l2_population }}" - mechanisms: "openvswitch{% if neutron_l2_population | bool %},l2population{% endif %}" + mechanisms: "openvswitch" l3_agent_mode: "legacy" plugin_conf_ini_overrides: "{{ neutron_ml2_conf_ini_overrides }}" plugin_core: ml2 @@ -159,7 +172,7 @@ neutron_plugins: driver_interface: openvswitch drivers_type: "{{ neutron_ml2_drivers_type }}" l2_population: "{{ neutron_l2_population }}" - mechanisms: "openvswitch{% if neutron_l2_population | bool %},l2population{% endif %}" + mechanisms: "openvswitch" l3_agent_mode: "{% if 'nova_compute' in group_names %}dvr{% else %}dvr_snat{% endif %}" router_distributed: True plugin_conf_ini_overrides: "{{ neutron_ml2_conf_ini_overrides }}" @@ -179,6 +192,11 @@ neutron_plugins: plugin_core: neutron.plugins.nuage.plugin.NuagePlugin plugin_ini: plugins/nuage/nuage.ini plugin_conf_ini_overrides: "{{ neutron_nuage_conf_ini_overrides }}" + ml2.sriov: + driver_types: "{{ neutron_ml2_drivers_type }}" + mechanisms: "sriovnicswitch" + plugin_ini: plugins/ml2/sriov_nic_agent.ini + plugin_conf_ini_overrides: "{{ neutron_sriov_nic_agent_ini_overrides }}" neutron_services: neutron-dhcp-agent: @@ -298,6 +316,15 @@ neutron_services: config_options: "--config-file {{ neutron_conf_dir }}/neutron.conf" config_overrides: "{{ neutron_calico_dhcp_agent_ini_overrides }}" config_type: "ini" + neutron-sriov-nic-agent: + group: neutron_sriov_nic_agent + service_name: neutron-sriov-nic-agent + service_en: "{{ 'ml2.sriov' in neutron_plugin_types }}" + service_conf_path: "{{ neutron_conf_dir }}" + service_conf: plugins/ml2/sriov_nic_agent.ini + config_options: "--config-file {{ neutron_conf_dir }}/neutron.conf --config-file {{ neutron_conf_dir }}/plugins/ml2/ml2_conf.ini --config-file {{ neutron_conf_dir }}/plugins/ml2/sriov_nic_agent.ini --log-file=/var/log/neutron/neutron-sriov-nic-agent.log" + config_overrides: "{{ neutron_sriov_nic_agent_ini_overrides }}" + config_type: "ini" ## Neutron DHCP Agent neutron_dhcp: "{% if neutron_plugin_type.split('.')[0] == 'ml2' %}True{% else %}False{% endif %}" @@ -456,9 +483,12 @@ neutron_vxlan_group: "239.1.1.1" # network_types: "vxlan,flat,vlan" # network_vlan_ranges: "vlan:1:1,vlan:1024:1025" # network_vxlan_ranges: "1:1000" +# network_sriov_mappings: "vlan:p4p1" neutron_vxlan_enabled: true +neutron_sriov_excluded_devices: "" + neutron_dhcp_domain: openstacklocal # Comma-separated list of DNS servers which will be used by dnsmasq as forwarders. neutron_dnsmasq_dns_servers: "" @@ -570,3 +600,4 @@ neutron_openvswitch_agent_ini_overrides: {} neutron_bgp_dragent_ini_overrides: {} neutron_calico_dhcp_agent_ini_overrides: {} neutron_calico_felix_ini_overrides: {} +neutron_sriov_nic_agent_ini_overrides: {} diff --git a/doc/source/configure-network-services.rst b/doc/source/configure-network-services.rst index 68b66a1c..f9783ffc 100644 --- a/doc/source/configure-network-services.rst +++ b/doc/source/configure-network-services.rst @@ -18,6 +18,11 @@ BGP Dynamic Routing service Provides a means for advertising self-service (private) network prefixes to physical network devices that support BGP. +SR-IOV Support + Provides the ability to provision virtual or physical functions to guest + instances using SR-IOV and PCI passthrough. (Requires compatible NICs) + + Firewall service (optional) ~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -225,3 +230,131 @@ Routing plugin. # cd /opt/openstack-ansible/playbooks # openstack-ansible os-neutron-install.yml + + +SR-IOV Support (optional) +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The following procedure describes how to modify the OpenStack-Ansible +configuration to enable Neutron SR-IOV support. + +.. _SR-IOV-Passthrough-For-Networking: https://wiki.openstack.org/wiki/SR-IOV-Passthrough-For-Networking + + +#. Define SR-IOV capable physical host interface for a provider network + + As part of every Openstack-Ansible installation, all provider networks + known to Neutron need to be configured inside the + ``/etc/openstack_deploy/openstack_user_config.yml`` file. + For each supported network type (e.g. vlan), the attribute + ``sriov_host_interfaces`` can be defined to map ML2 network names + (``net_name`` attribute) to one or many physical interfaces. + Additionally, the network will need to be assigned to the + ``neutron_sriov_nic_agent`` container group. + +Example configuration: + + .. code-block:: yaml + + provider_networks + - network: + container_bridge: "br-vlan" + container_type: "veth" + container_interface: "eth11" + type: "vlan" + range: "1000:2000" + net_name: "physnet1" + sriov_host_interfaces: "p1p1,p4p1" + group_binds: + - neutron_linuxbridge_agent + - neutron_sriov_nic_agent + +#. Configure Nova + + With SR-IOV, Nova uses PCI passthrough to allocate VFs and PFs to guest + instances. Virtual Functions (VFs) represent a slice of a physical NIC, + and are passed as virtual NICs to guest instances. Physical Functions + (PFs), on the other hand, represent an entire physical interface and are + passed through to a single guest. + + To use PCI passthrough in Nova, the ``PciPassthroughFilter`` filter + needs to be added to the `conf override`_ + ``nova_scheduler_default_filters``. + Finally, PCI devices available for passthrough need to be allow via + the `conf override`_ + ``nova_pci_passthrough_whitelist``. + + Possible options which can be configured: + + .. code-block:: yaml + + # Single device configuration + nova_pci_passthrough_whitelist: '{ "physical_network":"physnet1", "devname":"p1p1" }' + + # Multi device configuration + nova_pci_passthrough_whitelist: '[{"physical_network":"physnet1", "devname":"p1p1"}, {"physical_network":"physnet1", "devname":"p4p1"}]' + + # Whitelisting by PCI Device Location + # The example pattern for the bus location '0000:04:*.*' is very wide. Make sure that + # no other, unintended devices, are whitelisted (see lspci -nn) + nova_pci_passthrough_whitelist: '{"address":"0000:04:*.*", "physical_network":"physnet1"}' + + # Whitelisting by PCI Device Vendor + # The example pattern limits matches to PCI cards with vendor id 8086 (Intel) and + # product id 10ed (82599 Virtual Function) + nova_pci_passthrough_whitelist: '{"vendor_id":"8086", "product_id":"10ed", "physical_network":"physnet1"}' + + # Additionally, devices can be matched by their type, VF or PF, using the dev_type parameter + # and type-VF or type-PF options + nova_pci_passthrough_whitelist: '{"vendor_id":"8086", "product_id":"10ed", "dev_type":"type-VF", physical_network":"physnet1"}' + + It is recommended to use whitelisting by either the Linux device name + (devname attribute) or by the PCI vendor and product id combination + (``vendor_id`` and ``product_id`` attributes) + +#. Enable the SR-IOV ML2 plugin + + The `conf override`_ ``neutron_plugin_type`` variable defines the core + ML2 plugin, and only one plugin can be defined at any given time. + The `conf override`_ ``neutron_plugin_types`` variable can contain a list + of additional ML2 plugins to load. Make sure that only compatible + ML2 plugins are loaded at all times. + The SR-IOV ML2 plugin is known to work with the linuxbridge (``ml2.lxb``) + and openvswitch (``ml2.ovs``) ML2 plugins. + ``ml2.lxb`` is the standard activated core ML2 plugin. + + .. code-block:: yaml + + neutron_plugin_types: + - ml2.sriov + + +#. Execute the Neutron install playbook in order to update the configuration: + + .. code-block:: shell-session + + # cd /opt/openstack-ansible/playbooks + # openstack-ansible os-neutron-install.yml + # openstack-ansible os-nova-install.yml + + +#. Check Neutron SR-IOV agent state + + After the playbooks have finished configuring Neutron and Nova, the new + Neutron Agent state can be verified with: + + .. code-block:: shell-session + + # neutron agent-list --agent_type 'NIC Switch agent' + +--------------------------------------+------------------+-----------+-------+----------------+-------------------------+ + | id | agent_type | host | alive | admin_state_up | binary | + +--------------------------------------+------------------+-----------+-------+----------------+-------------------------+ + | 3012ff0e-de35-447b-aff6-fdb55b04c518 | NIC Switch agent | compute01 | :-) | True | neutron-sriov-nic-agent | + | bb0c0385-394d-4e72-8bfe-26fd020df639 | NIC Switch agent | compute02 | :-) | True | neutron-sriov-nic-agent | + +--------------------------------------+------------------+-----------+-------+----------------+-------------------------+ + + +Deployers can make changes to the SR-IOV nic agent default configuration +options via the ``neutron_sriov_nic_agent_ini_overrides`` dict. +Review the documentation on the `conf override`_ mechanism for more details. + diff --git a/releasenotes/notes/neutron-sriov-50c0099554574d01.yaml b/releasenotes/notes/neutron-sriov-50c0099554574d01.yaml new file mode 100644 index 00000000..d5945e90 --- /dev/null +++ b/releasenotes/notes/neutron-sriov-50c0099554574d01.yaml @@ -0,0 +1,7 @@ +--- +features: + - Neutron SR-IOV can now be optionally deployed and configured. + For details about the what the service is and what it provides, see the + `SR-IOV Installation Guide + `_ + for more information. diff --git a/tasks/neutron_init.yml b/tasks/neutron_init.yml index cdc07479..57ee48c0 100644 --- a/tasks/neutron_init.yml +++ b/tasks/neutron_init.yml @@ -132,3 +132,16 @@ when: - neutron_services['neutron-vpnaas-agent']['group'] in group_names - neutron_services['neutron-vpnaas-agent'].service_en | bool + +- include: neutron_init_common.yml + vars: + program_name: "{{ neutron_services['neutron-sriov-nic-agent'].service_name }}" + program_config_options: "{{ neutron_services['neutron-sriov-nic-agent'].config_options }}" + service_name: "{{ neutron_service_name }}" + system_user: "{{ neutron_system_user_name }}" + system_group: "{{ neutron_system_group_name }}" + service_home: "{{ neutron_system_home_folder }}" + when: + - neutron_services['neutron-sriov-nic-agent']['group'] in group_names + - neutron_services['neutron-sriov-nic-agent'].service_en | bool + diff --git a/tasks/neutron_post_install.yml b/tasks/neutron_post_install.yml index 881974c0..0702d3c3 100644 --- a/tasks/neutron_post_install.yml +++ b/tasks/neutron_post_install.yml @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -- name: Copy neutron config +- name: Copy common neutron config config_template: src: "{{ item.src }}" dest: "{{ item.dest }}" @@ -48,6 +48,17 @@ notify: - Restart neutron services +- name: Copy neutron ml2 plugin config + config_template: + src: "{{ neutron_plugins[item].plugin_ini }}.j2" + dest: "{{ neutron_conf_dir }}/{{ neutron_plugins[item].plugin_ini }}" + owner: "{{ neutron_system_user_name }}" + group: "{{ neutron_system_group_name }}" + mode: "0644" + config_overrides: "{{ neutron_plugins[item].plugin_conf_ini_overrides }}" + config_type: "ini" + with_items: "{{ neutron_plugin_types }}" + - name: Generate neutron dnsmasq Config config_template: src: "dnsmasq-neutron.conf.j2" diff --git a/templates/plugins/ml2/linuxbridge_agent.ini.j2 b/templates/plugins/ml2/linuxbridge_agent.ini.j2 index df0f6eed..ddbd0225 100644 --- a/templates/plugins/ml2/linuxbridge_agent.ini.j2 +++ b/templates/plugins/ml2/linuxbridge_agent.ini.j2 @@ -25,6 +25,14 @@ enable_vxlan = False # Agent [agent] +{% if 'ml2.sriov' in neutron_plugin_types and 'nova_compute' in group_names %} +extensions = fdb + +# If an interface is shared between sriov ports and normal ports, it must be defined here +# to allow traffic between the two ports on the same host +[FDB] +shared_physical_device_mappings = {{ neutron_provider_networks.network_sriov_mappings }} +{% endif %} # Security groups [securitygroup] diff --git a/templates/plugins/ml2/ml2_conf.ini.j2 b/templates/plugins/ml2/ml2_conf.ini.j2 index 87a084be..46324da4 100644 --- a/templates/plugins/ml2/ml2_conf.ini.j2 +++ b/templates/plugins/ml2/ml2_conf.ini.j2 @@ -4,7 +4,7 @@ [ml2] type_drivers = {{ neutron_plugins[neutron_plugin_type].drivers_type }} tenant_network_types = {{ neutron_provider_networks.network_types if neutron_provider_networks.network_types != '' else 'local' }} -mechanism_drivers = {{ neutron_plugins[neutron_plugin_type].mechanisms }} +mechanism_drivers = {{ neutron_ml2_mechanism_drivers }} extension_drivers = port_security{% if 'qos' in neutron_plugin_base %},qos{% endif %} {% if neutron_provider_networks.network_flat_networks is defined %} diff --git a/templates/plugins/ml2/sriov_nic_agent.ini.j2 b/templates/plugins/ml2/sriov_nic_agent.ini.j2 new file mode 100644 index 00000000..285eb2a1 --- /dev/null +++ b/templates/plugins/ml2/sriov_nic_agent.ini.j2 @@ -0,0 +1,9 @@ +# {{ ansible_managed }} + +[securitygroup] +firewall_driver = neutron.agent.firewall.NoopFirewallDriver + +[sriov_nic] +physical_device_mappings = "{{ neutron_provider_networks.network_sriov_mappings }}" +exclude_devices = "{{ neutron_sriov_excluded_devices }}" +