From 61c19f3e9c9314811ea9695690391ef2a018ce6e Mon Sep 17 00:00:00 2001 From: Shih-Hao Li Date: Thu, 25 Feb 2016 13:10:15 -0800 Subject: [PATCH] Add internal metadata network on demand Previously, an internal metadata network is created for a router whenever a subnet is attached to this router. The purpose of this internal network is to help processing metadata requests from instances on DHCP-disabled networks. This commit adds a config option to create internal metadata networks only when a DHCP-disabled subnet is attached to a router. This will help saving system resources because each metadata network consumes one DHCP name space. Change-Id: Ia56050b3f431dbd65bb39da29ba6dbf8e62e36ea --- etc/nsx.ini | 12 +++ vmware_nsx/common/config.py | 14 +++ vmware_nsx/dhcp_meta/rpc.py | 35 +++++-- vmware_nsx/plugins/nsx_mh/plugin.py | 3 +- vmware_nsx/plugins/nsx_v3/plugin.py | 20 +++- .../tests/unit/extensions/test_metadata.py | 96 +++++++++++++++---- vmware_nsx/tests/unit/nsx_mh/test_plugin.py | 6 ++ 7 files changed, 160 insertions(+), 26 deletions(-) diff --git a/etc/nsx.ini b/etc/nsx.ini index 49c50f3584..70c31e0c07 100644 --- a/etc/nsx.ini +++ b/etc/nsx.ini @@ -391,3 +391,15 @@ # The default is 8 nested groups, which allows a maximum of 4k security-groups, # to allow creation of more security-groups, modify this figure. # number_of_nested_groups = + +# Acceptable values for 'metadata_mode' are: +# - 'access_network': this enables a dedicated connection to the metadata +# proxy for metadata server access via Neutron router. +# - 'dhcp_host_route': this enables host route injection via the dhcp agent. +# This option is only useful if running on a host that does not support +# namespaces otherwise access_network should be used. +# metadata_mode = access_network + +# If True, an internal metadata network will be created for a router only when +# the router is attached to a DHCP-disabled subnet. +# metadata_on_demand = True diff --git a/vmware_nsx/common/config.py b/vmware_nsx/common/config.py index 5e41dfcc2e..b9908f0b7c 100644 --- a/vmware_nsx/common/config.py +++ b/vmware_nsx/common/config.py @@ -243,6 +243,20 @@ nsx_v3_opts = [ cfg.IntOpt('number_of_nested_groups', default=8, help=_("The number of nested NSGroups to use.")), + cfg.StrOpt('metadata_mode', + default=MetadataModes.DIRECT, + help=_("If set to access_network this enables a dedicated " + "connection to the metadata proxy for metadata server " + "access via Neutron router. If set to dhcp_host_route " + "this enables host route injection via the dhcp agent. " + "This option is only useful if running on a host that " + "does not support namespaces otherwise access_network " + "should be used.")), + cfg.BoolOpt('metadata_on_demand', + default=True, + help=_("If true, an internal metadata network will be created " + "for a router only when the router is attached to a " + "DHCP-disabled subnet.")), ] DEFAULT_STATUS_CHECK_INTERVAL = 2000 diff --git a/vmware_nsx/dhcp_meta/rpc.py b/vmware_nsx/dhcp_meta/rpc.py index 1f987797c5..cecd7872c9 100644 --- a/vmware_nsx/dhcp_meta/rpc.py +++ b/vmware_nsx/dhcp_meta/rpc.py @@ -51,7 +51,8 @@ def handle_port_metadata_access(plugin, context, port, is_delete=False): # For instances supporting DHCP option 121 and created in a # DHCP-enabled but isolated network. This method is useful # only when no network namespace support. - if (cfg.CONF.NSX.metadata_mode == config.MetadataModes.INDIRECT and + plugin_cfg = getattr(cfg.CONF, plugin.cfg_group) + if (plugin_cfg.metadata_mode == config.MetadataModes.INDIRECT and port.get('device_owner') == const.DEVICE_OWNER_DHCP): if not port.get('fixed_ips'): # If port does not have an IP, the associated subnet is in @@ -93,7 +94,10 @@ def handle_port_metadata_access(plugin, context, port, is_delete=False): def handle_router_metadata_access(plugin, context, router_id, interface=None): # For instances created in a DHCP-disabled network but connected to # a router. - if cfg.CONF.NSX.metadata_mode != config.MetadataModes.DIRECT: + # The parameter "interface" is only used as a Boolean flag to indicate + # whether to add (True) or delete (False) an internal metadata network. + plugin_cfg = getattr(cfg.CONF, plugin.cfg_group) + if plugin_cfg.metadata_mode != config.MetadataModes.DIRECT: LOG.debug("Metadata access network is disabled") return if not cfg.CONF.allow_overlapping_ips: @@ -108,12 +112,19 @@ def handle_router_metadata_access(plugin, context, router_id, interface=None): plugin, ctx_elevated, filters=device_filter) try: if ports: - if (interface and - not _find_metadata_port(plugin, ctx_elevated, ports)): - _create_metadata_access_network( - plugin, ctx_elevated, router_id) - elif len(ports) == 1: - # The only port left might be the metadata port + on_demand = getattr(plugin_cfg, 'metadata_on_demand', False) + if interface: + if (not on_demand or _find_dhcp_disabled_subnet( + plugin, ctx_elevated, ports)) and ( + not _find_metadata_port(plugin, ctx_elevated, ports)): + _create_metadata_access_network( + plugin, ctx_elevated, router_id) + elif (len(ports) == 1 and _find_metadata_port( + plugin, ctx_elevated, ports)) or (on_demand and + not _find_dhcp_disabled_subnet(plugin, ctx_elevated, ports)): + # Delete the internal metadata network if the router port + # is the last port left or no more DHCP-disabled subnet + # attached to the router. _destroy_metadata_access_network( plugin, ctx_elevated, router_id, ports) else: @@ -139,6 +150,14 @@ def _find_metadata_port(plugin, context, ports): return port +def _find_dhcp_disabled_subnet(plugin, context, ports): + for port in ports: + for fixed_ip in port['fixed_ips']: + subnet = plugin.get_subnet(context, fixed_ip['subnet_id']) + if not subnet['enable_dhcp']: + return subnet + + def _create_metadata_access_network(plugin, context, router_id): # Add network # Network name is likely to be truncated on NSX diff --git a/vmware_nsx/plugins/nsx_mh/plugin.py b/vmware_nsx/plugins/nsx_mh/plugin.py index 667ea1e7aa..b780ca6ea0 100644 --- a/vmware_nsx/plugins/nsx_mh/plugin.py +++ b/vmware_nsx/plugins/nsx_mh/plugin.py @@ -168,6 +168,7 @@ class NsxPluginV2(addr_pair_db.AllowedAddressPairsMixin, neutron_extensions.append_api_extensions_path( [vmware_nsx.NSX_EXT_PATH]) + self.cfg_group = 'NSX' # group name for nsx section in nsx.ini self.nsx_opts = cfg.CONF.NSX self.nsx_sync_opts = cfg.CONF.NSX_SYNC self.cluster = nsx_utils.create_nsx_cluster( @@ -1750,7 +1751,7 @@ class NsxPluginV2(addr_pair_db.AllowedAddressPairsMixin, # is removed (with the network) if this the last subnet # on the router self.handle_router_metadata_access( - context, router_id, interface=info) + context, router_id, interface=None) if not subnet: subnet = self._get_subnet(context, subnet_id) router = self._get_router(context, router_id) diff --git a/vmware_nsx/plugins/nsx_v3/plugin.py b/vmware_nsx/plugins/nsx_v3/plugin.py index 7ed0cbc093..a7ae3d9f7a 100644 --- a/vmware_nsx/plugins/nsx_v3/plugin.py +++ b/vmware_nsx/plugins/nsx_v3/plugin.py @@ -123,6 +123,7 @@ class NsxV3Plugin(addr_pair_db.AllowedAddressPairsMixin, self._nsx_client = nsx_client.NSX3Client(self._api_cluster) nsx_client._set_default_api_cluster(self._api_cluster) + self.cfg_group = 'nsx_v3' # group name for nsx_v3 section in nsx.ini self.base_binding_dict = { pbin.VIF_TYPE: pbin.VIF_TYPE_OVS, pbin.VIF_DETAILS: { @@ -563,6 +564,22 @@ class NsxV3Plugin(addr_pair_db.AllowedAddressPairsMixin, # TODO(berlin): cancel public external subnet announcement return super(NsxV3Plugin, self).delete_subnet(context, subnet_id) + def update_subnet(self, context, subnet_id, subnet): + updated_subnet = super(NsxV3Plugin, self).update_subnet( + context, subnet_id, subnet) + if cfg.CONF.nsx_v3.metadata_on_demand: + # If enable_dhcp is changed on a subnet attached to a router, + # update internal metadata network accordingly. + if 'enable_dhcp' in subnet['subnet']: + port_filters = {'device_owner': const.ROUTER_INTERFACE_OWNERS, + 'fixed_ips': {'subnet_id': [subnet_id]}} + ports = self.get_ports(context, filters=port_filters) + for port in ports: + nsx_rpc.handle_router_metadata_access( + self, context, port['device_id'], + interface=not updated_subnet['enable_dhcp']) + return updated_subnet + def _build_address_bindings(self, port): address_bindings = [] for fixed_ip in port['fixed_ips']: @@ -1493,7 +1510,8 @@ class NsxV3Plugin(addr_pair_db.AllowedAddressPairsMixin, info = super(NsxV3Plugin, self).remove_router_interface( context, router_id, interface_info) # Ensure the connection to the 'metadata access network' is removed - # (with the network) if this the last subnet on the router. + # (with the network) if this is the last DHCP-disabled subnet on the + # router. nsx_rpc.handle_router_metadata_access(self, context, router_id) return info diff --git a/vmware_nsx/tests/unit/extensions/test_metadata.py b/vmware_nsx/tests/unit/extensions/test_metadata.py index 9c1c60d3a8..bf408a45cf 100644 --- a/vmware_nsx/tests/unit/extensions/test_metadata.py +++ b/vmware_nsx/tests/unit/extensions/test_metadata.py @@ -28,11 +28,38 @@ from vmware_nsx.common import config class MetaDataTestCase(object): - def _metadata_setup(self, mode=config.MetadataModes.DIRECT): - cfg.CONF.set_override('metadata_mode', mode, 'NSX') + def _metadata_setup(self, mode=config.MetadataModes.DIRECT, + on_demand=False): + cfg.CONF.set_override('metadata_mode', mode, self.plugin.cfg_group) + if hasattr(getattr(cfg.CONF, self.plugin.cfg_group), + 'metadata_on_demand'): + cfg.CONF.set_override('metadata_on_demand', on_demand, + self.plugin.cfg_group) def _metadata_teardown(self): - cfg.CONF.set_override('metadata_mode', None, 'NSX') + cfg.CONF.set_override('metadata_mode', None, self.plugin.cfg_group) + if hasattr(getattr(cfg.CONF, self.plugin.cfg_group), + 'metadata_on_demand'): + cfg.CONF.set_override('metadata_on_demand', False, + self.plugin.cfg_group) + + def _check_metadata(self, expected_subnets, expected_ports): + subnets = self._list('subnets')['subnets'] + self.assertEqual(len(subnets), expected_subnets) + meta_net_id, meta_sub_id = None, None + meta_cidr = netaddr.IPNetwork('169.254.0.0/16') + for subnet in subnets: + cidr = netaddr.IPNetwork(subnet['cidr']) + if meta_cidr == cidr or meta_cidr in cidr.supernet(16): + meta_sub_id = subnet['id'] + meta_net_id = subnet['network_id'] + break + ports = self._list( + 'ports', + query_params='network_id=%s' % meta_net_id)['ports'] + self.assertEqual(len(ports), expected_ports) + meta_port_id = ports[0]['id'] if ports else None + return meta_net_id, meta_sub_id, meta_port_id def test_router_add_interface_subnet_with_metadata_access(self): self._metadata_setup() @@ -178,19 +205,8 @@ class MetaDataTestCase(object): with self.subnet() as s: self._router_interface_action('add', r['router']['id'], s['subnet']['id'], None) - subnets = self._list('subnets')['subnets'] - self.assertEqual(len(subnets), 2) - meta_cidr = netaddr.IPNetwork('169.254.0.0/16') - for subnet in subnets: - cidr = netaddr.IPNetwork(subnet['cidr']) - if meta_cidr == cidr or meta_cidr in cidr.supernet(16): - meta_sub_id = subnet['id'] - meta_net_id = subnet['network_id'] - ports = self._list( - 'ports', - query_params='network_id=%s' % meta_net_id)['ports'] - self.assertEqual(len(ports), 1) - meta_port_id = ports[0]['id'] + meta_net_id, meta_sub_id, meta_port_id = self._check_metadata( + expected_subnets=2, expected_ports=1) self._router_interface_action('remove', r['router']['id'], s['subnet']['id'], None) self._show('networks', meta_net_id, @@ -240,6 +256,54 @@ class MetaDataTestCase(object): webob.exc.HTTPOk.code) self._metadata_teardown() + def test_metadata_network_with_update_subnet_dhcp_enable(self): + self._metadata_setup(on_demand=True) + with self.router() as r: + # Create a DHCP-disabled subnet. + with self.subnet(enable_dhcp=False) as s: + self._router_interface_action('add', r['router']['id'], + s['subnet']['id'], None) + meta_net_id, meta_sub_id, meta_port_id = self._check_metadata( + expected_subnets=2, expected_ports=1) + # Update subnet to DHCP-enabled. + data = {'subnet': {'enable_dhcp': True}} + req = self.new_update_request('subnets', data, + s['subnet']['id']) + res = self.deserialize(self.fmt, req.get_response(self.api)) + self.assertEqual(True, res['subnet']['enable_dhcp']) + self._check_metadata(expected_subnets=1, expected_ports=0) + self._show('networks', meta_net_id, + webob.exc.HTTPNotFound.code) + self._show('ports', meta_port_id, + webob.exc.HTTPNotFound.code) + self._show('subnets', meta_sub_id, + webob.exc.HTTPNotFound.code) + self._metadata_teardown() + + def test_metadata_network_with_update_subnet_dhcp_disable(self): + self._metadata_setup(on_demand=True) + with self.router() as r: + # Create a DHCP-enabled subnet. + with self.subnet(enable_dhcp=True) as s: + self._router_interface_action('add', r['router']['id'], + s['subnet']['id'], None) + self._check_metadata(expected_subnets=1, expected_ports=0) + # Update subnet to DHCP-disabled. + data = {'subnet': {'enable_dhcp': False}} + req = self.new_update_request('subnets', data, + s['subnet']['id']) + res = self.deserialize(self.fmt, req.get_response(self.api)) + self.assertEqual(False, res['subnet']['enable_dhcp']) + meta_net_id, meta_sub_id, meta_port_id = self._check_metadata( + expected_subnets=2, expected_ports=1) + self._show('networks', meta_net_id, + webob.exc.HTTPOk.code) + self._show('ports', meta_port_id, + webob.exc.HTTPOk.code) + self._show('subnets', meta_sub_id, + webob.exc.HTTPOk.code) + self._metadata_teardown() + def test_metadata_dhcp_host_route(self): self._metadata_setup(config.MetadataModes.INDIRECT) subnets = self._list('subnets')['subnets'] diff --git a/vmware_nsx/tests/unit/nsx_mh/test_plugin.py b/vmware_nsx/tests/unit/nsx_mh/test_plugin.py index 5c380225d1..bbe956690c 100644 --- a/vmware_nsx/tests/unit/nsx_mh/test_plugin.py +++ b/vmware_nsx/tests/unit/nsx_mh/test_plugin.py @@ -889,6 +889,12 @@ class TestL3NatTestCase(L3NatTest, def test_floatingip_disassociate_notification(self): self.skipTest('not supported') + def test_metadata_network_with_update_subnet_dhcp_enable(self): + self.skipTest('not supported') + + def test_metadata_network_with_update_subnet_dhcp_disable(self): + self.skipTest('not supported') + class ExtGwModeTestCase(NsxPluginV2TestCase, test_ext_gw_mode.ExtGwModeIntTestCase):