From b004985c245e8daba4102c7c1f773a88fce21d4a Mon Sep 17 00:00:00 2001 From: Aaron Rosen Date: Tue, 24 May 2016 17:12:57 -0400 Subject: [PATCH] NSX-v3: Initial framework for api-replay-mode This patch includes the initial framework to allow existing neutron deployments running different backends to be migrated over to the nsx-v3 plugin. The main logic that is required to do this is to allow the ability of an id to be specified for a given resource. This patch makes this possible with the addition of a new extension api-replay. The reason why a new extension is needed is because the RESOURCE_MAP is loaded after the plugin is loaded. Therefore, there is no way for me to change the mapping directly in the plugin without creating an extension to do so. This patch also adds support for migrating the router-uplink and floatingips which was missing in the previous patchset. Here's an example output of the migration tool running: http://codepad.org/I7x6Rq3u Change-Id: I2ee9778374a8d137e06125f2732524c7c662c002 --- vmware_nsx/api_replay/__init__.py | 0 vmware_nsx/api_replay/cli.py | 2 +- vmware_nsx/api_replay/client.py | 79 +++++++++++++------ vmware_nsx/api_replay/utils.py | 41 ++++++++++ vmware_nsx/common/config.py | 6 ++ vmware_nsx/extensions/api_replay.py | 73 +++++++++++++++++ .../plugins/nsx_v3/api_replay/__init__.py | 0 vmware_nsx/plugins/nsx_v3/plugin.py | 62 ++++++++++++--- .../tests/unit/nsx_v3/test_api_replay.py | 45 +++++++++++ 9 files changed, 273 insertions(+), 35 deletions(-) create mode 100644 vmware_nsx/api_replay/__init__.py create mode 100644 vmware_nsx/api_replay/utils.py create mode 100644 vmware_nsx/extensions/api_replay.py create mode 100644 vmware_nsx/plugins/nsx_v3/api_replay/__init__.py create mode 100644 vmware_nsx/tests/unit/nsx_v3/test_api_replay.py diff --git a/vmware_nsx/api_replay/__init__.py b/vmware_nsx/api_replay/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/vmware_nsx/api_replay/cli.py b/vmware_nsx/api_replay/cli.py index e859f59b09..5aab5eb661 100644 --- a/vmware_nsx/api_replay/cli.py +++ b/vmware_nsx/api_replay/cli.py @@ -12,7 +12,7 @@ import argparse -from vmware_nsx.plugins.nsx_v3.api_replay import client +from vmware_nsx.api_replay import client class ApiReplayCli(object): diff --git a/vmware_nsx/api_replay/client.py b/vmware_nsx/api_replay/client.py index f8b73c6bf0..dbf3e17920 100644 --- a/vmware_nsx/api_replay/client.py +++ b/vmware_nsx/api_replay/client.py @@ -47,6 +47,7 @@ class ApiReplayClient(object): self.migrate_security_groups() self.migrate_routers() self.migrate_networks_subnets_ports() + self.migrate_floatingips() def find_subnet_by_id(self, subnet_id, subnets): for subnet in subnets: @@ -104,7 +105,7 @@ class ApiReplayClient(object): dest_sec_group = self.have_id(sg['id'], dest_sec_groups) # If the security group already exists on the the dest_neutron if dest_sec_group: - # make sure all the security group rules are theree and + # make sure all the security group rules are there and # create them if not for sg_rule in sg['security_group_rules']: if(self.have_id(sg_rule['id'], @@ -126,18 +127,19 @@ class ApiReplayClient(object): else: sg_rules = sg.pop('security_group_rules') try: - print(self.dest_neutron.create_security_group( - {'security_group': sg})) + new_sg = self.dest_neutron.create_security_group( + {'security_group': sg}) + print ("Created security-group %s" % new_sg) except Exception as e: # TODO(arosen): improve exception handing here. print (e) - pass for sg_rule in sg_rules: try: - print (self.dest_neutron.create_security_group_rule( - {'security_group_rule': sg_rule})) - except n_exc.Conflict: + rule = self.dest_neutron.create_security_group_rule( + {'security_group_rule': sg_rule}) + print ("created security group rule %s " % rule['id']) + except Exception: # NOTE(arosen): when you create a default # security group it is automatically populated # with some rules. When we go to create the rules @@ -155,13 +157,15 @@ class ApiReplayClient(object): if dest_router is False: drop_router_fields = ['status', 'routes', + 'ha', 'external_gateway_info'] body = self.drop_fields(router, drop_router_fields) - print (self.dest_neutron.create_router( - {'router': body})) + new_router = (self.dest_neutron.create_router( + {'router': body})) + print ("created router %s" % new_router) def migrate_networks_subnets_ports(self): - """Migrates routers from source to dest neutron.""" + """Migrates networks/ports/router-uplinks from src to dest neutron.""" source_ports = self.source_neutron.list_ports()['ports'] source_subnets = self.source_neutron.list_subnets()['subnets'] source_networks = self.source_neutron.list_networks()['networks'] @@ -185,10 +189,12 @@ class ApiReplayClient(object): 'port_security_enabled', 'binding:vif_details', 'binding:vif_type', - 'binding:host_id'] + 'binding:host_id', 'qos_policy_id'] drop_network_fields = ['status', 'subnets', 'availability_zones', - 'created_at', 'updated_at', 'tags'] + 'created_at', 'updated_at', 'tags', + 'qos_policy_id', 'ipv4_address_scope', + 'ipv6_address_scope', 'mtu'] for network in source_networks: body = self.drop_fields(network, drop_network_fields) @@ -202,7 +208,7 @@ class ApiReplayClient(object): if self.have_id(network['id'], dest_networks) is False: created_net = self.dest_neutron.create_network( {'network': body})['network'] - print ("Created network: " + created_net['id']) + print ("Created network: %s " % created_net) for subnet_id in network['subnets']: subnet = self.find_subnet_by_id(subnet_id, source_subnets) @@ -240,24 +246,49 @@ class ApiReplayClient(object): # only create port if the dest server doesn't have it if self.have_id(port['id'], dest_ports) is False: + if port['device_owner'] == 'network:router_gateway': + body = { + "external_gateway_info": + {"network_id": port['network_id']}} + router_uplink = self.dest_neutron.update_router( + port['device_id'], # router_id + {'router': body}) + print ("Uplinked router %s" % router_uplink) + continue - if port['device_owner'] in ['network:router_interface', - 'network:router_gateway']: - if port['allowed_address_pairs'] == []: - del body['allowed_address_pairs'] - created_port = self.dest_neutron.create_port( - {'port': body})['port'] - print ("Created port: " + created_port['id']) + # Let the neutron dhcp-agent recreate this on it's own + if port['device_owner'] == 'network:dhcp': + continue + + # ignore these as we create them ourselves later + if port['device_owner'] == 'network:floatingip': + continue if port['device_owner'] == 'network:router_interface': try: - print (self.dest_neutron.add_interface_router( + # uplink router_interface ports + self.dest_neutron.add_interface_router( port['device_id'], - {'port_id': port['id']})) - + {'subnet_id': created_subnet['id']}) + print ("Uplinked router %s to subnet %s" % + (port['device_id'], created_subnet['id'])) + continue except n_exc.BadRequest as e: # NOTE(arosen): this occurs here if you run the # script multiple times as we don't track this. print (e) + raise - # TODO(arosen): handle 'network:router_gateway' uplinking + created_port = self.dest_neutron.create_port( + {'port': body})['port'] + print ("Created port: " + created_port['id']) + + def migrate_floatingips(self): + """Migrates floatingips from source to dest neutron.""" + source_fips = self.source_neutron.list_floatingips()['floatingips'] + drop_fip_fields = ['status', 'router_id', 'id'] + + for source_fip in source_fips: + body = self.drop_fields(source_fip, drop_fip_fields) + fip = self.dest_neutron.create_floatingip({'floatingip': body}) + print ("Created floatingip %s" % fip) diff --git a/vmware_nsx/api_replay/utils.py b/vmware_nsx/api_replay/utils.py new file mode 100644 index 0000000000..0c915d8fa5 --- /dev/null +++ b/vmware_nsx/api_replay/utils.py @@ -0,0 +1,41 @@ +# Copyright 2016 VMware, Inc. +# All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +from neutron.api.v2 import attributes +from oslo_config import cfg +from oslo_utils import uuidutils +import webob.exc + + +def _fixup_res_dict(context, attr_name, res_dict, check_allow_post=True): + # This method is a replacement of _fixup_res_dict which is used in + # neutron.plugin.common.utils. All this mock does is insert a uuid + # for the id field if one is not found ONLY if running in api_replay_mode. + if cfg.CONF.api_replay_mode and 'id' not in res_dict: + res_dict['id'] = uuidutils.generate_uuid() + attr_info = attributes.RESOURCE_ATTRIBUTE_MAP[attr_name] + try: + attributes.populate_tenant_id(context, res_dict, attr_info, True) + attributes.verify_attributes(res_dict, attr_info) + except webob.exc.HTTPBadRequest as e: + # convert webob exception into ValueError as these functions are + # for internal use. webob exception doesn't make sense. + raise ValueError(e.detail) + + attributes.fill_default_value(attr_info, res_dict, + check_allow_post=check_allow_post) + attributes.convert_value(attr_info, res_dict) + return res_dict diff --git a/vmware_nsx/common/config.py b/vmware_nsx/common/config.py index 84ef400436..6895369c64 100644 --- a/vmware_nsx/common/config.py +++ b/vmware_nsx/common/config.py @@ -230,6 +230,12 @@ nsx_common_opts = [ "parameter to tooz coordinator. By default, value is " "None and oslo_concurrency is used for single-node " "lock management.")), + cfg.BoolOpt('api_replay_mode', + default=False, + help=_("If true, the server then allows the caller to " + "specify the id of resources. This should only " + "be enabled in order to allow one to migrate an " + "existing install of neutron to the nsx-v3 plugin.")), ] nsx_v3_opts = [ diff --git a/vmware_nsx/extensions/api_replay.py b/vmware_nsx/extensions/api_replay.py new file mode 100644 index 0000000000..8b44916e65 --- /dev/null +++ b/vmware_nsx/extensions/api_replay.py @@ -0,0 +1,73 @@ +# Copyright 2016 VMware, Inc. +# +# All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + +from neutron.api import extensions + + +RESOURCE_ATTRIBUTE_MAP = { + 'ports': { + 'id': {'allow_post': True, 'allow_put': False, + 'validate': {'type:uuid': None}, + 'is_visible': True}, + }, + 'networks': { + 'id': {'allow_post': True, 'allow_put': False, + 'validate': {'type:uuid': None}, + 'is_visible': True}, + }, + 'security_groups': { + 'id': {'allow_post': True, 'allow_put': False, + 'validate': {'type:uuid': None}, + 'is_visible': True}, + }, + 'security_group_rules': { + 'id': {'allow_post': True, 'allow_put': False, + 'validate': {'type:uuid': None}, + 'is_visible': True}, + }, + 'routers': { + 'id': {'allow_post': True, 'allow_put': False, + 'validate': {'type:uuid': None}, + 'is_visible': True}, + }, +} + + +class Api_replay(extensions.ExtensionDescriptor): + """Extension for api replay which allows us to specify ids of resources.""" + + @classmethod + def get_name(cls): + return "Api Replay" + + @classmethod + def get_alias(cls): + return 'api-replay' + + @classmethod + def get_description(cls): + return "Enables mode to allow api to be replayed" + + @classmethod + def get_updated(cls): + return "2016-05-05T10:00:00-00:00" + + def get_extended_resources(self, version): + if version == "2.0": + return RESOURCE_ATTRIBUTE_MAP + else: + return {} diff --git a/vmware_nsx/plugins/nsx_v3/api_replay/__init__.py b/vmware_nsx/plugins/nsx_v3/api_replay/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/vmware_nsx/plugins/nsx_v3/plugin.py b/vmware_nsx/plugins/nsx_v3/plugin.py index 5fc4e4ad8c..45c6cf8776 100644 --- a/vmware_nsx/plugins/nsx_v3/plugin.py +++ b/vmware_nsx/plugins/nsx_v3/plugin.py @@ -12,6 +12,8 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. + +import mock import netaddr import six @@ -66,6 +68,7 @@ from oslo_utils import importutils from oslo_utils import uuidutils from vmware_nsx._i18n import _, _LE, _LI, _LW +from vmware_nsx.api_replay import utils as api_replay_utils from vmware_nsx.common import config # noqa from vmware_nsx.common import exceptions as nsx_exc from vmware_nsx.common import locking @@ -196,6 +199,8 @@ class NsxV3Plugin(agentschedulers_db.AZDhcpAgentSchedulerDbMixin, "switching profile: %s") % NSX_V3_DHCP_PROFILE_NAME raise nsx_exc.NsxPluginException(msg) self._unsubscribe_callback_events() + if cfg.CONF.api_replay_mode: + self.supported_extension_aliases.append('api-replay') # translate configured transport zones/rotuers names to uuid self._translate_configured_names_2_uuids() @@ -1658,8 +1663,12 @@ class NsxV3Plugin(agentschedulers_db.AZDhcpAgentSchedulerDbMixin, name = utils.get_name_and_uuid( router_name, port['id'], tag='port') self._port_client.update(nsx_port_id, None, name=name) - return super(NsxV3Plugin, self).update_router( - context, router_id, router) + + # NOTE(arosen): the mock.patch here is needed for api_replay_mode + with mock.patch("neutron.plugins.common.utils._fixup_res_dict", + side_effect=api_replay_utils._fixup_res_dict): + return super(NsxV3Plugin, self).update_router( + context, router_id, router) except nsx_exc.ResourceNotFound: with context.session.begin(subtransactions=True): router_db = self._get_router(context, router_id) @@ -1742,9 +1751,11 @@ class NsxV3Plugin(agentschedulers_db.AZDhcpAgentSchedulerDbMixin, # to routers self._validate_multiple_subnets_routers(context, router_id, interface_info) - - info = super(NsxV3Plugin, self).add_router_interface( - context, router_id, interface_info) + # NOTE(arosen): the mock.patch here is needed for api_replay_mode + with mock.patch("neutron.plugins.common.utils._fixup_res_dict", + side_effect=api_replay_utils._fixup_res_dict): + info = super(NsxV3Plugin, self).add_router_interface( + context, router_id, interface_info) try: subnet = self.get_subnet(context, info['subnet_ids'][0]) port = self.get_port(context, info['port_id']) @@ -1860,11 +1871,15 @@ class NsxV3Plugin(agentschedulers_db.AZDhcpAgentSchedulerDbMixin, return info def create_floatingip(self, context, floatingip): - new_fip = super(NsxV3Plugin, self).create_floatingip( - context, floatingip, initial_status=( - const.FLOATINGIP_STATUS_ACTIVE - if floatingip['floatingip']['port_id'] - else const.FLOATINGIP_STATUS_DOWN)) + # NOTE(arosen): the mock.patch here is needed for api_replay_mode + with mock.patch("neutron.plugins.common.utils._fixup_res_dict", + side_effect=api_replay_utils._fixup_res_dict): + + new_fip = super(NsxV3Plugin, self).create_floatingip( + context, floatingip, initial_status=( + const.FLOATINGIP_STATUS_ACTIVE + if floatingip['floatingip']['port_id'] + else const.FLOATINGIP_STATUS_DOWN)) router_id = new_fip['router_id'] if not router_id: return new_fip @@ -1973,6 +1988,33 @@ class NsxV3Plugin(agentschedulers_db.AZDhcpAgentSchedulerDbMixin, super(NsxV3Plugin, self).disassociate_floatingips( context, port_id, do_notify=False) + def _ensure_default_security_group(self, context, tenant_id): + # NOTE(arosen): if in replay mode we'll create all the default + # security groups for the user with their data so we don't + # want this to be called. + if (cfg.CONF.api_replay_mode is False): + return super(NsxV3Plugin, self)._ensure_default_security_group( + context, tenant_id) + + def _stub__validate_name_not_default(self): + # NOTE(arosen): if in replay mode we need stub out this validator to + # all default security groups to be created via the api + if cfg.CONF.api_replay_mode: + def _pass(data, foo=None): + pass + ext_sg.validators.validators['type:name_not_default'] = _pass + + def get_security_groups(self, context, filters=None, fields=None, + sorts=None, limit=None, + marker=None, page_reverse=False, default_sg=False): + + self._stub__validate_name_not_default() + return super(NsxV3Plugin, self).get_security_groups( + context, filters=filters, fields=fields, + sorts=sorts, limit=limit, + marker=marker, page_reverse=page_reverse, + default_sg=default_sg) + def create_security_group(self, context, security_group, default_sg=False): secgroup = security_group['security_group'] secgroup['id'] = secgroup.get('id') or uuidutils.generate_uuid() diff --git a/vmware_nsx/tests/unit/nsx_v3/test_api_replay.py b/vmware_nsx/tests/unit/nsx_v3/test_api_replay.py new file mode 100644 index 0000000000..0fe4cbc025 --- /dev/null +++ b/vmware_nsx/tests/unit/nsx_v3/test_api_replay.py @@ -0,0 +1,45 @@ +# Copyright (c) 2015 OpenStack Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from vmware_nsx.tests.unit.nsx_v3 import test_plugin + + +# FIXME(arosen): - these tests pass but seem to break the other tests +# as the attribute map doesn't get reset after each test class. I tried +# backing it up and restoring it here though that doesn't seem to be doing +# the trick either... +class TestApiReplay(test_plugin.NsxV3PluginTestCaseMixin): + + def setUp(self, plugin=None, ext_mgr=None, service_plugins=None): + # enables api_replay_mode for these tests + super(TestApiReplay, self).setUp() + + def test_create_port_specify_id(self): + self.skipTest("...fixme...") + specified_network_id = '555e762b-d7a1-4b44-b09b-2a34ada56c9f' + specified_port_id = 'e55e762b-d7a1-4b44-b09b-2a34ada56c9f' + network_res = self._create_network(self.fmt, + 'test-network', + True, + arg_list=('id',), + id=specified_network_id) + network = self.deserialize(self.fmt, network_res) + self.assertEqual(specified_network_id, network['network']['id']) + port_res = self._create_port(self.fmt, + network['network']['id'], + arg_list=('id',), + id=specified_port_id) + port = self.deserialize(self.fmt, port_res) + self.assertEqual(specified_port_id, port['port']['id'])