Advanced Networking Implemented

Change-Id: Ibebc5de57cdbba10ebeb0cbc600e2fade96d96b4

doc/source/index.rst

@@ -59,7 +59,23 @@ Configure
 
     # Directory where conductor's data directory located.
     # "data" must be subdirectory to this.
-    data_dir = /etc/murano-conductor
+    data_dir = /etc/murano-conductor/metadata-cache
+
+    # Provide url to Murano Metadata repository
+    murano_metadata_url = http://localhost:8084
+
+    # Maximum number of environments that can be processed simultaneously
+    max_environments = 20
+
+    # Maximum number of VMs per environment
+    max_hosts = 250
+
+    # Template IP address for generating environment subnet cidrs
+    env_ip_template = 10.0.0.0
+
+    # Enforces flat network topology by default.
+    # If set to "False", routed topology will be used
+    flat_by_default = False
 
     [keystone]
     # URL of OpenStack KeyStone service REST API.
@@ -89,6 +105,14 @@ Configure
     # Valid endpoint types: publicURL (default), internalURL, adminURL
     endpoint_type = publicURL
 
+    [neutron]
+    # Optional CA cert file to use in SSL connections
+    #ca_cert =
+    # Allow self signed server certificate
+    insecure = False
+    # Valid endpoint types: publicURL (default), internalURL, adminURL
+    endpoint_type = publicURL
+
     [rabbitmq]
     # Connection parameters to RabbitMQ service
 

etc/conductor.conf

@@ -23,6 +23,16 @@ murano_metadata_url = http://localhost:8084
 # Maximum number of environments that can be processed simultaneously
 max_environments = 20
 
+# Maximum number of VMs per environment
+max_hosts = 250
+
+# Template IP address for generating environment subnet cidrs
+env_ip_template = 10.0.0.0
+
+# Enforces flat network topology by default.
+# If set to "False", routed topology will be used
+flat_by_default = False
+
 [keystone]
 # URL of OpenStack KeyStone service REST API.
 # Typically only hostname (or IP) needs to be changed
@@ -51,6 +61,14 @@ insecure = False
 # Valid endpoint types: publicURL (default), internalURL, adminURL
 endpoint_type = publicURL
 
+[neutron]
+# Optional CA cert file to use in SSL connections
+#ca_cert =
+# Allow self signed server certificate
+insecure = False
+# Valid endpoint types: publicURL (default), internalURL, adminURL
+endpoint_type = publicURL
+
 [rabbitmq]
 # Connection parameters to RabbitMQ service
 

muranoconductor/app.py

@@ -30,6 +30,7 @@ from muranocommon.helpers.token_sanitizer import TokenSanitizer
 from muranoconductor import metadata
 import vm_agent
 import cloud_formation
+import network
 
 log = logging.getLogger(__name__)
 

muranoconductor/commands/dispatcher.py

@@ -15,17 +15,19 @@
 
 import command
 import cloud_formation
+import network
 import vm_agent
 
 
 class CommandDispatcher(command.CommandBase):
-    def __init__(self, environment, rmqclient, token,
-                 tenant_id, reporter):
+    def __init__(self, environment, rmqclient, token, tenant_id, reporter):
         self._command_map = {
             'cf': cloud_formation.HeatExecutor(environment, token, tenant_id,
                                                reporter),
-            'agent': vm_agent.VmAgentExecutor(environment, rmqclient,
-                                              reporter)
+            'agent': vm_agent.VmAgentExecutor(
+                environment, rmqclient, reporter),
+
+            'net': network.NeutronExecutor(tenant_id, token)
         }
 
     def execute(self, name, **kwargs):

muranoconductor/commands/network.py

@@ -0,0 +1,196 @@
+# Copyright (c) 2013 Mirantis Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import math
+import muranoconductor.config
+from keystoneclient.v2_0 import client as ksclient
+import netaddr
+from netaddr.strategy import ipv4
+from neutronclient.v2_0 import client as client
+from muranoconductor.commands.command import CommandBase
+
+
+class NeutronExecutor(CommandBase):
+    def __init__(self, tenant_id, token):
+        keystone_settings = muranoconductor.config.CONF.keystone
+        neutron_settings = muranoconductor.config.CONF.neutron
+
+        self.env_count = muranoconductor.config.CONF.max_environments
+        self.host_count = muranoconductor.config.CONF.max_hosts
+        self.address = muranoconductor.config.CONF.env_ip_template
+
+        self.cidr_waiting_per_router = {}
+        self.router_requests = []
+        self.network_requests = []
+
+        keystone_client = ksclient.Client(
+            endpoint=keystone_settings.auth_url,
+            cacert=keystone_settings.ca_file or None,
+            cert=keystone_settings.cert_file or None,
+            key=keystone_settings.key_file or None,
+            insecure=keystone_settings.insecure)
+
+        if not keystone_client.authenticate(
+                auth_url=keystone_settings.auth_url,
+                tenant_id=tenant_id,
+                token=token):
+            raise client.exceptions.Unauthorized()
+
+        neutron_url = keystone_client.service_catalog.url_for(
+            service_type='network',
+            endpoint_type=neutron_settings.endpoint_type)
+        self.neutron = client.Client(endpoint_url=neutron_url,
+                                     token=token,
+                                     ca_cert=neutron_settings.ca_cert or None,
+                                     insecure=neutron_settings.insecure)
+
+        self.command_map = {
+            "get_subnet": self._schedule_get_subnet,
+            "get_router": self._schedule_get_router,
+            "get_network": self._schedule_get_network
+        }
+
+    def execute(self, command, callback, **kwargs):
+        if command in self.command_map:
+            self.command_map[command](callback, **kwargs)
+
+    def has_pending_commands(self):
+        return len(self.cidr_waiting_per_router) + len(
+            self.router_requests) + len(self.network_requests) > 0
+
+    def execute_pending(self):
+        r1 = self._execute_pending_cidr_requests()
+        r2 = self._execute_pending_net_requests()
+        r3 = self._execute_pending_router_requests()
+        return r1 or r2 or r3
+
+    def _execute_pending_cidr_requests(self):
+        if not len(self.cidr_waiting_per_router):
+            return False
+        for router, callbacks in self.cidr_waiting_per_router.items():
+            results = self._get_subnet(router, len(callbacks))
+            for callback, result in zip(callbacks, results):
+                callback(result)
+        self.cidr_waiting_per_router = {}
+        return True
+
+    def _execute_pending_router_requests(self):
+        if not len(self.router_requests):
+            return False
+
+        routers = self.neutron.list_routers().get("routers")
+        if not len(routers):
+            routerId = None
+        else:
+            routerId = routers[0]["id"]
+
+        if len(routers) > 1:
+            for router in routers:
+                if "murano" in router["name"].lower():
+                    routerId = router["id"]
+                    break
+        for callback in self.router_requests:
+            callback(routerId)
+        self.router_requests = []
+        return True
+
+    def _execute_pending_net_requests(self):
+        if not len(self.network_requests):
+            return False
+
+        nets = self.neutron.list_networks()["networks"]
+        if not len(nets):
+            netId = None
+        else:
+            netId = nets[0]["id"]
+        if len(nets) > 1:
+            murano_id = None
+            ext_id = None
+            shared_id = None
+            for net in nets:
+                if "murano" in net.get("name").lower():
+                    murano_id = net["id"]
+                    break
+                if net.get("router:external") and not ext_id:
+                    ext_id = net["id"]
+                if net.get("shared") and not shared_id:
+                    shared_id = net["id"]
+            if murano_id:
+                netId = murano_id
+            elif ext_id:
+                netId = ext_id
+            elif shared_id:
+                netId = shared_id
+        for callback in self.network_requests:
+            callback(netId)
+        self.network_requests = []
+        return True
+
+    def _get_subnet(self, routerId, count):
+        if routerId == "*":
+            routerId = None
+        if routerId:
+            taken_cidrs = self._get_taken_cidrs_by_router(routerId)
+        else:
+            taken_cidrs = self._get_all_taken_cidrs()
+        results = []
+        for i in range(0, count):
+            res = self._generate_cidr(taken_cidrs)
+            results.append(res)
+            taken_cidrs.append(res)
+        return results
+
+    def _get_taken_cidrs_by_router(self, routerId):
+        ports = self.neutron.list_ports(device_id=routerId)["ports"]
+        subnet_ids = []
+        for port in ports:
+            for fixed_ip in port["fixed_ips"]:
+                subnet_ids.append(fixed_ip["subnet_id"])
+
+        all_subnets = self.neutron.list_subnets()["subnets"]
+        filtered_cidrs = [subnet["cidr"] for subnet in all_subnets if
+                          subnet["id"] in subnet_ids]
+
+        return filtered_cidrs
+
+    def _get_all_taken_cidrs(self):
+        return [subnet["cidr"] for subnet in
+                self.neutron.list_subnets()["subnets"]]
+
+    def _generate_cidr(self, taken_cidrs):
+        bits_for_envs = int(math.ceil(math.log(self.env_count, 2)))
+        bits_for_hosts = int(math.ceil(math.log(self.host_count, 2)))
+        width = ipv4.width
+        mask_width = width - bits_for_hosts - bits_for_envs
+        net = netaddr.IPNetwork(self.address + "/" + str(mask_width))
+        for subnet in net.subnet(width - bits_for_hosts):
+            if str(subnet) in taken_cidrs:
+                continue
+            return str(subnet)
+        return None
+
+    def _schedule_get_subnet(self, callback, **kwargs):
+        routerId = kwargs.get("routerId")
+        if not routerId:
+            routerId = "*"
+        if routerId in self.cidr_waiting_per_router:
+            self.cidr_waiting_per_router[routerId].append(callback)
+        else:
+            self.cidr_waiting_per_router[routerId] = [callback]
+
+    def _schedule_get_router(self, callback, **kwargs):
+        self.router_requests.append(callback)
+
+    def _schedule_get_network(self, callback, **kwargs):
+        self.network_requests.append(callback)

muranoconductor/config.py

@@ -58,6 +58,12 @@ heat_opts = [
     cfg.StrOpt('endpoint_type', default='publicURL')
 ]
 
+neutron_opts = [
+    cfg.BoolOpt('insecure', default=False),
+    cfg.StrOpt('ca_cert'),
+    cfg.StrOpt('endpoint_type', default='publicURL')
+]
+
 keystone_opts = [
     cfg.StrOpt('auth_url'),
     cfg.BoolOpt('insecure', default=False),
@@ -70,6 +76,7 @@ CONF = cfg.CONF
 CONF.register_opts(paste_deploy_opts, group='paste_deploy')
 CONF.register_opts(rabbit_opts, group='rabbitmq')
 CONF.register_opts(heat_opts, group='heat')
+CONF.register_opts(neutron_opts, group='neutron')
 CONF.register_opts(keystone_opts, group='keystone')
 CONF.register_opts(directories)
 CONF.register_opt(cfg.StrOpt('file_server'))
@@ -77,6 +84,9 @@ CONF.register_cli_opt(cfg.StrOpt('murano_metadata_url'))
 
 
 CONF.register_opt(cfg.IntOpt('max_environments', default=20))
+CONF.register_opt(cfg.IntOpt('max_hosts', default=250))
+CONF.register_opt(cfg.StrOpt('env_ip_template', default='10.0.0.0'))
+CONF.register_opt(cfg.BoolOpt('flat_by_default', default=False))
 
 
 CONF.import_opt('verbose', 'muranoconductor.openstack.common.log')

muranoconductor/network.py

@@ -0,0 +1,95 @@
+# Copyright (c) 2013 Mirantis Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from muranoconductor import xml_code_engine
+import muranoconductor.config
+
+from openstack.common import log as logging
+
+log = logging.getLogger(__name__)
+
+
+def get_available_subnet(engine, context, body, routerId=None, result=None):
+    command_dispatcher = context['/commandDispatcher']
+
+    def callback(result_value):
+        if result is not None:
+            context[result] = {"cidr": result_value}
+
+        success_handler = body.find('success')
+        if success_handler is not None:
+            engine.evaluate_content(success_handler, context)
+
+    command_dispatcher.execute(
+        name="net",
+        command="get_subnet",
+        routerId=routerId,
+        callback=callback)
+
+
+def get_default_router(engine, context, body, result=None):
+    command_dispatcher = context['/commandDispatcher']
+
+    def callback(result_value):
+        if result is not None:
+            context[result] = {"routerId": result_value}
+
+        success_handler = body.find('success')
+        if success_handler is not None:
+            engine.evaluate_content(success_handler, context)
+
+    command_dispatcher.execute(
+        name="net",
+        command="get_router",
+        callback=callback)
+
+
+def get_default_network(engine, context, body, result=None):
+    command_dispatcher = context['/commandDispatcher']
+
+    def callback(result_value):
+        if result is not None:
+            context[result] = {"networkId": result_value}
+
+        success_handler = body.find('success')
+        if success_handler is not None:
+            engine.evaluate_content(success_handler, context)
+
+    command_dispatcher.execute(
+        name="net",
+        command="get_network",
+        callback=callback)
+
+
+def get_network_topology(engine, context, body, result=None):
+    if muranoconductor.config.CONF.flat_by_default:
+        return "flat"
+    else:
+        return "routed"
+
+
+xml_code_engine.XmlCodeEngine.register_function(
+    get_available_subnet, "get-cidr")
+
+xml_code_engine.XmlCodeEngine.register_function(
+    get_default_router, "get-default-router-id")
+
+xml_code_engine.XmlCodeEngine.register_function(
+    get_default_network, "get-default-network-id")
+
+xml_code_engine.XmlCodeEngine.register_function(
+    get_default_router, "get-default-router-id")
+
+xml_code_engine.XmlCodeEngine.register_function(
+    get_network_topology, "get-net-topology")

muranoconductor/xml_code_engine.py

@@ -12,6 +12,7 @@
 # implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import uuid
 
 import xml.etree.ElementTree as etree
 import types
@@ -127,6 +128,10 @@ def _false_func(**kwargs):
     return False
 
 
+def _gen_id(**kwargs):
+    return uuid.uuid4().hex
+
+
 XmlCodeEngine.register_function(_dict_func, "map")
 XmlCodeEngine.register_function(_array_func, "list")
 XmlCodeEngine.register_function(_text_func, "text")
@@ -135,3 +140,4 @@ XmlCodeEngine.register_function(_function_func, "function")
 XmlCodeEngine.register_function(_null_func, "null")
 XmlCodeEngine.register_function(_true_func, "true")
 XmlCodeEngine.register_function(_false_func, "false")
+XmlCodeEngine.register_function(_gen_id, "uuid")

requirements.txt

@@ -14,3 +14,4 @@ deep
 murano-common>=0.2.2
 PyYAML>=3.1.0
 murano-metadataclient==0.4.a13.gd65dfd2
+python-neutronclient>=2.3.1

test-requirements.txt

@@ -7,4 +7,3 @@ nosexcover
 pep8
 sphinx>=1.1.2
 mockfs
-