Merge branch 'master' into resource-compiler

config.yaml

@@ -11,6 +11,7 @@ file-system-db:
 
 template-dir: /vagrant/templates
 
+resources-directory: /tmp/git
 resources-files-mask: /vagrant/resources/*/*.yaml
 resources-compiled-file: /vagrant/resources_compiled.py
 node_resource_template: /vagrant/resources/ro_node/

example.py

@@ -10,6 +10,11 @@ from solar.core import signals
 from solar.core import validation
 
 from solar.interfaces.db import get_db
+from solar.core.resource_provider import  GitProvider, RemoteZipProvider
+
+
+GIT_KEYSTONE_RESOURCE_URL = 'https://github.com/CGenie/keystone-resource'
+ZIP_KEYSTONE_RESOURCE_URL = 'https://github.com/CGenie/keystone-resource/archive/master.zip'
 
 import resources_compiled
 
@@ -44,11 +49,11 @@ def deploy():
     keystone_db = resource.create('keystone_db', 'resources/mariadb_keystone_db/', {'db_name': 'keystone_db', 'login_user': 'root'})
     keystone_db_user = resource.create('keystone_db_user', 'resources/mariadb_keystone_user/', {'new_user_name': 'keystone', 'new_user_password': 'keystone', 'login_user': 'root'})
 
-    keystone_config1 = resource.create('keystone_config1', 'resources/keystone_config/', {'config_dir': '/etc/solar/keystone', 'admin_token': 'admin'})
+    keystone_config1 = resource.create('keystone_config1', GitProvider(GIT_KEYSTONE_RESOURCE_URL, path='keystone_config'), {'config_dir': '/etc/solar/keystone', 'admin_token': 'admin'})
-    keystone_service1 = resource.create('keystone_service1', 'resources/keystone_service/', {'port': 5001, 'admin_port': 35357})
+    keystone_service1 = resource.create('keystone_service1', RemoteZipProvider(ZIP_KEYSTONE_RESOURCE_URL, 'keystone_service'), {'port': 5001, 'admin_port': 35357})
 
-    keystone_config2 = resource.create('keystone_config2', 'resources/keystone_config/', {'config_dir': '/etc/solar/keystone', 'admin_token': 'admin'})
+    keystone_config2 = resource.create('keystone_config2', GitProvider(GIT_KEYSTONE_RESOURCE_URL, 'keystone_config'), {'config_dir': '/etc/solar/keystone', 'admin_token': 'admin'})
-    keystone_service2 = resource.create('keystone_service2', 'resources/keystone_service/', {'port': 5002, 'admin_port': 35358})
+    keystone_service2 = resource.create('keystone_service2', GitProvider(GIT_KEYSTONE_RESOURCE_URL, 'keystone_service'), {'port': 5002, 'admin_port': 35358})
 
     haproxy_keystone_config = resource.create('haproxy_keystone1_config', 'resources/haproxy_service_config/', {'name': 'keystone_config', 'listen_port': 5000, 'servers':[], 'ports':[]})
     haproxy_config = resource.create('haproxy_config', 'resources/haproxy_config', {'configs_names':[], 'configs_ports':[], 'listen_ports':[], 'configs':[]})
@@ -57,10 +62,10 @@ def deploy():
     glance_db = resource.create('glance_db', 'resources/mariadb_db/', {'db_name': 'glance_db', 'login_user': 'root'})
     glance_db_user = resource.create('glance_db_user', 'resources/mariadb_user/', {'new_user_name': 'glance', 'new_user_password': 'glance', 'login_user': 'root'})
 
-    services_tenant = resource.create('glance_keystone_tenant', 'resources/keystone_tenant', {'tenant_name': 'services'})
+    services_tenant = resource.create('glance_keystone_tenant', GitProvider(GIT_KEYSTONE_RESOURCE_URL, 'keystone_tenant'), {'tenant_name': 'services'})
 
-    glance_keystone_user = resource.create('glance_keystone_user', 'resources/keystone_user', {'user_name': 'glance_admin', 'user_password': 'password1234', 'tenant_name': 'service_admins'})
+    glance_keystone_user = resource.create('glance_keystone_user', GitProvider(GIT_KEYSTONE_RESOURCE_URL, 'keystone_user'), {'user_name': 'glance_admin', 'user_password': 'password1234', 'tenant_name': 'service_admins'})
-    glance_keystone_role = resource.create('glance_keystone_role', 'resources/keystone_role', {'role_name': 'admin'})
+    glance_keystone_role = resource.create('glance_keystone_role', GitProvider(GIT_KEYSTONE_RESOURCE_URL, 'keystone_role'), {'role_name': 'admin'})
 
     # TODO: add api_host and registry_host -- they can be different! Currently 'ip' is used.
     glance_config = resource.create('glance_config', 'resources/glance_config/', {'api_port': 9393})
@@ -69,15 +74,15 @@ def deploy():
     # TODO: admin_port should be refactored, we need to rethink docker
     # container resource and make it common for all
     # resources used in this demo
-    glance_api_endpoint = resource.create('glance_api_endpoint', 'resources/keystone_service_endpoint/', {'adminurl': 'http://{{ip}}:{{admin_port}}', 'internalurl': 'http://{{ip}}:{{port}}', 'publicurl': 'http://{{ip}}:{{port}}', 'description': 'OpenStack Image Service', 'type': 'image'})
+    glance_api_endpoint = resource.create('glance_api_endpoint', GitProvider(GIT_KEYSTONE_RESOURCE_URL, 'keystone_service_endpoint'), {'adminurl': 'http://{{ip}}:{{admin_port}}', 'internalurl': 'http://{{ip}}:{{port}}', 'publicurl': 'http://{{ip}}:{{port}}', 'description': 'OpenStack Image Service', 'type': 'image'})
     # TODO: ports value 9393 is a HACK -- fix glance_api_container's port and move to some config
     # TODO: glance registry container's API port needs to point to haproxy_config
     haproxy_glance_api_config = resource.create('haproxy_glance_api_config', 'resources/haproxy_service_config/', {'name': 'glance_api_config', 'listen_port': 9292, 'servers': [], 'ports':[{'value': 9393}]})
 
-    admin_tenant = resource.create('admin_tenant', 'resources/keystone_tenant', {'tenant_name': 'admin'})
+    admin_tenant = resource.create('admin_tenant', GitProvider(GIT_KEYSTONE_RESOURCE_URL, 'keystone_tenant'), {'tenant_name': 'admin'})
-    admin_user = resource.create('admin_user', 'resources/keystone_user', {'user_name': 'admin', 'user_password': 'admin'})
+    admin_user = resource.create('admin_user', GitProvider(GIT_KEYSTONE_RESOURCE_URL, 'keystone_user'), {'user_name': 'admin', 'user_password': 'admin'})
-    admin_role = resource.create('admin_role', 'resources/keystone_role', {'role_name': 'admin'})
+    admin_role = resource.create('admin_role', GitProvider(GIT_KEYSTONE_RESOURCE_URL, 'keystone_role'), {'role_name': 'admin'})
-    keystone_service_endpoint = resource.create('keystone_service_endpoint', 'resources/keystone_service_endpoint/', {'adminurl': 'http://{{ip}}:{{admin_port}}/v2.0', 'internalurl': 'http://{{ip}}:{{port}}/v2.0', 'publicurl': 'http://{{ip}}:{{port}}/v2.0', 'description': 'OpenStack Identity Service', 'type': 'identity'})
+    keystone_service_endpoint = resource.create('keystone_service_endpoint', GitProvider(GIT_KEYSTONE_RESOURCE_URL, 'keystone_service_endpoint'), {'adminurl': 'http://{{ip}}:{{admin_port}}/v2.0', 'internalurl': 'http://{{ip}}:{{port}}/v2.0', 'publicurl': 'http://{{ip}}:{{port}}/v2.0', 'description': 'OpenStack Identity Service', 'type': 'identity'})
 
 
     ####

solar/solar/core/resource/resource.py

@@ -177,8 +177,15 @@ def prepare_meta(meta):
 
 
 def create(name, base_path, args, tags=[], connections={}):
+    from solar.core import resource_provider
+
+    if isinstance(base_path, resource_provider.BaseProvider):
+        base_path = base_path.directory
+
     if not os.path.exists(base_path):
-        raise Exception('Base resource does not exist: {0}'.format(base_path))
+        raise Exception(
+            'Base resource does not exist: {0}'.format(base_path)
+        )
 
     base_meta_file = os.path.join(base_path, 'meta.yaml')
 

solar/solar/core/resource_provider.py

@@ -0,0 +1,96 @@
+import os
+import requests
+import StringIO
+import subprocess
+import zipfile
+
+from solar import utils
+
+
+class BaseProvider(object):
+    def run(self):
+        pass
+
+
+class DirectoryProvider(BaseProvider):
+    def __init__(self, directory):
+        self.directory = directory
+
+
+class GitProvider(BaseProvider):
+    def __init__(self, repository, path='.'):
+        self.repository = repository
+        self.path = path
+
+        resources_directory = self._resources_directory()
+
+        if not os.path.exists(resources_directory):
+            self._clone_repo()
+
+        if path != '.':
+            self.directory = os.path.join(resources_directory, path)
+        else:
+            self.directory = resources_directory
+
+    def _resources_directory(self):
+        repo_name = os.path.split(self.repository)[1]
+
+        return os.path.join(
+            utils.read_config()['resources-directory'],
+            repo_name
+        )
+
+    def _clone_repo(self):
+        resources_directory = self._resources_directory()
+
+        with open('/tmp/git-provider.yaml', 'w') as f:
+            f.write("""
+---
+
+- hosts: all
+  tasks:
+    - git: repo={repository} dest={destination} clone={clone} update=yes
+            """.format(
+                repository=self.repository,
+                destination=resources_directory,
+                clone='yes'
+            ))
+
+        subprocess.check_call([
+            'ansible-playbook',
+            '-i', '"localhost,"',
+            '-c', 'local',
+            '/tmp/git-provider.yaml'
+        ])
+
+
+class RemoteZipProvider(BaseProvider):
+    """Download & extract zip from some URL.
+
+    Assumes zip structure of the form:
+    <group-name>
+      <resource1>
+      <resource2>
+      ...
+    """
+
+    def __init__(self, url, path='.'):
+        self.url = url
+        self.path = path
+
+        r = requests.get(url)
+        s = StringIO.StringIO(r.content)
+        z = zipfile.ZipFile(s)
+
+        group_name = os.path.dirname(z.namelist()[0])
+        base_resources_directory = utils.read_config()['resources-directory']
+        resources_directory = os.path.join(
+            base_resources_directory, group_name
+        )
+        if not os.path.exists(resources_directory):
+            z.extractall(base_resources_directory)
+
+        if path != '.':
+            self.directory = os.path.join(resources_directory, path)
+        else:
+            self.directory = resources_directory