Overall refresh to current install from source approach

Makefile

@@ -3,7 +3,7 @@ PYTHON := /usr/bin/env python
 
 lint:
 	@echo "Running flake8 tests: "
-	@flake8 --exclude hooks/charmhelpers hooks unit_tests tests
+	@flake8 --exclude hooks/charmhelpers actions hooks unit_tests tests
 	@echo "OK"
 	@echo "Running charm proof: "
 	@charm proof
@@ -27,7 +27,9 @@ test:
 	# raise_status() messages to stderr:
 	#   https://bugs.launchpad.net/amulet/+bug/1320357
 	@juju test -v -p AMULET_HTTP_PROXY --timeout 900 \
-        00-setup 14-basic-precise-icehouse 15-basic-trusty-icehouse
+        00-setup 14-basic-precise-icehouse 15-basic-trusty-icehouse \
+        16-basic-trusty-icehouse-git 17-basic-trusty-juno \
+        18-basic-trusty-juno-git
 
 publish: lint unit_test
 	bzr push lp:charms/glance

README.md

@@ -84,92 +84,84 @@ Swift providing backing image storage.
 Deploying from source
 ---------------------
 
-The minimal openstack-origin-git config required to deploy from source is:
+The minimum openstack-origin-git config required to deploy from source is:
 
   openstack-origin-git:
-      "{'glance':
+      "repositories:
-           {'repository': 'git://git.openstack.org/openstack/glance.git',
+         - {name: requirements,
-            'branch': 'stable/icehouse'}}"
+            repository: 'git://git.openstack.org/openstack/requirements',
+            branch: stable/juno}
+         - {name: glance,
+            repository: 'git://git.openstack.org/openstack/glance',
+            branch: stable/juno}"
 
-If you specify a 'requirements' repository, it will be used to update the
+Note that there are only two 'name' values the charm knows about: 'requirements'
-requirements.txt files of all other git repos that it applies to, before
+and 'glance'. These repositories must correspond to these 'name' values.
-they are installed:
+Additionally, the requirements repository must be specified first and the
+glance repository must be specified last. All other repostories are installed
+in the order in which they are specified.
+
+The following is a full list of current tip repos (may not be up-to-date):
 
   openstack-origin-git:
-      "{'requirements':
+      "repositories:
-           {'repository': 'git://git.openstack.org/openstack/requirements.git',
+         - {name: requirements,
-            'branch': 'master'},
+            repository: 'git://git.openstack.org/openstack/requirements',
-        'glance':
+            branch: master}
-           {'repository': 'git://git.openstack.org/openstack/glance.git',
+         - {name: oslo-concurrency,
-            'branch': 'master'}}"
+            repository: 'git://git.openstack.org/openstack/oslo.concurrency',
-
+            branch: master}
-Note that there are only two key values the charm knows about for the outermost
+         - {name: oslo-config,
-dictionary: 'glance' and 'requirements'. These repositories must correspond to
+            repository: 'git://git.openstack.org/openstack/oslo.config',
-these keys. If the requirements repository is specified, it will be installed
+            branch: master}
-first. The glance repository is always installed last.  All other repostories
+         - {name: oslo-db,
-will be installed in between.
+            repository: 'git://git.openstack.org/openstack/oslo.db',
-
+            branch: master}
-NOTE(coreycb): The following is temporary to keep track of the full list of
+         - {name: oslo-i18n,
-current tip repos (may not be up-to-date).
+            repository: 'git://git.openstack.org/openstack/oslo.i18n',
-
+            branch: master}
-  openstack-origin-git:
+         - {name: oslo-messaging,
-      "{'requirements':
+            repository: 'git://git.openstack.org/openstack/oslo.messaging',
-           {'repository': 'git://git.openstack.org/openstack/requirements.git',
+            branch: master}
-            'branch': 'master'},
+         - {name: oslo-serialization,
-        'glance-store':
+            repository: 'git://git.openstack.org/openstack/oslo.serialization',
-           {'repository': 'git://git.openstack.org/openstack/glance_store.git',
+            branch: master}
-            'branch': 'master'},
+         - {name: oslo-utils,
-        'keystonemiddleware:
+            repository: 'git://git.openstack.org/openstack/oslo.utils',
-           {'repository': 'git://git.openstack.org/openstack/keystonemiddleware.git',
+            branch: master}
-            'branch: 'master'},
+         - {name: oslo-vmware,
-        'oslo-concurrency':
+            repository: 'git://git.openstack.org/openstack/oslo.vmware',
-           {'repository': 'git://git.openstack.org/openstack/oslo.concurrency.git',
+            branch: master}
-            'branch: 'master'},
+         - {name: osprofiler,
-        'oslo-config':
+            repository: 'git://git.openstack.org/stackforge/osprofiler',
-           {'repository': 'git://git.openstack.org/openstack/oslo.config.git',
+            branch: master}
-            'branch: 'master'},
+         - {name: pbr,
-        'oslo-db':
+            repository: 'git://git.openstack.org/openstack-dev/pbr',
-           {'repository': 'git://git.openstack.org/openstack/oslo.db.git',
+            branch: master}
-            'branch: 'master'},
+         - {name: python-keystoneclient,
-        'oslo-i18n':
+            repository: 'git://git.openstack.org/openstack/python-keystoneclient',
-           {'repository': 'git://git.openstack.org/openstack/oslo.i18n.git',
+            branch: master}
-            'branch: 'master'},
+         - {name: python-swiftclient,
-        'oslo-messaging':
+            repository: 'git://git.openstack.org/openstack/python-swiftclient',
-           {'repository': 'git://git.openstack.org/openstack/oslo.messaging.git',
+            branch: master}
-            'branch: 'master'},
+         - {name: sqlalchemy-migrate,
-        'oslo-serialization':
+            repository: 'git://git.openstack.org/stackforge/sqlalchemy-migrate',
-           {'repository': 'git://git.openstack.org/openstack/oslo.serialization.git',
+            branch: master}
-            'branch: 'master'},
+         - {name: stevedore,
-        'oslo-utils':
+            repository: 'git://git.openstack.org/openstack/stevedore',
-           {'repository': 'git://git.openstack.org/openstack/oslo.utils.git',
+            branch: master}
-            'branch: 'master'},
+         - {name: wsme,
-        'oslo-vmware':
+            repository: 'git://git.openstack.org/stackforge/wsme',
-           {'repository': 'git://git.openstack.org/openstack/oslo.vmware.git',
+            branch: master}
-            'branch: 'master'},
+         - {name: keystonemiddleware,
-        'osprofiler':
+            repository: 'git://git.openstack.org/openstack/keystonemiddleware',
-           {'repository': 'git://git.openstack.org/stackforge/osprofiler.git',
+            branch: master}
-            'branch: 'master'},
+         - {name: glance-store,
-        'pbr':
+            repository: 'git://git.openstack.org/openstack/glance_store',
-           {'repository': 'git://git.openstack.org/openstack-dev/pbr.git',
+            branch: master}
-            'branch: 'master'},
+         - {name: glance,
-        'python-keystoneclient':
+            repository: 'git://git.openstack.org/openstack/glance',
-           {'repository': 'git://git.openstack.org/openstack/python-keystoneclient.git',
+            branch: master}"
-            'branch: 'master'},
-        'python-swiftclient':
-           {'repository': 'git://git.openstack.org/openstack/python-swiftclient.git',
-            'branch: 'master'},
-        'stevedore':
-           {'repository': 'git://git.openstack.org/openstack/stevedore.git',
-            'branch: 'master'},
-        'sqlalchemy-migrate':
-           {'repository': 'git://git.openstack.org/stackforge/sqlalchemy-migrate.git',
-            'branch: 'master'},
-        'wsme':
-           {'repository': 'git://git.openstack.org/stackforge/wsme.git',
-            'branch': 'master'},
-        'glance':
-           {'repository': 'git://git.openstack.org/openstack/glance.git',
-            'branch': 'master'}}"
 
 Contact Information
 -------------------

actions.yaml

@@ -0,0 +1,2 @@
+git-reinstall:
+  description: Reinstall glance from the openstack-origin-git repositories.

actions/git-reinstall

@@ -0,0 +1 @@
+git_reinstall.py

actions/git_reinstall.py

@@ -0,0 +1,40 @@
+#!/usr/bin/python
+import sys
+import traceback
+
+sys.path.append('hooks/')
+
+from charmhelpers.contrib.openstack.utils import (
+    git_install_requested,
+)
+
+from charmhelpers.core.hookenv import (
+    action_set,
+    action_fail,
+    config,
+)
+
+from glance_utils import (
+    git_install,
+)
+
+
+def git_reinstall():
+    """Reinstall from source and restart services.
+
+    If the openstack-origin-git config option was used to install openstack
+    from source git repositories, then this action can be used to reinstall
+    from updated git repositories, followed by a restart of services."""
+    if not git_install_requested():
+        action_fail('openstack-origin-git is not configured')
+        return
+
+    try:
+        git_install(config('openstack-origin-git'))
+    except:
+        action_set({'traceback': traceback.format_exc()})
+        action_fail('git-reinstall resulted in an unexpected error')
+
+
+if __name__ == '__main__':
+    git_reinstall()

charm-helpers-hooks.yaml

@@ -1,5 +1,4 @@
-#branch: lp:charm-helpers
+branch: lp:charm-helpers
-branch: /home/corey/src/charms/git/charm-helpers
 destination: hooks/charmhelpers
 include:
     - core

charm-helpers-tests.yaml

@@ -1,5 +1,4 @@
-#branch: lp:charm-helpers
+branch: lp:charm-helpers
-branch: /home/corey/src/charms/git/charm-helpers
 destination: tests/charmhelpers
 include:
     - contrib.amulet

config.yaml

@@ -15,23 +15,20 @@ options:
       provide a later version of OpenStack will trigger a software
       upgrade.
 
-      Note that when openstack-origin-git is specified, openstack-specific
+      Note that when openstack-origin-git is specified, openstack
-      packages will be installed from source rather than from the
+      specific packages will be installed from source rather than
-      openstack-origin repository.
+      from the openstack-origin repository.
   openstack-origin-git:
     default: None
     type: string
     description: |
-      Specifies a YAML-formatted two-dimensional array listing the git
+      Specifies a YAML-formatted dictionary listing the git
-      repositories and branches from which to install OpenStack and its
+      repositories and branches from which to install OpenStack and
-      dependencies.
+      its dependencies.
 
       Note that the installed config files will be determined based on
       the OpenStack release of the openstack-origin option.
 
-      Note also that this option is processed for the initial install
-      only. Setting this option after deployment is not supported.
-
       For more details see README.md.
   database-user:
     default: glance

hooks/glance_relations.py

@@ -105,8 +105,7 @@ def install_hook():
     apt_update(fatal=True)
     apt_install(determine_packages(), fatal=True)
 
-    # NOTE(coreycb): This is temporary for sstack proxy, unless we decide
+    # NOTE(coreycb): This is temporary until bug #1431286 is fixed.
-    # we need to code proxy support into the charms.
     os.environ["http_proxy"] = "http://squid.internal:3128"
     os.environ["https_proxy"] = "https://squid.internal:3128"
 
@@ -325,7 +324,10 @@ def config_changed():
         sync_db_with_multi_ipv6_addresses(config('database'),
                                           config('database-user'))
 
-    if not git_install_requested():
+    if git_install_requested():
+        if config_value_changed('openstack-origin-git'):
+            git_install(config('openstack-origin-git'))
+    else:
         if openstack_upgrade_available('glance-common'):
             juju_log('Upgrading OpenStack release')
             do_openstack_upgrade(CONFIGS)

hooks/glance_utils.py

@@ -47,6 +47,7 @@ from charmhelpers.contrib.openstack.utils import (
     get_os_codename_package,
     git_install_requested,
     git_clone_and_install,
+    git_src_dir,
     configure_installation_source,
     os_release,
 )
@@ -302,16 +303,16 @@ def setup_ipv6():
         apt_install('haproxy/trusty-backports', fatal=True)
 
 
-def git_install(projects):
+def git_install(projects_yaml):
     """Perform setup, and install git repos specified in yaml parameter."""
     if git_install_requested():
         git_pre_install()
-        git_clone_and_install(yaml.load(projects), core_project='glance')
+        git_clone_and_install(projects_yaml, core_project='glance')
-        git_post_install()
+        git_post_install(projects_yaml)
 
 
 def git_pre_install():
-    """Perform pre glance installation setup."""
+    """Perform glance pre-install setup."""
     dirs = [
         '/var/lib/glance',
         '/var/lib/glance/images',
@@ -339,26 +340,10 @@ def git_pre_install():
         write_file(l, '', owner='glance', group='glance', perms=0600)
 
 
-def git_post_install():
+def git_post_install(projects_yaml):
-    """Perform post glance installation setup."""
+    """Perform glance post-install setup."""
-    src_etc = os.path.join(charm_dir(), '/mnt/openstack-git/glance.git/etc/')
+    src_etc = os.path.join(git_src_dir(projects_yaml, 'glance'), 'etc')
     configs = {
-        'glance-api-paste': {
-            'src': os.path.join(src_etc, 'glance-api-paste.ini'),
-            'dest': '/etc/glance/glance-api-paste.ini',
-        },
-        'glance-api': {
-            'src': os.path.join(src_etc, 'glance-api.conf'),
-            'dest': '/etc/glance/glance-api.conf',
-        },
-        'glance-registry-paste': {
-            'src': os.path.join(src_etc, 'glance-registry-paste.ini'),
-            'dest': '/etc/glance/glance-registry-paste.ini',
-        },
-        'glance-registry': {
-            'src': os.path.join(src_etc, 'glance-registry.conf'),
-            'dest': '/etc/glance/glance-registry.conf',
-        },
         'glance-cache': {
             'src': os.path.join(src_etc, 'glance-cache.conf'),
             'dest': '/etc/glance/glance-cache.conf',
@@ -398,10 +383,13 @@ def git_post_install():
         'executable_name': '/usr/local/bin/glance-registry',
     }
 
-    render('upstart/glance.upstart', '/etc/init/glance-api.conf',
+    # NOTE(coreycb): Needs systemd support
-           glance_api_context, perms=0o644)
+    templates_dir = 'hooks/charmhelpers/contrib/openstack/templates'
-    render('upstart/glance.upstart', '/etc/init/glance-registry.conf',
+    templates_dir = os.path.join(charm_dir(), templates_dir)
-           glance_registry_context, perms=0o644)
+    render('git.upstart', '/etc/init/glance-api.conf',
+           glance_api_context, perms=0o644, templates_dir=templates_dir)
+    render('git.upstart', '/etc/init/glance-registry.conf',
+           glance_registry_context, perms=0o644, templates_dir=templates_dir)
 
     service_start('glance-api')
     service_start('glance-registry')

templates/upstart/glance.upstart

@@ -1,11 +0,0 @@
-description "{{ service_description }}"
-author "Juju {{ service_name }} Charm <juju@localhost>"
-
-start on runlevel [2345]
-stop on runlevel [!2345]
-
-respawn
-
-exec start-stop-daemon --start --chuid {{ user_name }} \
-            --chdir {{ start_dir }} --name {{ process_name }} \
-            --exec {{ executable_name }}

tests/10-basic-precise-essex

@@ -1,9 +0,0 @@
-#!/usr/bin/python
-
-"""Amulet tests on a basic glance deployment on precise-essex."""
-
-from basic_deployment import GlanceBasicDeployment
-
-if __name__ == '__main__':
-    deployment = GlanceBasicDeployment(series='precise')
-    deployment.run_tests()

tests/11-basic-precise-folsom

@@ -1,11 +0,0 @@
-#!/usr/bin/python
-
-"""Amulet tests on a basic glance deployment on precise-folsom."""
-
-from basic_deployment import GlanceBasicDeployment
-
-if __name__ == '__main__':
-    deployment = GlanceBasicDeployment(series='precise',
-                                         openstack='cloud:precise-folsom',
-                                         source='cloud:precise-updates/folsom')
-    deployment.run_tests()

tests/12-basic-precise-grizzly

@@ -1,11 +0,0 @@
-#!/usr/bin/python
-
-"""Amulet tests on a basic glance deployment on precise-grizzly."""
-
-from basic_deployment import GlanceBasicDeployment
-
-if __name__ == '__main__':
-    deployment = GlanceBasicDeployment(series='precise',
-                                         openstack='cloud:precise-grizzly',
-                                         source='cloud:precise-updates/grizzly')
-    deployment.run_tests()

tests/13-basic-precise-havana

@@ -1,11 +0,0 @@
-#!/usr/bin/python
-
-"""Amulet tests on a basic glance deployment on precise-havana."""
-
-from basic_deployment import GlanceBasicDeployment
-
-if __name__ == '__main__':
-    deployment = GlanceBasicDeployment(series='precise',
-                                         openstack='cloud:precise-havana',
-                                         source='cloud:precise-updates/havana')
-    deployment.run_tests()

tests/16-basic-trusty-icehouse-git

@@ -0,0 +1,9 @@
+#!/usr/bin/python
+
+"""Amulet tests on a basic Glance git deployment on trusty-icehouse."""
+
+from basic_deployment import GlanceBasicDeployment
+
+if __name__ == '__main__':
+    deployment = GlanceBasicDeployment(series='trusty', git=True)
+    deployment.run_tests()

tests/17-basic-trusty-juno

@@ -0,0 +1,11 @@
+#!/usr/bin/python
+
+"""Amulet tests on a basic Glance deployment on trusty-juno."""
+
+from basic_deployment import GlanceBasicDeployment
+
+if __name__ == '__main__':
+    deployment = GlanceBasicDeployment(series='trusty',
+                                       openstack='cloud:trusty-juno',
+                                       source='cloud:trusty-updates/juno')
+    deployment.run_tests()

tests/18-basic-trusty-juno-git

@@ -0,0 +1,12 @@
+#!/usr/bin/python
+
+"""Amulet tests on a basic Glance git deployment on trusty-juno."""
+
+from basic_deployment import GlanceBasicDeployment
+
+if __name__ == '__main__':
+    deployment = GlanceBasicDeployment(series='trusty',
+                                       openstack='cloud:trusty-juno',
+                                       source='cloud:trusty-updates/juno',
+                                       git=True)
+    deployment.run_tests()

tests/basic_deployment.py

@@ -23,9 +23,11 @@ class GlanceBasicDeployment(OpenStackAmuletDeployment):
 #    * Add tests with different storage back ends
 #    * Resolve Essex->Havana juju set charm bug
 
-    def __init__(self, series=None, openstack=None, source=None, stable=False):
+    def __init__(self, series=None, openstack=None, source=None, git=False,
+                 stable=False):
         '''Deploy the entire test environment.'''
         super(GlanceBasicDeployment, self).__init__(series, openstack, source, stable)
+        self.git = git
         self._add_services()
         self._add_relations()
         self._configure_services()
@@ -56,13 +58,24 @@ class GlanceBasicDeployment(OpenStackAmuletDeployment):
     def _configure_services(self):
         '''Configure all of the services.'''
         # NOTE(coreycb): Added the following temporarily to test deploy from source
-        glance_config = {'openstack-origin-git':
+        glance_config = {}
-                         "{'glance':"
+        if self.git:
-                         "   {'repository': 'git://git.openstack.org/openstack/glance.git',"
+            branch = 'stable/' + self._get_openstack_release_string()
-                         "    'branch': 'stable/icehouse'}}"}
+            openstack_origin_git = {
+                'repositories': [
+                    {'name': 'requirements',
+                     'repository': 'git://git.openstack.org/openstack/requirements',
+                     'branch': branch},
+                    {'name': 'glance',
+                     'repository': 'git://git.openstack.org/openstack/glance',
+                     'branch': branch},
+                ],
+                'directory': '/mnt/openstack-git',
+            }
+            glance_config['openstack-origin-git'] = yaml.dump(openstack_origin_git)
+
         keystone_config = {'admin-password': 'openstack',
                            'admin-token': 'ubuntutesting'}
-
         mysql_config = {'dataset-size': '50%'}
         configs = {'glance': glance_config,
                    'keystone': keystone_config,

unit_tests/__init__.py

@@ -1,3 +1,4 @@
 import sys
 
+sys.path.append('actions/')
 sys.path.append('hooks/')

unit_tests/test_actions_git_reinstall.py

@@ -0,0 +1,85 @@
+from mock import patch
+
+with patch('charmhelpers.core.hookenv.config') as config:
+    config.return_value = 'glance'
+    import glance_utils as utils  # noqa
+
+import git_reinstall
+
+from test_utils import (
+    CharmTestCase
+)
+
+TO_PATCH = [
+    'config',
+]
+
+
+openstack_origin_git = \
+    """repositories:
+         - {name: requirements,
+            repository: 'git://git.openstack.org/openstack/requirements',
+            branch: stable/juno}
+         - {name: glance,
+            repository: 'git://git.openstack.org/openstack/glance',
+            branch: stable/juno}"""
+
+
+class TestKeystoneActions(CharmTestCase):
+
+    def setUp(self):
+        super(TestKeystoneActions, self).setUp(git_reinstall, TO_PATCH)
+        self.config.side_effect = self.test_config.get
+
+    @patch.object(git_reinstall, 'action_set')
+    @patch.object(git_reinstall, 'action_fail')
+    @patch.object(git_reinstall, 'git_install')
+    def test_git_reinstall(self, git_install, action_fail, action_set):
+        self.test_config.set('openstack-origin-git', openstack_origin_git)
+
+        git_reinstall.git_reinstall()
+
+        git_install.assert_called_with(openstack_origin_git)
+        self.assertTrue(git_install.called)
+        self.assertFalse(action_set.called)
+        self.assertFalse(action_fail.called)
+
+    @patch.object(git_reinstall, 'action_set')
+    @patch.object(git_reinstall, 'action_fail')
+    @patch.object(git_reinstall, 'git_install')
+    @patch('charmhelpers.contrib.openstack.utils.config')
+    def test_git_reinstall_not_configured(self, _config, git_install,
+                                          action_fail, action_set):
+        _config.return_value = 'none'
+
+        git_reinstall.git_reinstall()
+
+        msg = 'openstack-origin-git is not configured'
+        action_fail.assert_called_with(msg)
+        self.assertFalse(git_install.called)
+        self.assertFalse(action_set.called)
+
+    @patch.object(git_reinstall, 'action_set')
+    @patch.object(git_reinstall, 'action_fail')
+    @patch.object(git_reinstall, 'git_install')
+    @patch('charmhelpers.contrib.openstack.utils.config')
+    def test_git_reinstall_exception(self, _config, git_install,
+                                     action_fail, action_set):
+        _config.return_value = openstack_origin_git
+        e = OSError('something bad happened')
+        git_install.side_effect = e
+        traceback = (
+            "Traceback (most recent call last):\n"
+            "  File \"actions/git_reinstall.py\", line 33, in git_reinstall\n"
+            "    git_install(config(\'openstack-origin-git\'))\n"
+            "  File \"/usr/lib/python2.7/dist-packages/mock.py\", line 964, in __call__\n"  # noqa
+            "    return _mock_self._mock_call(*args, **kwargs)\n"
+            "  File \"/usr/lib/python2.7/dist-packages/mock.py\", line 1019, in _mock_call\n"  # noqa
+            "    raise effect\n"
+            "OSError: something bad happened\n")
+
+        git_reinstall.git_reinstall()
+
+        msg = 'git-reinstall resulted in an unexpected error'
+        action_fail.assert_called_with(msg)
+        action_set.assert_called_with({'traceback': traceback})