From 928cd15cf9697fb3327b2b0d70e92949c78ad198 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Wed, 22 Apr 2015 18:49:48 +0000 Subject: [PATCH 01/31] Sync charm-helpers --- .../contrib/openstack/amulet/deployment.py | 2 +- hooks/charmhelpers/contrib/openstack/utils.py | 12 ++++++++++-- hooks/charmhelpers/contrib/python/packages.py | 7 +++++-- tests/charmhelpers/contrib/amulet/utils.py | 6 +++++- 4 files changed, 21 insertions(+), 6 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py index 4538e961..461a702f 100644 --- a/hooks/charmhelpers/contrib/openstack/amulet/deployment.py +++ b/hooks/charmhelpers/contrib/openstack/amulet/deployment.py @@ -109,7 +109,7 @@ class OpenStackAmuletDeployment(AmuletDeployment): # Must be ordered by OpenStack release (not by Ubuntu release): (self.precise_essex, self.precise_folsom, self.precise_grizzly, self.precise_havana, self.precise_icehouse, - self.trusty_icehouse, self.trusty_juno, self.utopic_juno, + self.trusty_icehouse, self.trusty_juno, self.utopic_juno, self.trusty_kilo, self.vivid_kilo) = range(10) releases = { diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index f90a0289..f4824f99 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -517,6 +517,7 @@ def git_clone_and_install(projects_yaml, core_project): """ global requirements_dir parent_dir = '/mnt/openstack-git' + http_proxy = None if not projects_yaml: return @@ -527,6 +528,7 @@ def git_clone_and_install(projects_yaml, core_project): old_environ = dict(os.environ) if 'http_proxy' in projects.keys(): + http_proxy = projects['http_proxy'] os.environ['http_proxy'] = projects['http_proxy'] if 'https_proxy' in projects.keys(): os.environ['https_proxy'] = projects['https_proxy'] @@ -539,10 +541,12 @@ def git_clone_and_install(projects_yaml, core_project): branch = p['branch'] if p['name'] == 'requirements': repo_dir = _git_clone_and_install_single(repo, branch, parent_dir, + http_proxy, update_requirements=False) requirements_dir = repo_dir else: repo_dir = _git_clone_and_install_single(repo, branch, parent_dir, + http_proxy, update_requirements=True) os.environ = old_environ @@ -574,7 +578,8 @@ def _git_ensure_key_exists(key, keys): error_out('openstack-origin-git key \'{}\' is missing'.format(key)) -def _git_clone_and_install_single(repo, branch, parent_dir, update_requirements): +def _git_clone_and_install_single(repo, branch, parent_dir, http_proxy, + update_requirements): """ Clone and install a single git repository. """ @@ -598,7 +603,10 @@ def _git_clone_and_install_single(repo, branch, parent_dir, update_requirements) _git_update_requirements(repo_dir, requirements_dir) juju_log('Installing git repo from dir: {}'.format(repo_dir)) - pip_install(repo_dir) + if http_proxy: + pip_install(repo_dir, ignore=True, proxy=http_proxy) + else: + pip_install(repo_dir, ignore=True) return repo_dir diff --git a/hooks/charmhelpers/contrib/python/packages.py b/hooks/charmhelpers/contrib/python/packages.py index 8659516b..ceaa5971 100644 --- a/hooks/charmhelpers/contrib/python/packages.py +++ b/hooks/charmhelpers/contrib/python/packages.py @@ -51,17 +51,20 @@ def pip_install_requirements(requirements, **options): pip_execute(command) -def pip_install(package, fatal=False, upgrade=False, **options): +def pip_install(package, fatal=False, upgrade=False, ignore=False, **options): """Install a python package""" command = ["install"] - available_options = ('proxy', 'src', 'log', "index-url", ) + available_options = ('proxy', 'src', 'log', 'index-url', ) for option in parse_options(options, available_options): command.append(option) if upgrade: command.append('--upgrade') + if ignore: + command.append('--ignore-installed') + if isinstance(package, list): command.extend(package) else: diff --git a/tests/charmhelpers/contrib/amulet/utils.py b/tests/charmhelpers/contrib/amulet/utils.py index 58233714..f61c2e8b 100644 --- a/tests/charmhelpers/contrib/amulet/utils.py +++ b/tests/charmhelpers/contrib/amulet/utils.py @@ -89,7 +89,11 @@ class AmuletUtils(object): def _get_config(self, unit, filename): """Get a ConfigParser object for parsing a unit's config file.""" file_contents = unit.file_contents(filename) - config = ConfigParser.ConfigParser() + + # NOTE(beisner): by default, ConfigParser does not handle options + # with no value, such as the flags used in the mysql my.cnf file. + # https://bugs.python.org/issue7005 + config = ConfigParser.ConfigParser(allow_no_value=True) config.readfp(io.StringIO(file_contents)) return config From 59bc871eca4e485a837d5f265fe9f1952cb36d73 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 23 Apr 2015 11:55:32 +0000 Subject: [PATCH 02/31] Sync charm-helpers --- hooks/charmhelpers/contrib/openstack/utils.py | 4 ++-- hooks/charmhelpers/contrib/python/packages.py | 5 +---- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index f4824f99..0460e076 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -604,9 +604,9 @@ def _git_clone_and_install_single(repo, branch, parent_dir, http_proxy, juju_log('Installing git repo from dir: {}'.format(repo_dir)) if http_proxy: - pip_install(repo_dir, ignore=True, proxy=http_proxy) + pip_install(repo_dir, proxy=http_proxy) else: - pip_install(repo_dir, ignore=True) + pip_install(repo_dir) return repo_dir diff --git a/hooks/charmhelpers/contrib/python/packages.py b/hooks/charmhelpers/contrib/python/packages.py index ceaa5971..3f1fb09d 100644 --- a/hooks/charmhelpers/contrib/python/packages.py +++ b/hooks/charmhelpers/contrib/python/packages.py @@ -51,7 +51,7 @@ def pip_install_requirements(requirements, **options): pip_execute(command) -def pip_install(package, fatal=False, upgrade=False, ignore=False, **options): +def pip_install(package, fatal=False, upgrade=False, **options): """Install a python package""" command = ["install"] @@ -62,9 +62,6 @@ def pip_install(package, fatal=False, upgrade=False, ignore=False, **options): if upgrade: command.append('--upgrade') - if ignore: - command.append('--ignore-installed') - if isinstance(package, list): command.extend(package) else: From 39b9ae91eda367ee3ce04df2bb16ee67ceabc7f9 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 23 Apr 2015 13:01:33 +0000 Subject: [PATCH 03/31] Sync charm-helpers --- hooks/charmhelpers/contrib/openstack/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index 0460e076..86bf91f5 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -604,9 +604,9 @@ def _git_clone_and_install_single(repo, branch, parent_dir, http_proxy, juju_log('Installing git repo from dir: {}'.format(repo_dir)) if http_proxy: - pip_install(repo_dir, proxy=http_proxy) + pip_install(repo_dir, upgrade=True, proxy=http_proxy) else: - pip_install(repo_dir) + pip_install(repo_dir, upgrade=True) return repo_dir From ebb775740474edcbbe9768b6abdef0ccf348c02e Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 23 Apr 2015 14:43:27 +0000 Subject: [PATCH 04/31] Sync charm-helpers --- hooks/charmhelpers/contrib/openstack/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index 86bf91f5..0460e076 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -604,9 +604,9 @@ def _git_clone_and_install_single(repo, branch, parent_dir, http_proxy, juju_log('Installing git repo from dir: {}'.format(repo_dir)) if http_proxy: - pip_install(repo_dir, upgrade=True, proxy=http_proxy) + pip_install(repo_dir, proxy=http_proxy) else: - pip_install(repo_dir, upgrade=True) + pip_install(repo_dir) return repo_dir From 6f685dcf7c262c0ad53e45c5e4e35b2f121b192e Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Tue, 28 Apr 2015 19:05:23 +0000 Subject: [PATCH 05/31] Sync charm-helpers --- hooks/charmhelpers/fetch/__init__.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/hooks/charmhelpers/fetch/__init__.py b/hooks/charmhelpers/fetch/__init__.py index 792e629a..57a179b8 100644 --- a/hooks/charmhelpers/fetch/__init__.py +++ b/hooks/charmhelpers/fetch/__init__.py @@ -367,13 +367,15 @@ def install_remote(source, *args, **kwargs): # explaining why it can't handle a given source. handlers = [h for h in plugins() if h.can_handle(source) is True] installed_to = None - for handler in handlers: - try: - installed_to = handler.install(source, *args, **kwargs) - except UnhandledSource: - pass - if not installed_to: - raise UnhandledSource("No handler found for source {}".format(source)) + while not installed_to: + for handler in handlers: + try: + installed_to = handler.install(source, *args, **kwargs) + except UnhandledSource: + pass + log("CB: install loop - installed_to = |%s|".format(installed_to)) + #if not installed_to: + # raise UnhandledSource("No handler found for source {}".format(source)) return installed_to From 69b998ea5e8584c8ba1927b5b5cc455ea1029a57 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Fri, 1 May 2015 14:56:50 +0000 Subject: [PATCH 06/31] Sync charm-helpers --- hooks/charmhelpers/contrib/openstack/utils.py | 15 ++++++++------- hooks/charmhelpers/fetch/__init__.py | 16 +++++++--------- hooks/charmhelpers/fetch/giturl.py | 11 +++++++---- 3 files changed, 22 insertions(+), 20 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index 0460e076..b47ed937 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -497,7 +497,7 @@ def git_install_requested(): requirements_dir = None -def git_clone_and_install(projects_yaml, core_project): +def git_clone_and_install(projects_yaml, core_project, depth=1): """ Clone/install all specified OpenStack repositories. @@ -540,13 +540,13 @@ def git_clone_and_install(projects_yaml, core_project): repo = p['repository'] branch = p['branch'] if p['name'] == 'requirements': - repo_dir = _git_clone_and_install_single(repo, branch, parent_dir, - http_proxy, + repo_dir = _git_clone_and_install_single(repo, branch, depth, + parent_dir, http_proxy, update_requirements=False) requirements_dir = repo_dir else: - repo_dir = _git_clone_and_install_single(repo, branch, parent_dir, - http_proxy, + repo_dir = _git_clone_and_install_single(repo, branch, depth, + parent_dir, http_proxy, update_requirements=True) os.environ = old_environ @@ -578,7 +578,7 @@ def _git_ensure_key_exists(key, keys): error_out('openstack-origin-git key \'{}\' is missing'.format(key)) -def _git_clone_and_install_single(repo, branch, parent_dir, http_proxy, +def _git_clone_and_install_single(repo, branch, depth, parent_dir, http_proxy, update_requirements): """ Clone and install a single git repository. @@ -592,7 +592,8 @@ def _git_clone_and_install_single(repo, branch, parent_dir, http_proxy, if not os.path.exists(dest_dir): juju_log('Cloning git repo: {}, branch: {}'.format(repo, branch)) - repo_dir = install_remote(repo, dest=parent_dir, branch=branch) + repo_dir = install_remote(repo, dest=parent_dir, branch=branch, + depth=depth) else: repo_dir = dest_dir diff --git a/hooks/charmhelpers/fetch/__init__.py b/hooks/charmhelpers/fetch/__init__.py index 57a179b8..792e629a 100644 --- a/hooks/charmhelpers/fetch/__init__.py +++ b/hooks/charmhelpers/fetch/__init__.py @@ -367,15 +367,13 @@ def install_remote(source, *args, **kwargs): # explaining why it can't handle a given source. handlers = [h for h in plugins() if h.can_handle(source) is True] installed_to = None - while not installed_to: - for handler in handlers: - try: - installed_to = handler.install(source, *args, **kwargs) - except UnhandledSource: - pass - log("CB: install loop - installed_to = |%s|".format(installed_to)) - #if not installed_to: - # raise UnhandledSource("No handler found for source {}".format(source)) + for handler in handlers: + try: + installed_to = handler.install(source, *args, **kwargs) + except UnhandledSource: + pass + if not installed_to: + raise UnhandledSource("No handler found for source {}".format(source)) return installed_to diff --git a/hooks/charmhelpers/fetch/giturl.py b/hooks/charmhelpers/fetch/giturl.py index 93aae87b..06b0e767 100644 --- a/hooks/charmhelpers/fetch/giturl.py +++ b/hooks/charmhelpers/fetch/giturl.py @@ -45,14 +45,17 @@ class GitUrlFetchHandler(BaseFetchHandler): else: return True - def clone(self, source, dest, branch): + def clone(self, source, dest, branch, depth): if not self.can_handle(source): raise UnhandledSource("Cannot handle {}".format(source)) - repo = Repo.clone_from(source, dest) + if depth: + repo = Repo.clone_from(source, dest, depth) + else: + repo = Repo.clone_from(source, dest) repo.git.checkout(branch) - def install(self, source, branch="master", dest=None): + def install(self, source, branch="master", dest=None, depth=None): url_parts = self.parse_url(source) branch_name = url_parts.path.strip("/").split("/")[-1] if dest: @@ -63,7 +66,7 @@ class GitUrlFetchHandler(BaseFetchHandler): if not os.path.exists(dest_dir): mkdir(dest_dir, perms=0o755) try: - self.clone(source, dest_dir, branch) + self.clone(source, dest_dir, branch, depth) except GitCommandError as e: raise UnhandledSource(e.message) except OSError as e: From 0797dc06e5b24ff1ae2c4c7cd07378bb22e95061 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Sat, 2 May 2015 18:34:16 +0000 Subject: [PATCH 07/31] Sync charm-helpers --- hooks/charmhelpers/fetch/giturl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hooks/charmhelpers/fetch/giturl.py b/hooks/charmhelpers/fetch/giturl.py index 06b0e767..742cf319 100644 --- a/hooks/charmhelpers/fetch/giturl.py +++ b/hooks/charmhelpers/fetch/giturl.py @@ -50,7 +50,7 @@ class GitUrlFetchHandler(BaseFetchHandler): raise UnhandledSource("Cannot handle {}".format(source)) if depth: - repo = Repo.clone_from(source, dest, depth) + repo = Repo.clone_from(source, dest, depth=depth) else: repo = Repo.clone_from(source, dest) repo.git.checkout(branch) From 007ddd59c8ba61ab2ee9f1f22d3a0bdeea08400a Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Sat, 2 May 2015 20:10:44 +0000 Subject: [PATCH 08/31] Sync charm-helpers --- hooks/charmhelpers/fetch/giturl.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/hooks/charmhelpers/fetch/giturl.py b/hooks/charmhelpers/fetch/giturl.py index 742cf319..4e05c08a 100644 --- a/hooks/charmhelpers/fetch/giturl.py +++ b/hooks/charmhelpers/fetch/giturl.py @@ -50,10 +50,9 @@ class GitUrlFetchHandler(BaseFetchHandler): raise UnhandledSource("Cannot handle {}".format(source)) if depth: - repo = Repo.clone_from(source, dest, depth=depth) + repo = Repo.clone_from(source, dest, branch=branch, depth=depth) else: - repo = Repo.clone_from(source, dest) - repo.git.checkout(branch) + repo = Repo.clone_from(source, dest, branch=branch) def install(self, source, branch="master", dest=None, depth=None): url_parts = self.parse_url(source) From 6a13a3aa9753c82a76d4f7e2f8291c1f6cb04f80 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Tue, 5 May 2015 19:18:13 +0000 Subject: [PATCH 09/31] Sync charm-helpers --- hooks/charmhelpers/contrib/openstack/utils.py | 8 +++++++- hooks/charmhelpers/contrib/python/packages.py | 19 ++++++++++++++++++- hooks/charmhelpers/fetch/giturl.py | 4 ++-- 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index b47ed937..ce8c8213 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -53,9 +53,13 @@ from charmhelpers.contrib.network.ip import ( get_ipv6_addr ) +from charmhelpers.contrib.python.packages import ( + pip_create_virtualenv, + pip_install, +) + from charmhelpers.core.host import lsb_release, mounts, umount from charmhelpers.fetch import apt_install, apt_cache, install_remote -from charmhelpers.contrib.python.packages import pip_install from charmhelpers.contrib.storage.linux.utils import is_block_device, zap_disk from charmhelpers.contrib.storage.linux.loopback import ensure_loopback_device @@ -536,6 +540,8 @@ def git_clone_and_install(projects_yaml, core_project, depth=1): if 'directory' in projects.keys(): parent_dir = projects['directory'] + pip_create_virtualenv(proxy=http_proxy) + for p in projects['repositories']: repo = p['repository'] branch = p['branch'] diff --git a/hooks/charmhelpers/contrib/python/packages.py b/hooks/charmhelpers/contrib/python/packages.py index 3f1fb09d..e62240b4 100644 --- a/hooks/charmhelpers/contrib/python/packages.py +++ b/hooks/charmhelpers/contrib/python/packages.py @@ -17,8 +17,11 @@ # You should have received a copy of the GNU Lesser General Public License # along with charm-helpers. If not, see . +import os +import subprocess + from charmhelpers.fetch import apt_install, apt_update -from charmhelpers.core.hookenv import log +from charmhelpers.core.hookenv import charm_dir, log try: from pip import main as pip_execute @@ -94,3 +97,17 @@ def pip_list(): """Returns the list of current python installed packages """ return pip_execute(["list"]) + + +def pip_create_virtualenv(parent_dir=charm_dir(), proxy=None): + """Create and activate an isolated Python environment (virtualenv).""" + if proxy: + pip_install('virtualenv', proxy=proxy) + else: + pip_install('virtualenv') + + venv_dir = os.path.join(parent_dir, 'venv') + subprocess.check_call(['virtualenv', '--no-site-packages', venv_dir]) + + venv_activate = os.path.join(venv_dir, 'bin/activate') + subprocess.check_call(['.', venv_activate], shell=True) diff --git a/hooks/charmhelpers/fetch/giturl.py b/hooks/charmhelpers/fetch/giturl.py index 4e05c08a..7d842d49 100644 --- a/hooks/charmhelpers/fetch/giturl.py +++ b/hooks/charmhelpers/fetch/giturl.py @@ -50,9 +50,9 @@ class GitUrlFetchHandler(BaseFetchHandler): raise UnhandledSource("Cannot handle {}".format(source)) if depth: - repo = Repo.clone_from(source, dest, branch=branch, depth=depth) + Repo.clone_from(source, dest, branch=branch, depth=depth) else: - repo = Repo.clone_from(source, dest, branch=branch) + Repo.clone_from(source, dest, branch=branch) def install(self, source, branch="master", dest=None, depth=None): url_parts = self.parse_url(source) From 32d0181a06bf71a890bc94d5922718e44f121181 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Tue, 5 May 2015 19:57:28 +0000 Subject: [PATCH 10/31] Point upstart scripts at venv binaries --- hooks/neutron_ovs_utils.py | 4 ++++ templates/git/upstart/neutron-ovs-cleanup.upstart | 2 +- .../git/upstart/neutron-plugin-openvswitch-agent.upstart | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/hooks/neutron_ovs_utils.py b/hooks/neutron_ovs_utils.py index 1b732a24..4e3d5b94 100644 --- a/hooks/neutron_ovs_utils.py +++ b/hooks/neutron_ovs_utils.py @@ -21,6 +21,7 @@ from charmhelpers.contrib.network.ovs import ( full_restart, ) from charmhelpers.core.hookenv import ( + charm_dir, config, ) from charmhelpers.contrib.openstack.neutron import ( @@ -265,10 +266,12 @@ def git_post_install(projects_yaml): render('git/neutron_sudoers', '/etc/sudoers.d/neutron_sudoers', {}, perms=0o440) + bin_dir = os.path.join(charm_dir(), 'venv/bin') neutron_ovs_agent_context = { 'service_description': 'Neutron OpenvSwitch Plugin Agent', 'charm_name': 'neutron-openvswitch', 'process_name': 'neutron-openvswitch-agent', + 'executable_name': os.path.join(bin_dir, 'neutron-openvswitch-agent'), 'cleanup_process_name': 'neutron-ovs-cleanup', 'plugin_config': '/etc/neutron/plugins/ml2/ml2_conf.ini', 'log_file': '/var/log/neutron/openvswitch-agent.log', @@ -278,6 +281,7 @@ def git_post_install(projects_yaml): 'service_description': 'Neutron OpenvSwitch Cleanup', 'charm_name': 'neutron-openvswitch', 'process_name': 'neutron-ovs-cleanup', + 'executable_name': os.path.join(bin_dir, 'neutron-ovs-cleanup'), 'log_file': '/var/log/neutron/ovs-cleanup.log', } diff --git a/templates/git/upstart/neutron-ovs-cleanup.upstart b/templates/git/upstart/neutron-ovs-cleanup.upstart index cbbd0db2..e7e809d4 100644 --- a/templates/git/upstart/neutron-ovs-cleanup.upstart +++ b/templates/git/upstart/neutron-ovs-cleanup.upstart @@ -11,7 +11,7 @@ end script pre-start script [ ! -x /usr/bin/{{ process_name }} ] && exit 0 - start-stop-daemon --start --chuid neutron --exec /usr/local/bin/{{ process_name }} -- \ + start-stop-daemon --start --chuid neutron --exec {{ executable_name }} -- \ --log-file /var/log/neutron/{{ log_file }} \ --config-file /etc/neutron/neutron.conf --verbose end script diff --git a/templates/git/upstart/neutron-plugin-openvswitch-agent.upstart b/templates/git/upstart/neutron-plugin-openvswitch-agent.upstart index 17a82edb..40f002c2 100644 --- a/templates/git/upstart/neutron-plugin-openvswitch-agent.upstart +++ b/templates/git/upstart/neutron-plugin-openvswitch-agent.upstart @@ -13,6 +13,6 @@ pre-start script chown neutron:root /var/run/neutron end script -exec start-stop-daemon --start --chuid neutron --exec /usr/local/bin/{{ process_name }} -- \ +exec start-stop-daemon --start --chuid neutron --exec {{ executable_name }} -- \ --config-file=/etc/neutron/neutron.conf --config-file={{ plugin_config }} \ --log-file={{ log_file }} From 9069c17347d62b2273afa4a4b556d28f607b8113 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Tue, 5 May 2015 19:57:51 +0000 Subject: [PATCH 11/31] Update unit tests --- unit_tests/test_neutron_ovs_utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/unit_tests/test_neutron_ovs_utils.py b/unit_tests/test_neutron_ovs_utils.py index c72d0ab3..75f7c76e 100644 --- a/unit_tests/test_neutron_ovs_utils.py +++ b/unit_tests/test_neutron_ovs_utils.py @@ -282,6 +282,7 @@ class TestNeutronOVSUtils(CharmTestCase): 'service_description': 'Neutron OpenvSwitch Plugin Agent', 'charm_name': 'neutron-openvswitch', 'process_name': 'neutron-openvswitch-agent', + 'executable_name': 'joined-string', 'cleanup_process_name': 'neutron-ovs-cleanup', 'plugin_config': '/etc/neutron/plugins/ml2/ml2_conf.ini', 'log_file': '/var/log/neutron/openvswitch-agent.log', @@ -290,6 +291,7 @@ class TestNeutronOVSUtils(CharmTestCase): 'service_description': 'Neutron OpenvSwitch Cleanup', 'charm_name': 'neutron-openvswitch', 'process_name': 'neutron-ovs-cleanup', + 'executable_name': 'joined-string', 'log_file': '/var/log/neutron/ovs-cleanup.log', } expected = [ From 726784567a6f0d8e67071bfa79094714a527eec0 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Wed, 6 May 2015 16:00:24 +0000 Subject: [PATCH 12/31] Sync charm-helpers --- hooks/charmhelpers/contrib/openstack/utils.py | 6 ++-- hooks/charmhelpers/contrib/python/packages.py | 29 +++++++++---------- 2 files changed, 17 insertions(+), 18 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index ce8c8213..5909b0f5 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -540,7 +540,7 @@ def git_clone_and_install(projects_yaml, core_project, depth=1): if 'directory' in projects.keys(): parent_dir = projects['directory'] - pip_create_virtualenv(proxy=http_proxy) + pip_create_virtualenv() for p in projects['repositories']: repo = p['repository'] @@ -611,9 +611,9 @@ def _git_clone_and_install_single(repo, branch, depth, parent_dir, http_proxy, juju_log('Installing git repo from dir: {}'.format(repo_dir)) if http_proxy: - pip_install(repo_dir, proxy=http_proxy) + pip_install(repo_dir, proxy=http_proxy, venv=True) else: - pip_install(repo_dir) + pip_install(repo_dir, venv=True) return repo_dir diff --git a/hooks/charmhelpers/contrib/python/packages.py b/hooks/charmhelpers/contrib/python/packages.py index e62240b4..740eaa51 100644 --- a/hooks/charmhelpers/contrib/python/packages.py +++ b/hooks/charmhelpers/contrib/python/packages.py @@ -54,9 +54,13 @@ def pip_install_requirements(requirements, **options): pip_execute(command) -def pip_install(package, fatal=False, upgrade=False, **options): +def pip_install(package, fatal=False, upgrade=False, venv=False, **options): """Install a python package""" - command = ["install"] + if venv: + venv_python = os.path.join(charm_dir(), 'venv/bin/pip') + command = [venv_python, "install"] + else: + command = ["install"] available_options = ('proxy', 'src', 'log', 'index-url', ) for option in parse_options(options, available_options): @@ -72,7 +76,10 @@ def pip_install(package, fatal=False, upgrade=False, **options): log("Installing {} package with options: {}".format(package, command)) - pip_execute(command) + if venv: + subprocess.check_call(command) + else: + pip_execute(command) def pip_uninstall(package, **options): @@ -99,15 +106,7 @@ def pip_list(): return pip_execute(["list"]) -def pip_create_virtualenv(parent_dir=charm_dir(), proxy=None): - """Create and activate an isolated Python environment (virtualenv).""" - if proxy: - pip_install('virtualenv', proxy=proxy) - else: - pip_install('virtualenv') - - venv_dir = os.path.join(parent_dir, 'venv') - subprocess.check_call(['virtualenv', '--no-site-packages', venv_dir]) - - venv_activate = os.path.join(venv_dir, 'bin/activate') - subprocess.check_call(['.', venv_activate], shell=True) +def pip_create_virtualenv(): + """Create an isolated Python environment.""" + apt_install('python-virtualenv') + subprocess.check_call(['virtualenv', os.path.join(charm_dir(), 'venv')]) From 6f9edbd3ef57e9b38ce74075fca067407af75aa6 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 7 May 2015 12:44:47 +0000 Subject: [PATCH 13/31] Sync charm-helpers --- hooks/charmhelpers/contrib/openstack/utils.py | 17 +++++++++++++++- hooks/charmhelpers/contrib/python/packages.py | 20 ++++++++++++++++--- 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index 5909b0f5..29ac7b78 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -540,7 +540,7 @@ def git_clone_and_install(projects_yaml, core_project, depth=1): if 'directory' in projects.keys(): parent_dir = projects['directory'] - pip_create_virtualenv() + pip_create_virtualenv(os.path.join(parent_dir, 'venv')) for p in projects['repositories']: repo = p['repository'] @@ -655,3 +655,18 @@ def git_src_dir(projects_yaml, project): return os.path.join(parent_dir, os.path.basename(p['repository'])) return None + + +def git_yaml_value(projects_yaml, key): + """ + Return the value in projects_yaml for the specified key. + """ + if not projects_yaml: + return None + + projects = yaml.load(projects_yaml) + + if key in projects.keys(): + return projects[key] + + return None diff --git a/hooks/charmhelpers/contrib/python/packages.py b/hooks/charmhelpers/contrib/python/packages.py index 740eaa51..12838d24 100644 --- a/hooks/charmhelpers/contrib/python/packages.py +++ b/hooks/charmhelpers/contrib/python/packages.py @@ -30,6 +30,8 @@ except ImportError: apt_install('python-pip') from pip import main as pip_execute +pip_venv_path = None + __author__ = "Jorge Niedbalski " @@ -57,7 +59,7 @@ def pip_install_requirements(requirements, **options): def pip_install(package, fatal=False, upgrade=False, venv=False, **options): """Install a python package""" if venv: - venv_python = os.path.join(charm_dir(), 'venv/bin/pip') + venv_python = os.path.join(pip_venv_path, 'bin/pip') command = [venv_python, "install"] else: command = ["install"] @@ -106,7 +108,19 @@ def pip_list(): return pip_execute(["list"]) -def pip_create_virtualenv(): +def pip_create_virtualenv(path=None): """Create an isolated Python environment.""" + global pip_venv_path + apt_install('python-virtualenv') - subprocess.check_call(['virtualenv', os.path.join(charm_dir(), 'venv')]) + + if path: + pip_venv_path = path + else: + pip_venv_path = os.path.join(charm_dir(), 'venv') + + subprocess.check_call(['virtualenv', pip_venv_path]) + + +def pip_get_virtualenv_path(): + return pip_venv_path From aaa88f5d96640ab0927aa6a462a653fefb1c1d0b Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 7 May 2015 12:50:53 +0000 Subject: [PATCH 14/31] Use function to get pip venv path --- hooks/neutron_ovs_utils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/hooks/neutron_ovs_utils.py b/hooks/neutron_ovs_utils.py index 4e3d5b94..211a9e6d 100644 --- a/hooks/neutron_ovs_utils.py +++ b/hooks/neutron_ovs_utils.py @@ -31,6 +31,9 @@ from charmhelpers.contrib.openstack.context import ( ExternalPortContext, DataPortContext, ) +from charmhelpers.contrib.python.packages import ( + pip_get_virtualenv_path, +) from charmhelpers.core.host import ( adduser, add_group, @@ -266,7 +269,7 @@ def git_post_install(projects_yaml): render('git/neutron_sudoers', '/etc/sudoers.d/neutron_sudoers', {}, perms=0o440) - bin_dir = os.path.join(charm_dir(), 'venv/bin') + bin_dir = os.path.join(pip_get_virtualenv_path(), 'bin') neutron_ovs_agent_context = { 'service_description': 'Neutron OpenvSwitch Plugin Agent', 'charm_name': 'neutron-openvswitch', From acad7793ef5f7b883a185c66396bde84b6d7810e Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 7 May 2015 14:40:34 +0000 Subject: [PATCH 15/31] Sync charm-helpers --- hooks/charmhelpers/contrib/openstack/utils.py | 23 +++++++++++++++++-- hooks/charmhelpers/contrib/python/packages.py | 18 ++++----------- 2 files changed, 26 insertions(+), 15 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index 29ac7b78..c5e08ec2 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -611,9 +611,11 @@ def _git_clone_and_install_single(repo, branch, depth, parent_dir, http_proxy, juju_log('Installing git repo from dir: {}'.format(repo_dir)) if http_proxy: - pip_install(repo_dir, proxy=http_proxy, venv=True) + pip_install(repo_dir, proxy=http_proxy, + venv=os.path.join(parent_dir, 'venv')) else: - pip_install(repo_dir, venv=True) + pip_install(repo_dir, + venv=os.path.join(parent_dir, 'venv')) return repo_dir @@ -636,6 +638,23 @@ def _git_update_requirements(package_dir, reqs_dir): os.chdir(orig_dir) +def git_pip_venv_dir(projects_yaml): + """ + Return the pip virtualenv path. + """ + parent_dir = '/mnt/openstack-git' + + if not projects_yaml: + return + + projects = yaml.load(projects_yaml) + + if 'directory' in projects.keys(): + parent_dir = projects['directory'] + + return os.path.join(parent_dir, 'venv') + + def git_src_dir(projects_yaml, project): """ Return the directory where the specified project's source is located. diff --git a/hooks/charmhelpers/contrib/python/packages.py b/hooks/charmhelpers/contrib/python/packages.py index 12838d24..88c8887d 100644 --- a/hooks/charmhelpers/contrib/python/packages.py +++ b/hooks/charmhelpers/contrib/python/packages.py @@ -30,8 +30,6 @@ except ImportError: apt_install('python-pip') from pip import main as pip_execute -pip_venv_path = None - __author__ = "Jorge Niedbalski " @@ -56,10 +54,10 @@ def pip_install_requirements(requirements, **options): pip_execute(command) -def pip_install(package, fatal=False, upgrade=False, venv=False, **options): +def pip_install(package, fatal=False, upgrade=False, venv=None, **options): """Install a python package""" if venv: - venv_python = os.path.join(pip_venv_path, 'bin/pip') + venv_python = os.path.join(venv, 'bin/pip') command = [venv_python, "install"] else: command = ["install"] @@ -110,17 +108,11 @@ def pip_list(): def pip_create_virtualenv(path=None): """Create an isolated Python environment.""" - global pip_venv_path - apt_install('python-virtualenv') if path: - pip_venv_path = path + venv_path = path else: - pip_venv_path = os.path.join(charm_dir(), 'venv') + venv_path = os.path.join(charm_dir(), 'venv') - subprocess.check_call(['virtualenv', pip_venv_path]) - - -def pip_get_virtualenv_path(): - return pip_venv_path + subprocess.check_call(['virtualenv', venv_path]) From 85e0c8786cb05f23ff6878f97e75f3047dff00c7 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 7 May 2015 14:54:20 +0000 Subject: [PATCH 16/31] Use git_pip_venv_dir to get venv path --- hooks/neutron_ovs_utils.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/hooks/neutron_ovs_utils.py b/hooks/neutron_ovs_utils.py index 211a9e6d..f0ed65ca 100644 --- a/hooks/neutron_ovs_utils.py +++ b/hooks/neutron_ovs_utils.py @@ -9,6 +9,7 @@ from charmhelpers.contrib.openstack.utils import ( git_install_requested, git_clone_and_install, git_src_dir, + git_pip_venv_dir, ) from collections import OrderedDict from charmhelpers.contrib.openstack.utils import ( @@ -31,9 +32,6 @@ from charmhelpers.contrib.openstack.context import ( ExternalPortContext, DataPortContext, ) -from charmhelpers.contrib.python.packages import ( - pip_get_virtualenv_path, -) from charmhelpers.core.host import ( adduser, add_group, @@ -269,7 +267,7 @@ def git_post_install(projects_yaml): render('git/neutron_sudoers', '/etc/sudoers.d/neutron_sudoers', {}, perms=0o440) - bin_dir = os.path.join(pip_get_virtualenv_path(), 'bin') + bin_dir = os.path.join(git_pip_venv_dir(), 'bin') neutron_ovs_agent_context = { 'service_description': 'Neutron OpenvSwitch Plugin Agent', 'charm_name': 'neutron-openvswitch', From 379dab6fa40bf36b00e995d7b0c0acf8326e999c Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 7 May 2015 15:11:29 +0000 Subject: [PATCH 17/31] Add missing params --- hooks/neutron_ovs_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hooks/neutron_ovs_utils.py b/hooks/neutron_ovs_utils.py index f0ed65ca..bd312983 100644 --- a/hooks/neutron_ovs_utils.py +++ b/hooks/neutron_ovs_utils.py @@ -267,7 +267,7 @@ def git_post_install(projects_yaml): render('git/neutron_sudoers', '/etc/sudoers.d/neutron_sudoers', {}, perms=0o440) - bin_dir = os.path.join(git_pip_venv_dir(), 'bin') + bin_dir = os.path.join(git_pip_venv_dir(projects_yaml), 'bin') neutron_ovs_agent_context = { 'service_description': 'Neutron OpenvSwitch Plugin Agent', 'charm_name': 'neutron-openvswitch', From 54ecb3731f234d8757148c6ce4c989ed7a5db945 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 7 May 2015 16:35:54 +0000 Subject: [PATCH 18/31] Add rootwrap symlink --- hooks/neutron_ovs_utils.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/hooks/neutron_ovs_utils.py b/hooks/neutron_ovs_utils.py index bd312983..6fc19508 100644 --- a/hooks/neutron_ovs_utils.py +++ b/hooks/neutron_ovs_utils.py @@ -264,6 +264,17 @@ def git_post_install(projects_yaml): shutil.rmtree(c['dest']) shutil.copytree(c['src'], c['dest']) + symlinks = [ + {'src': os.path.join(git_pip_venv_dir(projects_yaml), + 'bin/neutron-rootwrap'), + 'link': '/usr/local/bin/neutron-rootwrap'}, + ] + + for s in symlinks: + if os.path.lexists(s['link']): + os.remove(s['link']) + os.symlink(s['src'], s['link']) + render('git/neutron_sudoers', '/etc/sudoers.d/neutron_sudoers', {}, perms=0o440) From 6d2d6dcac173531bcc5a382c036c6fa4e8a086a9 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 7 May 2015 17:14:18 +0000 Subject: [PATCH 19/31] Unit test updates --- hooks/neutron_ovs_utils.py | 1 - unit_tests/test_neutron_ovs_utils.py | 9 +++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/hooks/neutron_ovs_utils.py b/hooks/neutron_ovs_utils.py index 6fc19508..a97eafd4 100644 --- a/hooks/neutron_ovs_utils.py +++ b/hooks/neutron_ovs_utils.py @@ -22,7 +22,6 @@ from charmhelpers.contrib.network.ovs import ( full_restart, ) from charmhelpers.core.hookenv import ( - charm_dir, config, ) from charmhelpers.contrib.openstack.neutron import ( diff --git a/unit_tests/test_neutron_ovs_utils.py b/unit_tests/test_neutron_ovs_utils.py index 75f7c76e..20e6208c 100644 --- a/unit_tests/test_neutron_ovs_utils.py +++ b/unit_tests/test_neutron_ovs_utils.py @@ -265,10 +265,11 @@ class TestNeutronOVSUtils(CharmTestCase): @patch.object(nutils, 'render') @patch('os.path.join') @patch('os.path.exists') + @patch('os.symlink') @patch('shutil.copytree') @patch('shutil.rmtree') - def test_git_post_install(self, rmtree, copytree, exists, join, render, - service_restart, git_src_dir): + def test_git_post_install(self, rmtree, copytree, symlink, exists, join, + render, service_restart, git_src_dir): projects_yaml = openstack_origin_git join.return_value = 'joined-string' nutils.git_post_install(projects_yaml) @@ -278,6 +279,10 @@ class TestNeutronOVSUtils(CharmTestCase): call('joined-string', '/etc/neutron/rootwrap.d'), ] copytree.assert_has_calls(expected) + expected = [ + call('joined-string', '/usr/local/bin/neutron-rootwrap'), + ] + symlink.assert_has_calls(expected, any_order=True) neutron_ovs_agent_context = { 'service_description': 'Neutron OpenvSwitch Plugin Agent', 'charm_name': 'neutron-openvswitch', From 071093c697c83a8606cce4fd63b75a2e34f336d7 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 7 May 2015 17:36:42 +0000 Subject: [PATCH 20/31] Add base mysql support --- hooks/neutron_ovs_utils.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/hooks/neutron_ovs_utils.py b/hooks/neutron_ovs_utils.py index a97eafd4..ea45a5cc 100644 --- a/hooks/neutron_ovs_utils.py +++ b/hooks/neutron_ovs_utils.py @@ -10,6 +10,7 @@ from charmhelpers.contrib.openstack.utils import ( git_clone_and_install, git_src_dir, git_pip_venv_dir, + git_yaml_value, ) from collections import OrderedDict from charmhelpers.contrib.openstack.utils import ( @@ -31,6 +32,9 @@ from charmhelpers.contrib.openstack.context import ( ExternalPortContext, DataPortContext, ) +from charmhelpers.contrib.python.packages import ( + pip_install, +) from charmhelpers.core.host import ( adduser, add_group, @@ -44,6 +48,7 @@ from charmhelpers.core.host import ( from charmhelpers.core.templating import render BASE_GIT_PACKAGES = [ + 'libmysqlclient-dev', 'libxml2-dev', 'libxslt1-dev', 'openvswitch-switch', @@ -248,6 +253,14 @@ def git_pre_install(): def git_post_install(projects_yaml): """Perform post-install setup.""" + http_proxy = git_yaml_value(projects_yaml, 'http_proxy') + if http_proxy: + pip_install('mysql-python', proxy=http_proxy, + venv=git_pip_venv_dir(projects_yaml)) + else: + pip_install('mysql-python', + venv=git_pip_venv_dir(projects_yaml)) + src_etc = os.path.join(git_src_dir(projects_yaml, 'neutron'), 'etc') configs = [ {'src': src_etc, From f09e9909a11156249133ffabeae1ecedf65a3355 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 7 May 2015 17:51:05 +0000 Subject: [PATCH 21/31] Unit test updates --- unit_tests/test_neutron_ovs_utils.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/unit_tests/test_neutron_ovs_utils.py b/unit_tests/test_neutron_ovs_utils.py index 20e6208c..f081c001 100644 --- a/unit_tests/test_neutron_ovs_utils.py +++ b/unit_tests/test_neutron_ovs_utils.py @@ -263,15 +263,19 @@ class TestNeutronOVSUtils(CharmTestCase): @patch.object(nutils, 'git_src_dir') @patch.object(nutils, 'service_restart') @patch.object(nutils, 'render') + @patch.object(nutils, 'git_pip_venv_dir') @patch('os.path.join') @patch('os.path.exists') @patch('os.symlink') @patch('shutil.copytree') @patch('shutil.rmtree') - def test_git_post_install(self, rmtree, copytree, symlink, exists, join, - render, service_restart, git_src_dir): + @patch('subprocess.check_call') + def test_git_post_install(self, check_call, rmtree, copytree, symlink, + exists, join, venv, render, service_restart, + git_src_dir): projects_yaml = openstack_origin_git join.return_value = 'joined-string' + venv.return_value = '/mnt/openstack-git/venv' nutils.git_post_install(projects_yaml) expected = [ call('joined-string', '/etc/neutron'), From 304f28151c272e444e5bb6fa401612009611a59e Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 7 May 2015 18:18:21 +0000 Subject: [PATCH 22/31] Sync charm-helpers --- hooks/charmhelpers/contrib/python/packages.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hooks/charmhelpers/contrib/python/packages.py b/hooks/charmhelpers/contrib/python/packages.py index 88c8887d..07b0c1d7 100644 --- a/hooks/charmhelpers/contrib/python/packages.py +++ b/hooks/charmhelpers/contrib/python/packages.py @@ -115,4 +115,5 @@ def pip_create_virtualenv(path=None): else: venv_path = os.path.join(charm_dir(), 'venv') - subprocess.check_call(['virtualenv', venv_path]) + if not os.path.exists(venv_path): + subprocess.check_call(['virtualenv', venv_path]) From 9597ce88394cccf6b2115f5d909d97c2bf4f80e4 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 7 May 2015 18:43:46 +0000 Subject: [PATCH 23/31] Drop base mysql support - belongs in neutron-api --- hooks/neutron_ovs_utils.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/hooks/neutron_ovs_utils.py b/hooks/neutron_ovs_utils.py index ea45a5cc..a97eafd4 100644 --- a/hooks/neutron_ovs_utils.py +++ b/hooks/neutron_ovs_utils.py @@ -10,7 +10,6 @@ from charmhelpers.contrib.openstack.utils import ( git_clone_and_install, git_src_dir, git_pip_venv_dir, - git_yaml_value, ) from collections import OrderedDict from charmhelpers.contrib.openstack.utils import ( @@ -32,9 +31,6 @@ from charmhelpers.contrib.openstack.context import ( ExternalPortContext, DataPortContext, ) -from charmhelpers.contrib.python.packages import ( - pip_install, -) from charmhelpers.core.host import ( adduser, add_group, @@ -48,7 +44,6 @@ from charmhelpers.core.host import ( from charmhelpers.core.templating import render BASE_GIT_PACKAGES = [ - 'libmysqlclient-dev', 'libxml2-dev', 'libxslt1-dev', 'openvswitch-switch', @@ -253,14 +248,6 @@ def git_pre_install(): def git_post_install(projects_yaml): """Perform post-install setup.""" - http_proxy = git_yaml_value(projects_yaml, 'http_proxy') - if http_proxy: - pip_install('mysql-python', proxy=http_proxy, - venv=git_pip_venv_dir(projects_yaml)) - else: - pip_install('mysql-python', - venv=git_pip_venv_dir(projects_yaml)) - src_etc = os.path.join(git_src_dir(projects_yaml, 'neutron'), 'etc') configs = [ {'src': src_etc, From 68bdede60f0d682eefe4714404ad7987ebd29841 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Thu, 7 May 2015 18:48:58 +0000 Subject: [PATCH 24/31] Drop base mysql unit test updates --- unit_tests/test_neutron_ovs_utils.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/unit_tests/test_neutron_ovs_utils.py b/unit_tests/test_neutron_ovs_utils.py index f081c001..20e6208c 100644 --- a/unit_tests/test_neutron_ovs_utils.py +++ b/unit_tests/test_neutron_ovs_utils.py @@ -263,19 +263,15 @@ class TestNeutronOVSUtils(CharmTestCase): @patch.object(nutils, 'git_src_dir') @patch.object(nutils, 'service_restart') @patch.object(nutils, 'render') - @patch.object(nutils, 'git_pip_venv_dir') @patch('os.path.join') @patch('os.path.exists') @patch('os.symlink') @patch('shutil.copytree') @patch('shutil.rmtree') - @patch('subprocess.check_call') - def test_git_post_install(self, check_call, rmtree, copytree, symlink, - exists, join, venv, render, service_restart, - git_src_dir): + def test_git_post_install(self, rmtree, copytree, symlink, exists, join, + render, service_restart, git_src_dir): projects_yaml = openstack_origin_git join.return_value = 'joined-string' - venv.return_value = '/mnt/openstack-git/venv' nutils.git_post_install(projects_yaml) expected = [ call('joined-string', '/etc/neutron'), From 02aa648fac734f0f5103be93e96c280e0bc479b1 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Mon, 11 May 2015 12:33:15 +0000 Subject: [PATCH 25/31] Sync charm-helpers --- hooks/charmhelpers/contrib/openstack/utils.py | 30 +++++++++---------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index c5e08ec2..774eaba1 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -501,6 +501,16 @@ def git_install_requested(): requirements_dir = None +def _git_yaml_load(projects_yaml): + """ + Load the specified yaml into a dictionary. + """ + if not projects_yaml: + return None + + projects = yaml.load(projects_yaml) + + def git_clone_and_install(projects_yaml, core_project, depth=1): """ Clone/install all specified OpenStack repositories. @@ -523,10 +533,7 @@ def git_clone_and_install(projects_yaml, core_project, depth=1): parent_dir = '/mnt/openstack-git' http_proxy = None - if not projects_yaml: - return - - projects = yaml.load(projects_yaml) + projects = _git_yaml_load(projects_yaml) _git_validate_projects_yaml(projects, core_project) old_environ = dict(os.environ) @@ -644,10 +651,7 @@ def git_pip_venv_dir(projects_yaml): """ parent_dir = '/mnt/openstack-git' - if not projects_yaml: - return - - projects = yaml.load(projects_yaml) + projects = _git_yaml_load(projects_yaml) if 'directory' in projects.keys(): parent_dir = projects['directory'] @@ -661,10 +665,7 @@ def git_src_dir(projects_yaml, project): """ parent_dir = '/mnt/openstack-git' - if not projects_yaml: - return - - projects = yaml.load(projects_yaml) + projects = _git_yaml_load(projects_yaml) if 'directory' in projects.keys(): parent_dir = projects['directory'] @@ -680,10 +681,7 @@ def git_yaml_value(projects_yaml, key): """ Return the value in projects_yaml for the specified key. """ - if not projects_yaml: - return None - - projects = yaml.load(projects_yaml) + projects = _git_yaml_load(projects_yaml) if key in projects.keys(): return projects[key] From 87adf3fbcf8ee4878806e847574f53dcd8ebf382 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Mon, 11 May 2015 12:37:45 +0000 Subject: [PATCH 26/31] Add comment to fix bin symlinks --- hooks/neutron_ovs_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/hooks/neutron_ovs_utils.py b/hooks/neutron_ovs_utils.py index a97eafd4..ac7cea49 100644 --- a/hooks/neutron_ovs_utils.py +++ b/hooks/neutron_ovs_utils.py @@ -263,6 +263,7 @@ def git_post_install(projects_yaml): shutil.rmtree(c['dest']) shutil.copytree(c['src'], c['dest']) + # NOTE(coreycb): Need to find better solution than bin symlinks. symlinks = [ {'src': os.path.join(git_pip_venv_dir(projects_yaml), 'bin/neutron-rootwrap'), From 7eec93c8f765b9af1cb1c9969ae55914e2607359 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Mon, 11 May 2015 12:46:28 +0000 Subject: [PATCH 27/31] Sync charm-helpers --- hooks/charmhelpers/contrib/openstack/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index 774eaba1..8bd2356e 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -508,7 +508,7 @@ def _git_yaml_load(projects_yaml): if not projects_yaml: return None - projects = yaml.load(projects_yaml) + return yaml.load(projects_yaml) def git_clone_and_install(projects_yaml, core_project, depth=1): From 74e0d068868ac018cb2269d928088092c0522716 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Tue, 12 May 2015 14:25:12 +0000 Subject: [PATCH 28/31] Sync charm-helpers --- hooks/charmhelpers/contrib/openstack/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hooks/charmhelpers/contrib/openstack/utils.py b/hooks/charmhelpers/contrib/openstack/utils.py index 8bd2356e..d795a358 100644 --- a/hooks/charmhelpers/contrib/openstack/utils.py +++ b/hooks/charmhelpers/contrib/openstack/utils.py @@ -524,8 +524,8 @@ def git_clone_and_install(projects_yaml, core_project, depth=1): repository: 'git://git.openstack.org/openstack/requirements.git', branch: 'stable/icehouse'} directory: /mnt/openstack-git - http_proxy: http://squid.internal:3128 - https_proxy: https://squid.internal:3128 + http_proxy: squid-proxy-url + https_proxy: squid-proxy-url The directory, http_proxy, and https_proxy keys are optional. """ From 94e2b9622f523842fb2ea2397927cbaabdfae789 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Tue, 12 May 2015 14:49:28 +0000 Subject: [PATCH 29/31] Clone from github in deploy from source amulet tests --- tests/basic_deployment.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/basic_deployment.py b/tests/basic_deployment.py index d71d2620..d8569424 100644 --- a/tests/basic_deployment.py +++ b/tests/basic_deployment.py @@ -72,10 +72,10 @@ class NeutronOVSBasicDeployment(OpenStackAmuletDeployment): openstack_origin_git = { 'repositories': [ {'name': 'requirements', - 'repository': 'git://git.openstack.org/openstack/requirements', + 'repository': 'git://github.com/openstack/requirements', 'branch': branch}, {'name': 'neutron', - 'repository': 'git://git.openstack.org/openstack/neutron', + 'repository': 'git://github.com/openstack/neutron', 'branch': branch}, ], 'directory': '/mnt/openstack-git', From f38e7befaeba848d945301f99824f351fbecffc4 Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Tue, 12 May 2015 19:51:00 +0000 Subject: [PATCH 30/31] Add libyaml-dev as base git package --- hooks/neutron_ovs_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/hooks/neutron_ovs_utils.py b/hooks/neutron_ovs_utils.py index ac7cea49..ea49ee28 100644 --- a/hooks/neutron_ovs_utils.py +++ b/hooks/neutron_ovs_utils.py @@ -46,6 +46,7 @@ from charmhelpers.core.templating import render BASE_GIT_PACKAGES = [ 'libxml2-dev', 'libxslt1-dev', + 'libyaml-dev', 'openvswitch-switch', 'python-dev', 'python-pip', From e102ee9b66725f7f1c3ed6ef98d65307c622d5dc Mon Sep 17 00:00:00 2001 From: Corey Bryant Date: Wed, 27 May 2015 13:02:55 +0000 Subject: [PATCH 31/31] Sync charm-helpers --- hooks/charmhelpers/fetch/giturl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hooks/charmhelpers/fetch/giturl.py b/hooks/charmhelpers/fetch/giturl.py index 7d842d49..ddc25b7e 100644 --- a/hooks/charmhelpers/fetch/giturl.py +++ b/hooks/charmhelpers/fetch/giturl.py @@ -45,7 +45,7 @@ class GitUrlFetchHandler(BaseFetchHandler): else: return True - def clone(self, source, dest, branch, depth): + def clone(self, source, dest, branch, depth=None): if not self.can_handle(source): raise UnhandledSource("Cannot handle {}".format(source))