x/anvil
Code Issues Proposed changes

Use the pip code to get an initial pip installed list and same with rpms via yum code

Browse Source
This commit is contained in:
harlowja 2012-08-25 18:33:06 -07:00
parent de9bb1ae8b
commit 19a8a1d8b5
11 changed files with 168 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
anvil/components/__init__.py
View File

@ -43,11 +43,12 @@ from anvil import exceptions as excp
from anvil import importer
from anvil import log as logging
from anvil import packager
from anvil import pip
from anvil import shell as sh
from anvil import trace as tr
from anvil import utils
from anvil.packaging import pip
LOG = logging.getLogger(__name__)
####
@ -63,6 +64,14 @@ class ProgramStatus(object):
self.status = status
self.details = details
####
#### Utils...
####
def make_packager(package, distro, default_class):
cls = packager.get_packager_class(package, default_class)
return cls(distro)
####
#### INSTALL CLASSES
@ -72,7 +81,6 @@ class PkgInstallComponent(component.Component):
def __init__(self, *args, **kargs):
component.Component.__init__(self, *args, **kargs)
self.tracewriter = tr.TraceWriter(self.trace_files['install'], break_if_there=False)
self.package_registries = kargs.get('package_registries', {})
def _get_download_config(self):
return None
@ -127,12 +135,6 @@ class PkgInstallComponent(component.Component):
pkg_list = self._clear_package_duplicates(pkg_list)
return pkg_list
def _make_packager(self, name, pkg_info, default_cls):
if name not in self.package_registries:
self.package_registries[name] = packager.Registry()
cls = packager.get_packager_class(pkg_info, default_cls)
return cls(self.distro, self.package_registries[name])
def install(self):
LOG.debug('Preparing to install packages for: %r', self.name)
pkgs = self.packages
@ -142,7 +144,7 @@ class PkgInstallComponent(component.Component):
header="Setting up %s distribution packages" % (len(pkg_names)))
with utils.progress_bar('Installing', len(pkgs)) as p_bar:
for (i, p) in enumerate(pkgs):
installer = self._make_packager('distro', p, self.distro.package_manager_class)
installer = make_packager(p, self.distro, self.distro.package_manager_class)
self.tracewriter.package_installed(p)
installer.install(p)
p_bar.update(i + 1)
@ -150,13 +152,13 @@ class PkgInstallComponent(component.Component):
def pre_install(self):
pkgs = self.packages
for p in pkgs:
installer = self._make_packager('distro', p, self.distro.package_manager_class)
installer = make_packager(p, self.distro, self.distro.package_manager_class)
installer.pre_install(p, self.params)
def post_install(self):
pkgs = self.packages
for p in pkgs:
installer = self._make_packager('distro', p, self.distro.package_manager_class)
installer = make_packager(p, self.distro, self.distro.package_manager_class)
installer.post_install(p, self.params)
@property
@ -378,7 +380,7 @@ class PythonInstallComponent(PkgInstallComponent):
with utils.progress_bar('Installing', len(pips)) as p_bar:
for (i, p) in enumerate(pips):
self.tracewriter.pip_installed(p)
installer = self._make_packager('pip', p, pip.Packager)
installer = make_packager(p, self.distro, pip.Packager)
installer.install(p)
p_bar.update(i + 1)
@ -416,13 +418,13 @@ class PythonInstallComponent(PkgInstallComponent):
self._verify_pip_requires()
PkgInstallComponent.pre_install(self)
for p in self.pips:
installer = self._make_packager('pip', p, pip.Packager)
installer = make_packager(p, self.distro, pip.Packager)
installer.pre_install(p, self.params)
def post_install(self):
PkgInstallComponent.post_install(self)
for p in self.pips:
installer = self._make_packager('pip', p, pip.Packager)
installer = make_packager(p, self.distro, pip.Packager)
installer.post_install(p, self.params)
def _install_python_setups(self):
@ -630,13 +632,6 @@ class PkgUninstallComponent(component.Component):
def __init__(self, *args, **kargs):
component.Component.__init__(self, *args, **kargs)
self.tracereader = tr.TraceReader(self.trace_files['install'])
self.package_registries = kargs.get('package_registries', {})
def _make_packager(self, name, pkg_info, default_cls):
if name not in self.package_registries:
self.package_registries[name] = packager.Registry()
cls = packager.get_packager_class(pkg_info, default_cls)
return cls(self.distro, self.package_registries[name])
def unconfigure(self):
self._unconfigure_files()
@ -680,7 +675,7 @@ class PkgUninstallComponent(component.Component):
which_removed = set()
with utils.progress_bar('Uninstalling', len(pkgs), reverse=True) as p_bar:
for (i, p) in enumerate(pkgs):
uninstaller = self._make_packager('distro', p, self.distro.package_manager_class)
uninstaller = make_packager(p, self.distro, self.distro.package_manager_class)
if uninstaller.remove(p):
which_removed.add(p['name'])
p_bar.update(i + 1)
@ -731,7 +726,7 @@ class PythonUninstallComponent(PkgUninstallComponent):
with utils.progress_bar('Uninstalling', len(pips), reverse=True) as p_bar:
for (i, p) in enumerate(pips):
try:
uninstaller = self._make_packager('pip', p, pip.Packager)
uninstaller = make_packager(p, self.distro, pip.Packager)
uninstaller.remove(p)
except excp.ProcessExecutionError as e:
# NOTE(harlowja): pip seems to die if a pkg isn't there even in quiet mode

4
anvil/components/keystone.py
View File

@ -212,7 +212,9 @@ class KeystoneRuntime(comp.PythonRuntime):
**self.get_option('glance'))
params['nova'] = nhelper.get_shared_params(ip=self.get_option('ip'),
**self.get_option('nova'))
init_what = utils.load_yaml_text(utils.expand_template(sh.load_file(fn), params))
init_what = utils.load_yaml(sh.load_file(fn))
init_what = utils.expand_template_deep(init_what, params)
khelper.Initializer(params['keystone']['service_token'],
params['keystone']['endpoints']['admin']['uri']).initialize(**init_what)
# Writing this makes sure that we don't init again

5
anvil/distros/rhel.py
View File

@ -191,9 +191,8 @@ class YumPackagerWithRelinks(yum.YumPackager):
if not tgt or not src:
continue
src = glob.glob(src)
tgt = glob.glob(tgt)
if not tgt:
tgt = [entry.get('target')]
if not isinstance(tgt, (list, tuple)):
tgt = [tgt]
if len(src) != len(tgt):
raise RuntimeError("Unable to link %s sources to %s locations" % (len(src), len(tgt)))
for i in range(len(src)):

30
anvil/packager.py
View File

@ -15,7 +15,6 @@
# under the License.
import abc
import pkg_resources
from anvil import exceptions as excp
from anvil import colorizer
@ -29,7 +28,6 @@ VERSION_CHARS = ['=', '>', "<"]
class NullVersion(object):
def __init__(self, name):
self.name = name
@ -37,36 +35,26 @@ class NullVersion(object):
return True
def __str__(self):
return "%s (no version)" % (self.name)
return "%s (unknown version)" % (self.name)
class Registry(object):
def __init__(self):
self.installed = {}
self.removed = {}
class Packager(object):
__meta__ = abc.ABCMeta
def __init__(self, distro, registry):
def __init__(self, distro, registry=None):
self.distro = distro
if not registry:
registry = Registry()
self.registry = registry
def _parse_version(self, name, version):
if version:
# This won't work for all package versions (ie crazy names)
# but good enough for now...
if contains_version_check(version):
full_name = "%s%s" % (name, version)
else:
full_name = "%s==%s" % (name, version)
p_version = pkg_resources.Requirement.parse(full_name)
else:
p_version = NullVersion(name)
return p_version
return NullVersion(name)
def _compare_against_installed(self, incoming_version, installed_version):
if not incoming_version and installed_version:
@ -76,11 +64,6 @@ class Packager(object):
# Assume whats installed will work
# (not really the case all the time)
return True
if contains_version_check(incoming_version):
cleaned_version = incoming_version
for c in VERSION_CHARS:
cleaned_version = cleaned_version.replace(c, '')
return self._compare_against_installed(cleaned_version.strip(), installed_version)
if not incoming_version in installed_version:
# Not in the range of the installed version (bad!)
return False
@ -150,5 +133,4 @@ def get_packager_class(package_info, default_packager_class=None):
packager_name = packager_name.strip()
if not packager_name:
return default_packager_class
packager_class = importer.import_entry_point(packager_name)
return packager_class
return importer.import_entry_point(packager_name)

15
anvil/packaging/helpers/__init__.py Normal file
View File

@ -0,0 +1,15 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright (C) 2012 Yahoo! Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

41
anvil/packaging/helpers/pip_helper.py Normal file
View File

@ -0,0 +1,41 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright (C) 2012 Yahoo! Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from anvil import log as logging
from anvil import packager as pack
from anvil import utils
import pip
from pip.util import get_installed_distributions
LOG = logging.getLogger(__name__)
def make_registry():
installations = {}
for dist in get_installed_distributions(local_only=True):
freq = pip.FrozenRequirement.from_dist(dist, [])
if freq.req and freq.name:
name = freq.name.lower()
installations[name] = freq.req
# TODO(harlowja) use the pip version/requirement to enhance this...
reg = pack.Registry()
for (name, _req) in installations.items():
reg.installed[name] = pack.NullVersion(name)
LOG.debug("Identified %s packages already installed by pip", len(reg.installed))
utils.log_object(reg.installed, logger=LOG, level=logging.DEBUG)
return reg

35
anvil/packaging/helpers/yum_helper.py Normal file
View File

@ -0,0 +1,35 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright (C) 2012 Yahoo! Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from anvil import log as logging
from anvil import packager as pack
from anvil import utils
from yum import YumBase
LOG = logging.getLogger(__name__)
def make_registry():
reg = pack.Registry()
yb = YumBase()
yb.conf.cache = False
for p in yb.rpmdb.returnPackages():
# TODO(harlowja) use the rpm version comparision to enhance this...
reg.installed[p.name] = pack.NullVersion(p.name)
LOG.debug("Identified %s packages already installed by yum", len(reg.installed))
utils.log_object(reg.installed, logger=LOG, level=logging.DEBUG)
return reg

18
anvil/pip.py → anvil/packaging/pip.py
View File

@ -19,6 +19,8 @@ from anvil import log as logging
from anvil import shell as sh
from anvil import packager as pack
from anvil.packaging.helpers import pip_helper
LOG = logging.getLogger(__name__)
PIP_UNINSTALL_CMD_OPTS = ['-y', '-q']
@ -26,6 +28,10 @@ PIP_INSTALL_CMD_OPTS = ['-q']
class Packager(pack.Packager):
PIP_REGISTRY = pip_helper.make_registry()
def __init__(self, distro):
pack.Packager.__init__(self, distro, Packager.PIP_REGISTRY)
def _make_pip_name(self, name, version):
if version is None:
@ -35,6 +41,18 @@ class Packager(pack.Packager):
else:
return "%s==%s" % (name, version)
def _parse_version(self, name, version):
if version:
# This should work for all pip packages
if contains_version_check(version):
full_name = "%s%s" % (name, version)
else:
full_name = "%s==%s" % (name, version)
p_version = pkg_resources.Requirement.parse(full_name)
else:
p_version = pack.Packager._parse_version(self, name, version)
return p_version
def _get_pip_command(self):
return self.distro.get_command_config('pip')

10
anvil/packaging/yum.py
View File

@ -18,6 +18,8 @@ from anvil import log as logging
from anvil import packager as pack
from anvil import shell as sh
from anvil.packaging.helpers import yum_helper
LOG = logging.getLogger(__name__)
# Root yum command
@ -34,6 +36,10 @@ VERSION_TEMPL = "%s-%s"
class YumPackager(pack.Packager):
YUM_REGISTRY = yum_helper.make_registry()
def __init__(self, distro):
pack.Packager.__init__(self, distro, YumPackager.YUM_REGISTRY)
def _format_pkg_name(self, name, version):
if version:
@ -43,9 +49,7 @@ class YumPackager(pack.Packager):
def _execute_yum(self, cmd, **kargs):
full_cmd = YUM_CMD + cmd
return sh.execute(*full_cmd, run_as_root=True,
check_exit_code=True,
**kargs)
return sh.execute(*full_cmd, run_as_root=True, check_exit_code=True, **kargs)
def _remove_special(self, name, info):
return False

21
anvil/utils.py
View File

@ -74,6 +74,27 @@ def expand_template(contents, params):
return Template(str(contents), searchList=[params]).respond()
def expand_template_deep(root, params):
if isinstance(root, (basestring, str)):
return expand_template(root, params)
if isinstance(root, (list, tuple)):
n_list = []
for i in root:
n_list.append(expand_template_deep(i, params))
return n_list
if isinstance(root, (dict)):
n_dict = {}
for (k, v) in root.items():
n_dict[k] = expand_template_deep(v, params)
return n_dict
if isinstance(root, (set)):
n_set = set()
for v in root:
n_set.add(expand_template_deep(v, params))
return n_set
return root
def load_yaml(fn):
return load_yaml_text(sh.load_file(fn))

3
conf/templates/keystone/init_what.yaml
View File

@ -1,5 +1,6 @@
##
## This is a cheetah/yaml template!
## This is a yaml template (with cheetah template
## strings that will be filled in)...
##
---
endpoints: