diff --git a/anvil/__main__.py b/anvil/__main__.py index ee25aad2..cc7cbf6e 100644 --- a/anvil/__main__.py +++ b/anvil/__main__.py @@ -34,19 +34,17 @@ from anvil import log as logging from anvil import opts from anvil.packaging import yum from anvil import persona +from anvil import pprint from anvil import settings from anvil import shell as sh from anvil import utils -from anvil.pprint import center_text - LOG = logging.getLogger() def run(args): - """ - Starts the execution after args have been parsed and logging has been setup. + """Starts the execution after args have been parsed and logging has been setup. """ LOG.debug("CLI arguments are:") @@ -80,7 +78,7 @@ def run(args): root_dir = sh.abspth(root_dir) (repeat_string, line_max_len) = utils.welcome() - print(center_text("Action Runner", repeat_string, line_max_len)) + print(pprint.center_text("Action Runner", repeat_string, line_max_len)) # !! # Here on out we should be using the logger (and not print)!! @@ -176,10 +174,9 @@ def ensure_perms(): def main(): - """ - Starts the execution of without - injecting variables into the global namespace. Ensures that - logging is setup and that sudo access is available and in-use. + """Starts the execution of anvil without injecting variables into + the global namespace. Ensures that logging is setup and that sudo access + is available and in-use. Arguments: N/A Returns: 1 for success, 0 for failure and 2 for permission change failure. @@ -241,6 +238,6 @@ if __name__ == "__main__": if return_code != 2: try: sh.root_mode(quiet=False) - except: + except Exception: pass sys.exit(return_code) diff --git a/anvil/actions/__init__.py b/anvil/actions/__init__.py index 5b41d482..420d2e31 100644 --- a/anvil/actions/__init__.py +++ b/anvil/actions/__init__.py @@ -42,16 +42,12 @@ _RUNNER_TO_NAMES = dict((v, k) for k, v in _NAMES_TO_RUNNER.items()) def names(): - """ - Returns a list of the available action names. - """ + """Returns a list of the available action names.""" return list(sorted(_NAMES_TO_RUNNER.keys())) def class_for(action): - """ - Given an action name, look up the factory for that action runner. - """ + """Given an action name, look up the factory for that action runner.""" try: return _NAMES_TO_RUNNER[action] except KeyError: diff --git a/anvil/actions/base.py b/anvil/actions/base.py index 31958573..c4173956 100644 --- a/anvil/actions/base.py +++ b/anvil/actions/base.py @@ -177,9 +177,7 @@ class Action(object): return opts def _construct_instances(self, persona): - """ - Create component objects for each component in the persona. - """ + """Create component objects for each component in the persona.""" persona_subsystems = persona.wanted_subsystems or {} persona_opts = persona.component_options or {} wanted_components = persona.wanted_components or [] @@ -259,9 +257,7 @@ class Action(object): return sh.joinpths(self.phase_dir, "%s.phases" % (phase_name)) def _run_phase(self, functors, component_order, instances, phase_name, *inv_phase_names): - """ - Run a given 'functor' across all of the components, in order. - """ + """Run a given 'functor' across all of the components, in order.""" # All the results for each component end up in here # in the order in which they ran... component_results = OrderedDict() diff --git a/anvil/actions/status.py b/anvil/actions/status.py index 02197f38..55d1489d 100644 --- a/anvil/actions/status.py +++ b/anvil/actions/status.py @@ -22,8 +22,10 @@ from anvil.actions import base as action LOG = log.getLogger(__name__) -from anvil.components.base_runtime import (STATUS_INSTALLED, STATUS_STARTED, - STATUS_STOPPED, STATUS_UNKNOWN) +from anvil.components.base_runtime import STATUS_INSTALLED +from anvil.components.base_runtime import STATUS_STARTED +from anvil.components.base_runtime import STATUS_STOPPED +from anvil.components.base_runtime import STATUS_UNKNOWN STATUS_COLOR_MAP = { STATUS_INSTALLED: 'green', diff --git a/anvil/cfg.py b/anvil/cfg.py index fbc2673d..d7410374 100644 --- a/anvil/cfg.py +++ b/anvil/cfg.py @@ -16,15 +16,16 @@ # This one doesn't keep comments but does seem to work better import ConfigParser -from ConfigParser import (NoSectionError, NoOptionError) +from ConfigParser import NoOptionError +from ConfigParser import NoSectionError import re +from StringIO import StringIO + # This one keeps comments but has some weirdness with it import iniparse -from StringIO import StringIO - from anvil import log as logging from anvil import shell as sh from anvil import utils diff --git a/anvil/components/configurators/base.py b/anvil/components/configurators/base.py index 4b892877..c4b7ea50 100644 --- a/anvil/components/configurators/base.py +++ b/anvil/components/configurators/base.py @@ -17,8 +17,8 @@ import io import weakref -from anvil import shell as sh from anvil import cfg +from anvil import shell as sh from anvil import utils from anvil.components.helpers import db as dbhelper diff --git a/anvil/components/configurators/keystone.py b/anvil/components/configurators/keystone.py index cd534cc1..7d29c66a 100644 --- a/anvil/components/configurators/keystone.py +++ b/anvil/components/configurators/keystone.py @@ -14,8 +14,8 @@ # License for the specific language governing permissions and limitations # under the License. -from anvil import utils from anvil import shell as sh +from anvil import utils from anvil.components.helpers import keystone as khelper diff --git a/anvil/components/configurators/nova.py b/anvil/components/configurators/nova.py index c0cf7df4..cb193f40 100644 --- a/anvil/components/configurators/nova.py +++ b/anvil/components/configurators/nova.py @@ -14,10 +14,10 @@ # License for the specific language governing permissions and limitations # under the License. -from anvil import shell as sh from anvil import exceptions -from anvil import utils from anvil import log as logging +from anvil import shell as sh +from anvil import utils from anvil.components.helpers import quantum as qhelper from anvil.components.helpers import virt as lv @@ -56,10 +56,10 @@ class NovaConfigurator(base.Configurator): config.add_with_section('filter:authtoken', k, v) def _config_adjust_api(self, nova_conf): - ''' This method has the smarts to build the configuration file based on - various runtime values. A useful reference for figuring out this - is at http://docs.openstack.org/diablo/openstack-compute/admin/content/ch_configuring-openstack-compute.html - See also: https://github.com/openstack/nova/blob/master/etc/nova/nova.conf.sample + '''This method has the smarts to build the configuration file based on + various runtime values. A useful reference for figuring out this + is at http://docs.openstack.org/diablo/openstack-compute/admin/content/ch_configuring-openstack-compute.html + See also: https://github.com/openstack/nova/blob/master/etc/nova/nova.conf.sample ''' # Used more than once so we calculate it ahead of time @@ -333,16 +333,16 @@ class NovaConfigurator(base.Configurator): public_interface = self.installer.get_option('public_interface') vlan_interface = self.installer.get_option('vlan_interface', default_value=public_interface) known_interfaces = utils.get_interfaces() - if not public_interface in known_interfaces: + if public_interface not in known_interfaces: msg = "Public interface %r is not a known interface (is it one of %s??)" % (public_interface, ", ".join(known_interfaces)) raise exceptions.ConfigException(msg) - if not vlan_interface in known_interfaces: + if vlan_interface not in known_interfaces: msg = "VLAN interface %r is not a known interface (is it one of %s??)" % (vlan_interface, ", ".join(known_interfaces)) raise exceptions.ConfigException(msg) # Driver specific interface checks drive_canon = utils.canon_virt_driver(self.installer.get_option('virt_driver')) if drive_canon == 'libvirt': flat_interface = self.installer.get_option('flat_interface') - if flat_interface and not flat_interface in known_interfaces: + if flat_interface and flat_interface not in known_interfaces: msg = "Libvirt flat interface %s is not a known interface (is it one of %s??)" % (flat_interface, ", ".join(known_interfaces)) raise exceptions.ConfigException(msg) diff --git a/anvil/components/configurators/quantum.py b/anvil/components/configurators/quantum.py index bf156c23..e12941a5 100644 --- a/anvil/components/configurators/quantum.py +++ b/anvil/components/configurators/quantum.py @@ -14,13 +14,12 @@ # License for the specific language governing permissions and limitations # under the License. -from anvil import shell as sh from anvil import importer +from anvil import shell as sh from anvil.components.configurators import base -from anvil.components.configurators.quantum_plugins import l3 from anvil.components.configurators.quantum_plugins import dhcp - +from anvil.components.configurators.quantum_plugins import l3 # Special generated conf API_CONF = "quantum.conf" diff --git a/anvil/components/helpers/db.py b/anvil/components/helpers/db.py index 33facc94..07d57e9b 100644 --- a/anvil/components/helpers/db.py +++ b/anvil/components/helpers/db.py @@ -74,9 +74,7 @@ def create_db(distro, dbtype, user, pw, dbname, **kwargs): def grant_permissions(dbtype, distro, user, pw, restart_func=None): - """ - Grant permissions on the database. - """ + """Grant permissions on the database.""" dbactions = distro.get_command_config(dbtype, quiet=True) if dbactions: grant_cmd = distro.get_command(dbtype, 'grant_all') diff --git a/anvil/components/helpers/glance.py b/anvil/components/helpers/glance.py index 52cf831f..02dc5935 100644 --- a/anvil/components/helpers/glance.py +++ b/anvil/components/helpers/glance.py @@ -104,8 +104,7 @@ class Unpacker(object): return False def _find_pieces(self, files, files_location): - """ - Match files against the patterns in KERNEL_CHECKS, + """Match files against the patterns in KERNEL_CHECKS, RAMDISK_CHECKS, and ROOT_CHECKS to determine which files contain which image parts. """ @@ -137,8 +136,7 @@ class Unpacker(object): return sh.pipe_in_out(mfh, ofh) def _describe(self, root_fn, ramdisk_fn, kernel_fn): - """ - Make an "info" dict that describes the path, disk format, and + """Make an "info" dict that describes the path, disk format, and container format of each component of an image. """ info = dict() @@ -206,8 +204,7 @@ class Unpacker(object): header="Found %s images from a %s" % (len(pieces), src_type)) def _unpack_dir(self, dir_path): - """ - Pick through a directory to figure out which files are which + """Pick through a directory to figure out which files are which image pieces, and create a dict that describes them. """ potential_files = set() diff --git a/anvil/components/helpers/keystone.py b/anvil/components/helpers/keystone.py index 58dbf353..bf8d727b 100644 --- a/anvil/components/helpers/keystone.py +++ b/anvil/components/helpers/keystone.py @@ -83,9 +83,9 @@ class Initializer(object): tenant_name = t if not role_name or not tenant_name: raise RuntimeError("Role or tenant name missing for user %s" % (name)) - if not role_name in roles_made: + if role_name not in roles_made: raise RuntimeError("Role %s not previously created for user %s" % (role_name, name)) - if not tenant_name in tenants_made: + if tenant_name not in tenants_made: raise RuntimeError("Tenant %s not previously created for user %s" % (tenant_name, name)) user_role = { 'user': user, diff --git a/anvil/components/helpers/nova.py b/anvil/components/helpers/nova.py index f265f348..8402cedb 100644 --- a/anvil/components/helpers/nova.py +++ b/anvil/components/helpers/nova.py @@ -21,8 +21,8 @@ import weakref from anvil import shell as sh from anvil import utils -from anvil.components.helpers import virt as lv from anvil.components.configurators import nova as nconf +from anvil.components.helpers import virt as lv def get_shared_params(ip, protocol, diff --git a/anvil/components/keystone.py b/anvil/components/keystone.py index 00ed011f..935df028 100644 --- a/anvil/components/keystone.py +++ b/anvil/components/keystone.py @@ -25,11 +25,11 @@ from anvil.components import base_install as binstall from anvil.components import base_runtime as bruntime from anvil.components import base_testing as btesting +from anvil.components.helpers import cinder as chelper from anvil.components.helpers import glance as ghelper from anvil.components.helpers import keystone as khelper from anvil.components.helpers import nova as nhelper from anvil.components.helpers import quantum as qhelper -from anvil.components.helpers import cinder as chelper from anvil.components.configurators import keystone as kconf diff --git a/anvil/distro.py b/anvil/distro.py index ea58ec54..c07bc68e 100644 --- a/anvil/distro.py +++ b/anvil/distro.py @@ -49,7 +49,7 @@ class Distro(object): self._components = components def get_command_config(self, key, *more_keys, **kargs): - """ Gets a end object for a given set of keys """ + """Gets a end object for a given set of keys """ root = self._commands acutal_keys = [key] + list(more_keys) run_over_keys = acutal_keys[0:-1] diff --git a/anvil/downloader.py b/anvil/downloader.py index e00b65de..6981a965 100644 --- a/anvil/downloader.py +++ b/anvil/downloader.py @@ -18,8 +18,7 @@ import abc import contextlib import functools import urllib2 - -from urlparse import parse_qs +import urlparse import progressbar @@ -55,7 +54,7 @@ class GitDownloader(Downloader): # If we use urlparser here it doesn't seem to work right?? # TODO(harlowja), why?? (uri, params) = uri.split("?", 1) - params = parse_qs(params) + params = urlparse.parse_qs(params) if 'branch' in params: branch = params['branch'][0].strip() if 'tag' in params: diff --git a/anvil/importer.py b/anvil/importer.py index b1032192..10c354bf 100644 --- a/anvil/importer.py +++ b/anvil/importer.py @@ -36,9 +36,7 @@ def construct_entry_point(fullname, *args, **kwargs): def partition(fullname): - """ - The name should be in dotted.path:ClassName syntax. - """ + """The name should be in dotted.path:ClassName syntax.""" if ':' not in fullname: raise ValueError('Invalid entry point specifier %r' % fullname) (module_name, _sep, classname) = fullname.partition(':') @@ -46,9 +44,7 @@ def partition(fullname): def import_entry_point(fullname): - """ - Given a name import the class and return it. - """ + """Given a name import the class and return it.""" (module_name, classname) = partition(fullname) try: import_module(module_name) diff --git a/anvil/log.py b/anvil/log.py index 52624218..002234d1 100644 --- a/anvil/log.py +++ b/anvil/log.py @@ -22,9 +22,6 @@ import logging import sys -from logging.handlers import SysLogHandler -from logging.handlers import WatchedFileHandler - from anvil import colorizer @@ -57,8 +54,6 @@ Formatter = logging.Formatter # Handlers StreamHandler = logging.StreamHandler -WatchedFileHandler = WatchedFileHandler -SysLogHandler = SysLogHandler class TermFormatter(logging.Formatter): diff --git a/anvil/opts.py b/anvil/opts.py index cc55b066..ef9c1729 100644 --- a/anvil/opts.py +++ b/anvil/opts.py @@ -17,7 +17,9 @@ import multiprocessing from optparse import IndentedHelpFormatter -from optparse import (OptionParser, OptionGroup, OptionValueError) +from optparse import OptionGroup +from optparse import OptionParser +from optparse import OptionValueError from anvil import actions from anvil import settings diff --git a/anvil/packaging/base.py b/anvil/packaging/base.py index 4d7f3b77..8ed2fff0 100644 --- a/anvil/packaging/base.py +++ b/anvil/packaging/base.py @@ -69,9 +69,6 @@ class DependencyHandler(object): """Basic class for handler of OpenStack dependencies. """ MAX_PIP_DOWNLOAD_ATTEMPTS = 4 - multipip_executable = sh.which("multipip", ["tools/"]) - pip_executable = sh.which_first(['pip-python', 'pip']) - pipdownload_executable = sh.which("pip-download", ["tools"]) def __init__(self, distro, root_dir, instances, opts=None): self.distro = distro @@ -86,7 +83,11 @@ class DependencyHandler(object): self.gathered_requires_filename = sh.joinpths(self.deps_dir, "pip-requires") self.forced_requires_filename = sh.joinpths(self.deps_dir, "forced-requires") self.download_requires_filename = sh.joinpths(self.deps_dir, "download-requires") - # List of requirement strings + # Executables we require to operate + self.multipip_executable = sh.which("multipip", ["tools/"]) + self.pip_executable = sh.which_first(['pip-python', 'pip']) + self.pipdownload_executable = sh.which("pip-download", ["tools/"]) + # List of requirements self.pips_to_install = [] self.forced_packages = [] # Instances to there app directory (with a setup.py inside) @@ -184,7 +185,7 @@ class DependencyHandler(object): try: req = pip_helper.extract_requirement(line) new_lines.append(str(forced_by_key[req.key])) - except: + except Exception: # we don't force the package or it has a bad format new_lines.append(line) contents = "# Cleaned on %s\n\n%s\n" % (utils.iso8601(), "\n".join(new_lines)) @@ -241,7 +242,8 @@ class DependencyHandler(object): "\n".join(str(req) for req in self.forced_packages)) def filter_download_requires(self): - """ + """Shrinks the pips that were downloaded into a smaller set. + :returns: a list of all requirements that must be downloaded :rtype: list of str """ diff --git a/anvil/packaging/helpers/pip_helper.py b/anvil/packaging/helpers/pip_helper.py index 94d03b84..5940c775 100644 --- a/anvil/packaging/helpers/pip_helper.py +++ b/anvil/packaging/helpers/pip_helper.py @@ -14,12 +14,12 @@ # License for the specific language governing permissions and limitations # under the License. -import re import copy - import pkg_resources -from pip import util as pip_util +import re + from pip import req as pip_req +from pip import util as pip_util from anvil import log as logging from anvil import shell as sh diff --git a/anvil/packaging/helpers/yum_helper.py b/anvil/packaging/helpers/yum_helper.py index 896b31c7..b3b75a68 100644 --- a/anvil/packaging/helpers/yum_helper.py +++ b/anvil/packaging/helpers/yum_helper.py @@ -23,12 +23,15 @@ LOG = logging.getLogger(__name__) class Helper(object): - yyoom_executable = sh.which("yyoom", ["tools/"]) def __init__(self, log_dir): + # Executables we require to operate + self.yyoom_executable = sh.which("yyoom", ["tools/"]) + # Executable logs will go into this directory + self._log_dir = log_dir + # Caches of installed and available packages self._installed = None self._available = None - self._log_dir = log_dir def _yyoom(self, arglist, cmd_type): cmdline = [self.yyoom_executable, '--verbose'] diff --git a/anvil/packaging/yum.py b/anvil/packaging/yum.py index 733aac09..c79afda4 100644 --- a/anvil/packaging/yum.py +++ b/anvil/packaging/yum.py @@ -88,11 +88,7 @@ class YumDependencyHandler(base.DependencyHandler): " --config-file=/etc/quantum/quantum.conf'"), } REPOS = ["anvil-deps", "anvil"] - py2rpm_executable = sh.which("py2rpm", ["tools/"]) - rpmbuild_executable = sh.which("rpmbuild") - specprint_executable = sh.which('specprint', ["tools/"]) - yumfind_executable = sh.which("yumfind", ["tools/"]) - jobs = 2 + JOBS = 2 def __init__(self, distro, root_dir, instances, opts=None): super(YumDependencyHandler, self).__init__(distro, root_dir, instances, opts) @@ -103,9 +99,13 @@ class YumDependencyHandler(base.DependencyHandler): self.anvil_repo_filename = sh.joinpths(self.deps_dir, self.REPO_FN) self.rpm_sources_dir = sh.joinpths(self.rpmbuild_dir, "SOURCES") self.anvil_repo_dir = sh.joinpths(self.root_dir, "repo") + # Executables we require to operate + self.py2rpm_executable = sh.which("py2rpm", ["tools/"]) + self.rpmbuild_executable = sh.which("rpmbuild") + self.specprint_executable = sh.which('specprint', ["tools/"]) + self.yumfind_executable = sh.which("yumfind", ["tools/"]) # We inspect yum for packages, this helper allows us to do this. self.helper = yum_helper.Helper(self.log_dir) - self._no_remove = None def py2rpm_start_cmdline(self): cmdline = [ @@ -206,7 +206,7 @@ class YumDependencyHandler(base.DependencyHandler): utils.log_iterable(src_repo_files, header=('Building %s RPM packages from their' ' SRPMs for repo %s using %s jobs') % - (len(src_repo_files), self.SRC_REPOS[repo_name], self.jobs), + (len(src_repo_files), self.SRC_REPOS[repo_name], self.JOBS), logger=LOG) makefile_path = sh.joinpths(self.deps_dir, "binary-%s.mk" % repo_name) marks_dir = sh.joinpths(self.deps_dir, "marks-binary") @@ -233,7 +233,7 @@ class YumDependencyHandler(base.DependencyHandler): self._create_repo(repo_name) def _execute_make(self, filename, marks_dir): - cmdline = ["make", "-f", filename, "-j", str(self.jobs)] + cmdline = ["make", "-f", filename, "-j", str(self.JOBS)] out_filename = sh.joinpths(self.log_dir, "%s.log" % sh.basename(filename)) sh.execute_save_output(cmdline, cwd=marks_dir, out_filename=out_filename) @@ -393,7 +393,7 @@ class YumDependencyHandler(base.DependencyHandler): sh.write_file(makefile_path, utils.expand_template(content, params), tracewriter=self.tracewriter) utils.log_iterable(package_files, - header="Building %s SRPM packages using %s jobs" % (len(package_files), self.jobs), + header="Building %s SRPM packages using %s jobs" % (len(package_files), self.JOBS), logger=LOG) self._execute_make(makefile_path, marks_dir) diff --git a/anvil/utils.py b/anvil/utils.py index b5f5ea5a..63f53947 100644 --- a/anvil/utils.py +++ b/anvil/utils.py @@ -390,8 +390,7 @@ def tempdir(**kwargs): def get_host_ip(default_ip='127.0.0.1'): - """ - Returns the actual ip of the local machine. + """Returns the actual ip of the local machine. This code figures out what source address would be used if some traffic were to be sent out to some well known address on the Internet. In this diff --git a/setup.cfg b/setup.cfg index 1ee760e8..31793315 100644 --- a/setup.cfg +++ b/setup.cfg @@ -29,6 +29,14 @@ setup-hooks = packages = anvil +scripts = + tools/multipip + tools/pip-download + tools/py2rpm + tools/specprint + tools/yumfind + tools/yyoom + [nosetests] verbosity=2 diff --git a/test-requirements.txt b/test-requirements.txt index 37799f2d..68110862 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -3,3 +3,4 @@ pep8==1.4.5 pyflakes==0.7.2 flake8==2.0 pylint==0.25.2 +hacking>=0.5.3,<0.6 diff --git a/tox.ini b/tox.ini index 98daa249..c7288194 100644 --- a/tox.ini +++ b/tox.ini @@ -39,6 +39,6 @@ setenv = NOSE_WITH_COVERAGE=1 commands = {posargs} [flake8] -ignore = H402,H403,E121,E123,E124,E125,E126,E127,E128,E202,E501 +ignore = H302,H402,H403,E121,E123,E124,E125,E126,E127,E128,E202,E501 builtins = _ exclude = .venv,.tox,dist,doc,*egg,.git,build,tools