Handling subsystems and runner kargs cleanups

conf/distros/ubuntu-oneiric.yaml

@@ -1,7 +1,5 @@
 # Ubuntu 11 (Oneiric)
 
-# FIXME: Component dependencies should go into personas, not distros.
-
 name: ubuntu-oneiric
 distro_pattern: Ubuntu(.*)oneiric
 
@@ -33,6 +31,7 @@ commands:
     drop_db: ['mysql', '--user=%USER%', '--password=%PASSWORD%', '-e', 'DROP DATABASE IF EXISTS %DB%;']
     grant_all: ["mysql", "--user=%USER%", "--password=%PASSWORD%", '-e', 
                 "\"GRANT ALL PRIVILEGES ON *.* TO '%USER%'@'%' IDENTIFIED BY '%PASSWORD%'; FLUSH PRIVILEGES;\""]
+
   iscsi:
     start: ['service', 'tgt', 'start']
     stop: ['service', 'tgt', 'stop']
@@ -158,10 +157,6 @@ components:
     uninstall: devstack.components.glance:GlanceUninstaller
     start: devstack.components.glance:GlanceRuntime
     stop: devstack.components.glance:GlanceRuntime
-    dependencies:
-      - general
-      - keystone
-      - db
     packages:
       - name: python-eventlet
         version: 0.9*
@@ -205,12 +200,6 @@ components:
     uninstall: devstack.components.horizon:HorizonUninstaller
     start: devstack.components.horizon:HorizonRuntime
     stop: devstack.components.horizon:HorizonRuntime
-    dependencies:
-      - general
-      - keystone-client
-      - glance
-      - nova-client
-      - quantum-client
     packages:
       - name: apache2
         removable: True
@@ -285,8 +274,6 @@ components:
     uninstall: devstack.components.keystone_client:KeyStoneClientUninstaller
     start: devstack.components.keystone_client:KeyStoneClientRuntime
     stop: devstack.components.keystone_client:KeyStoneClientRuntime
-    dependencies:
-      - general
     packages:
       - name: python-argparse
         removable: True
@@ -300,10 +287,6 @@ components:
     uninstall: devstack.components.keystone:KeystoneUninstaller
     start: devstack.components.keystone:KeystoneRuntime
     stop: devstack.components.keystone:KeystoneRuntime
-    dependencies:
-      - general
-      - db
-      - keystone-client
     packages:
       - name: libldap2-dev
         removable: True
@@ -368,9 +351,6 @@ components:
     uninstall: devstack.components.melange:MelangeUninstaller
     start: devstack.components.melange:MelangeRuntime
     stop: devstack.components.melange:MelangeRuntime
-    dependencies:
-      - general
-      - db
     packages:
       - name: python-eventlet
         removable: True
@@ -397,88 +377,21 @@ components:
         removable: True
         version: 1.0*
 
-  nova-api:
-    # FIXME: This will report that it is installing/uninstalling
-    # "general" instead of the right name.
-    install: devstack.components.pkglist:Installer
-    uninstall: devstack.components.pkglist:Uninstaller
-    start: devstack.component:EmptyRuntime
-    stop: devstack.component:EmptyRuntime
-    packages:
-      - name: python-dateutil
-        removable: True
-        version: 1.4*
-
-  nova-cpu:
-    # FIXME: This will report that it is installing/uninstalling
-    # "general" instead of the right name.
-    install: devstack.components.pkglist:Installer
-    uninstall: devstack.components.pkglist:Uninstaller
-    start: devstack.component:EmptyRuntime
-    stop: devstack.component:EmptyRuntime
-    packages:
-      - name: kvm
-        removable: True
-        version: 1:84*
-      - name: libvirt-bin
-        removable: True
-        version: 0.9*
-      - name: libvirt0
-        removable: True
-        version: 0.9*
-      - name: lvm2
-        removable: True
-        version: 2.02*
-      - name: open-iscsi
-        removable: True
-        version: 2.0*
-      - name: open-iscsi-utils
-        removable: True
-        version: 2.0*
-      - name: python-libvirt
-        removable: True
-        version: 0.9.2*
-      - name: qemu-kvm
-        removable: True
-        version: 0.14.*
-
   no-vnc:
     install: devstack.components.novnc:NoVNCInstaller
     uninstall: devstack.components.novnc:NoVNCUninstaller
     start: devstack.components.novnc:NoVNCRuntime
     stop: devstack.components.novnc:NoVNCRuntime
-    dependencies:
-      - general
     packages:
       - name: python-numpy
         removable: True
         version: 1:1.5*
 
-  nova-vol:
-    # FIXME: This will report that it is installing/uninstalling
-    # "general" instead of the right name.
-    install: devstack.components.pkglist:Installer
-    uninstall: devstack.components.pkglist:Uninstaller
-    start: devstack.component:EmptyRuntime
-    stop: devstack.component:EmptyRuntime
-    packages:
-      - name: iscsitarget
-        removable: True
-        version: 1.4*
-      - name: lvm2
-        removable: True
-        version: 2.02*
-      - name: tgt
-        removable: True
-        version: 1:1*
-
   nova-client:
     install: devstack.components.nova_client:NovaClientInstaller
     uninstall: devstack.components.nova_client:NovaClientUninstaller
     start: devstack.components.nova_client:NovaClientRuntime
     stop: devstack.components.nova_client:NovaClientRuntime
-    dependencies:
-      - general
     packages:
       - name: python-argparse
         removable: True
@@ -492,16 +405,6 @@ components:
     uninstall: devstack.components.nova:NovaUninstaller
     start: devstack.components.nova:NovaRuntime
     stop: devstack.components.nova:NovaRuntime
-    dependencies:
-      - general
-      - keystone
-      - glance
-      - db
-      - rabbit
-      - nova-client
-      - nova-cpu
-      - nova-vol
-      - nova-api
     packages:
       - name: dnsmasq-base
         removable: True
@@ -603,51 +506,65 @@ components:
     pips:
       - name: iso8601
         version: 0.1.4
+    subsystems:
+      vol:
+        packages:
+          - name: iscsitarget
+            removable: True
+            version: 1.4*
+          - name: lvm2
+            removable: True
+            version: 2.02*
+          - name: tgt
+            removable: True
+            version: 1:1*
+      api:
+        packages:
+          - name: python-dateutil
+            removable: True
+            version: 1.4*
+      cpu:
+        packages:
+          - name: kvm
+            removable: True
+            version: 1:84*
+          - name: libvirt-bin
+            removable: True
+            version: 0.9*
+          - name: libvirt0
+            removable: True
+            version: 0.9*
+          - name: lvm2
+            removable: True
+            version: 2.02*
+          - name: open-iscsi
+            removable: True
+            version: 2.0*
+          - name: open-iscsi-utils
+            removable: True
+            version: 2.0*
+          - name: python-libvirt
+            removable: True
+            version: 0.9.2*
+          - name: qemu-kvm
+            removable: True
+            version: 0.14.*
                             
   quantum-client:
     install: devstack.components.quantum_client:QuantumClientInstaller
     uninstall: devstack.components.quantum_client:QuantumClientUninstaller
     start: devstack.components.quantum_client:QuantumClientRuntime
     stop: devstack.components.quantum_client:QuantumClientRuntime
-    dependencies:
-      - general
     packages:
       - name: python-gflags
         removable: True
         version: 1.5*
 
-  quantum-openvswitch:
-    # FIXME: This will report that it is installing/uninstalling
-    # "general" instead of the right name.
-    install: devstack.components.pkglist:Installer
-    uninstall: devstack.components.pkglist:Uninstaller
-    start: devstack.component:EmptyRuntime
-    stop: devstack.component:EmptyRuntime
-    packages:
-      - name: openvswitch-datapath-dkms
-        removable: True
-        version: 1.2*
-      - name: openvswitch-switch
-        removable: True
-        version: 1.2*
-      - name: python-mysqldb
-        removable: True
-        version: 1.2*
-      - name: python-sqlalchemy
-        removable: True
-        version: 0.6*
-
   quantum:
     install: devstack.components.quantum:QuantumInstaller
     uninstall: devstack.components.quantum:QuantumUninstaller
     start: devstack.components.quantum:QuantumRuntime
     stop: devstack.components.quantum:QuantumRuntime
-    dependencies:
-      - general
-      - quantum-client
-      # Default is to include openvswitch so it is here until
-      # we add proper persona support.
-      - quantum-openvswitch
     packages:
       - name: python-eventlet
         removable: True
@@ -670,6 +587,21 @@ components:
       - name: python-routes
         removable: True
         version: 1.12*
+    subsystems:
+      openvswitch:
+        packages:
+          - name: openvswitch-datapath-dkms
+            removable: True
+            version: 1.2*
+          - name: openvswitch-switch
+            removable: True
+            version: 1.2*
+          - name: python-mysqldb
+            removable: True
+            version: 1.2*
+          - name: python-sqlalchemy
+            removable: True
+            version: 0.6*
 
   rabbit-mq:
     install: devstack.components.rabbit:RabbitInstaller
@@ -686,9 +618,6 @@ components:
     uninstall: devstack.components.swift:SwiftUninstaller
     start: devstack.components.swift:SwiftRuntime
     stop: devstack.components.swift:SwiftRuntime
-    dependencies:
-      - general
-      - keystone-client
     packages:
       - name: memcached
         removable: True

devstack/component.py

@@ -54,7 +54,7 @@ BASE_LINK_DIR = "/etc"
 
 class ComponentBase(object):
     def __init__(self,
-                 subsystems,
+                 active_subsystems,
                  runner,
                  component_dir,
                  all_instances,
@@ -62,7 +62,7 @@ class ComponentBase(object):
                  *args,
                  **kargs):
 
-        self.subsystems = subsystems
+        self.active_subsystems = active_subsystems
         self.instances = all_instances
 
         # The runner has a reference to us, so use a weakref here to
@@ -107,6 +107,7 @@ class PkgInstallComponent(ComponentBase):
         self.tracewriter = tr.TraceWriter(tr.trace_fn(self.trace_dir,
                                                       tr.IN_TRACE))
         self.packages = kargs.get('packages', list())
+        self.subsystems = kargs.get('subsystems', dict())
 
     def _get_download_locations(self):
         return list()
@@ -150,7 +151,14 @@ class PkgInstallComponent(ComponentBase):
         return dict()
 
     def _get_packages(self):
-        return self.packages
+        pkg_list = list(self.packages)
+        for name in self.active_subsystems:
+            if name in self.subsystems:
+                # Todo handle duplicates/version differences?
+                LOG.debug("Extending package list with packages for subsystem %s" % (name))
+                subsystem_pkgs = self.subsystems[name].get('packages', list()) 
+                pkg_list.extend(subsystem_pkgs)
+        return pkg_list
 
     def install(self):
         LOG.debug('Preparing to install packages for %s',
@@ -206,13 +214,9 @@ class PkgInstallComponent(ComponentBase):
         if configs:
             LOG.info("Configuring %s files", len(configs))
             for fn in configs:
-                #get the params and where it should come from and
-                #where it should go
                 parameters = self._get_param_map(fn)
                 tgt_fn = self._get_target_config_name(fn)
-                #ensure directory is there (if not created previously)
                 self.tracewriter.dirs_made(*sh.mkdirslist(sh.dirname(tgt_fn)))
-                #now configure it
                 LOG.info("Configuring file %s", fn)
                 (source_fn, contents) = self._get_source_config(fn)
                 LOG.debug("Replacing parameters in file %s", source_fn)
@@ -221,7 +225,6 @@ class PkgInstallComponent(ComponentBase):
                 LOG.debug("Applying side-effects of param replacement for template %s", source_fn)
                 contents = self._config_adjust(contents, fn)
                 LOG.info("Writing configuration file %s", tgt_fn)
-                #this trace is used to remove the files configured
                 self.tracewriter.cfg_file_written(sh.write_file(tgt_fn,
                                                                 contents))
         return len(configs)
@@ -257,7 +260,14 @@ class PythonInstallComponent(PkgInstallComponent):
         return py_dirs
 
     def _get_pips(self):
-        return self.pips
+        pip_list = list(self.pips)
+        for name in self.active_subsystems:
+            if name in self.subsystems:
+                # Todo handle duplicates/version differences?
+                LOG.debug("Extending pip list with pips for subsystem %s" % (name))
+                subsystem_pips = self.subsystems[name].get('pips', list()) 
+                pip_list.extend(subsystem_pkgs)
+        return pip_list
 
     def _install_pips(self):
         pips = self._get_pips()

devstack/components/nova.py

@@ -238,7 +238,7 @@ class NovaUninstaller(comp.PythonUninstallComponent):
         #these environment additions are important
         #in that they eventually affect how this script runs
         env = dict()
-        env['ENABLED_SERVICES'] = ",".join(self.subsystems)
+        env['ENABLED_SERVICES'] = ",".join(self.active_subsystems)
         env['BIN_DIR'] = self.bin_dir
         env['VOLUME_NAME_PREFIX'] = self.cfg.getdefaulted('nova', 'volume_name_prefix', DEF_VOL_PREFIX)
         cleaner_fn = sh.joinpths(self.bin_dir, CLEANER_DATA_CONF)
@@ -262,10 +262,10 @@ class NovaInstaller(comp.PythonInstallComponent):
         self.cfg_dir = sh.joinpths(self.app_dir, CONFIG_DIR)
         self.paste_conf_fn = self._get_target_config_name(PASTE_CONF)
         self.volumes_enabled = False
-        if NVOL in self.subsystems:
+        if NVOL in self.active_subsystems:
             self.volumes_enabled = True
         self.xvnc_enabled = False
-        if NXVNC in self.subsystems:
+        if NXVNC in self.active_subsystems:
             self.xvnc_enabled = True
 
     def _get_symlinks(self):

devstack/distro.py

@@ -81,12 +81,6 @@ class Distro(object):
     def __repr__(self):
         return "\"%s\" using packager \"%s\"" % (self.name, self.packager_name)
 
-    def get_packages(self, name):
-        return self.components[name].get('packages', list())
-
-    def get_pips(self, name):
-        return self.components[name].get('pips', list())
-
     def get_command(self, cmd_key, quiet=False):
         if not quiet:
             return self.commands[cmd_key]
@@ -104,11 +98,19 @@ class Distro(object):
         """Return a factory for a package manager."""
         return importer.import_entry_point(self.packager_name)
 
-    def get_component_action_class(self, name, action):
+    def extract_component(self, name, action):
-        """Return the class to use for doing the action w/the component."""
+        """Return the class + component info to use for doing the action w/the component."""
         try:
-            entry_point = self.components[name][action]
+            # Use a copy instead of the original
+            component_info = dict(self.components[name])
+            entry_point = component_info[action]
+            cls = importer.import_entry_point(entry_point)
+            # Knock all action class info (and any other keys)
+            key_deletions = [action] + settings.ACTIONS
+            for k in key_deletions:
+                if k in component_info:
+                    del component_info[k]
+            return (cls, component_info)
         except KeyError:
             raise RuntimeError('No class configured to %s %s on %s' %
                                (action, name, self.name))
-        return importer.import_entry_point(entry_point)

devstack/pip.py

@@ -38,7 +38,7 @@ def install(pip, distro):
     if options is not None:
         LOG.debug("Using pip options: %s" % (str(options)))
         real_cmd += [str(options)]
-    real_cmd += [pipfull]
+    real_cmd += [name_full]
     sh.execute(*real_cmd, run_as_root=True)
 
 

devstack/progs/actions.py

@@ -189,20 +189,23 @@ class ActionRunner(object):
         subsystems = persona.get('subsystems') or dict()  # Not required
         instances = dict()
         for c in components:
-            cls = self.distro.get_component_action_class(c, action)
+            (cls, my_info) = self.distro.extract_component(c, action)
             LOG.debug("Constructing class %s" % (cls))
             cls_kvs = dict()
             cls_kvs['runner'] = self
             cls_kvs['component_dir'] = sh.joinpths(root_dir, c)
-            cls_kvs['subsystems'] = set(subsystems.get(c, list()))
+            cls_kvs['active_subsystems'] = set(subsystems.get(c, list()))
             cls_kvs['all_instances'] = instances
             cls_kvs['name'] = c
-            # FIXME: we are always sending these... (even if not used)
             cls_kvs['keep_old'] = self.keep_old
-            cls_kvs['packages'] = self.distro.get_packages(c)
+            # The above is not overrideable...
-            cls_kvs['pips'] = self.distro.get_pips(c)
+            for (k, v) in my_info.items():
-            LOG.debug("Using k/v map %s", cls_kvs)
+                if k not in cls_kvs:
-            instances[c] = cls(*list(), **cls_kvs)
+                    cls_kvs[k] = v
+            LOG.debug("Using arg map %s", cls_kvs)
+            cls_args = list()
+            LOG.debug("Using arg list %s", cls_args)
+            instances[c] = cls(*cls_args, **cls_kvs)
         return instances
     
     def _verify_components(self, component_order, instances):