Merge pull request #170 from harlowja/master

Token cleanups + starting of rootwrap + image url cleanups

conf/distros/fedora-16.yaml

@@ -39,6 +39,9 @@ commands:
         status: service mysqld status
         stop: service mysqld stop
     pip: pip-python
+    # Used to know where to find nova root wrap
+    bin_dir: "/usr/bin"
+    sudoers_dir: "/etc/sudoers.d/"
     rabbit-mq:
         change_password: rabbitmqctl change_password guest
         restart: service rabbitmq-server restart

conf/distros/rhel-6.yaml

@@ -39,6 +39,9 @@ commands:
         status: service mysqld status
         stop: service mysqld stop
     pip: pip-python
+    # Used to know where to find nova root wrap
+    bin_dir: "/usr/bin"
+    sudoers_dir: "/etc/sudoers.d/"
     rabbit-mq:
         change_password: rabbitmqctl change_password guest
         restart: service rabbitmq-server restart

conf/distros/ubuntu-oneiric.yaml

@@ -44,6 +44,9 @@ commands:
         status: service mysql status
         stop: service mysql stop
     pip: pip
+    # Used to know where to find nova root wrap
+    bin_dir: "/usr/local/bin/"
+    sudoers_dir: "/etc/sudoers.d/"
     rabbit-mq:
         start: service rabbitmq-server start
         stop: service rabbitmq-server stop

conf/stack.ini

@@ -87,6 +87,9 @@ keystone_service_protocol = ${KEYSTONE_SERVICE_PROTOCOL:-http}
 verbose = ${NOVA_VERBOSE:-1}
 logdir = ${NOVA_LOGDIR:-/var/log/nova}
 
+# Should we setup root wrap or not
+do_root_wrap = 0
+
 # Set api_rate_limit = 0 (or blank) to turn OFF rate limiting
 api_rate_limit = ${API_RATE_LIMIT:-1}
 
@@ -319,28 +322,10 @@ partition_power_size = ${SWIFT_PARTITION_POWER_SIZE:-9}
 
 [img]
 
-# Specify a comma-separated list of uec images to download and install into glance.
+# Specify a comma-separated list of images to download and install into glance.
-# supported urls here are:
+image_urls = http://launchpad.net/cirros/trunk/0.3.0/+download/cirros-0.3.0-x86_64-uec.tar.gz,
-#
+             http://uec-images.ubuntu.com/oneiric/current/oneiric-server-cloudimg-amd64.tar.gz,
-#  * "uec-style" images:
+             http://smoser.brickies.net/ubuntu/ttylinux-uec/ttylinux-uec-amd64-11.2_2.6.35-15_1.tar.gz,
-#     If the file ends in .tar.gz, uncompress the tarball and and select the first
-#     .img file inside it as the image.  If present, use "*-vmlinuz*" as the kernel
-#     and "*-initrd*" as the ramdisk
-#     example: http://cloud-images.ubuntu.com/releases/oneiric/release/ubuntu-11.10-server-cloudimg-amd64.tar.gz
-#  * disk image (*.img,*.img.gz)
-#    if file ends in .img, then it will be uploaded and registered as a to
-#    glance as a disk image.  If it ends in .gz, it is uncompressed first.
-#    example:
-#      http://cloud-images.ubuntu.com/releases/oneiric/release/ubuntu-11.10-server-cloudimg-armel-disk1.img
-#      http://launchpad.net/cirros/trunk/0.3.0/+download/cirros-0.3.0-x86_64-rootfs.img.gz
-
-# old ttylinux-uec image
-#image_urls="http://smoser.brickies.net/ubuntu/ttylinux-uec/ttylinux-uec-amd64-11.2_2.6.35-15_1.tar.gz" 
-# cirros full disk image
-#image_urls="http://launchpad.net/cirros/trunk/0.3.0/+download/cirros-0.3.0-x86_64-disk.img" 
-
-# uec style cirros 0.3.0 (x86_64) and ubuntu oneiric (x86_64)
-image_urls = http://launchpad.net/cirros/trunk/0.3.0/+download/cirros-0.3.0-x86_64-uec.tar.gz, http://uec-images.ubuntu.com/oneiric/current/oneiric-server-cloudimg-amd64.tar.gz
 
 [passwords]
 

devstack/component.py

@@ -561,8 +561,7 @@ class ProgramRuntime(ComponentBase):
             # Adjust the program options now that we have real locations
             program_opts = utils.param_replace_list(self._get_app_options(app_name), self._get_param_map(app_name))
             # Start it with the given settings
-            LOG.debug("Starting %r with options (%s) using %r",
+            LOG.debug("Starting %r using %r", app_name,  run_type)
-                     app_name, ", ".join(program_opts), run_type)
             details_fn = instance.start(app_name,
                 app_pth=app_pth, app_dir=app_dir, opts=program_opts)
             LOG.info("Started %r details are in %r", app_name, details_fn)
@@ -578,6 +577,7 @@ class ProgramRuntime(ComponentBase):
             killcls = None
             try:
                 killcls = importer.import_entry_point(how)
+                LOG.debug("Stopping %r using %r", app_name, how)
             except RuntimeError as e:
                 LOG.warn("Could not load class %r which should be used to stop %r: %s", how, app_name, e)
             if killcls in killer_instances:

devstack/components/keystone.py

@@ -176,8 +176,6 @@ class KeystoneInstaller(comp.PythonInstallComponent):
             config.set('pipeline:admin_api', 'pipeline', ('token_auth admin_token_auth xml_body '
                             'json_body debug ec2_extension s3_extension crud_extension admin_service'))
             contents = config.stringify(fn)
-        # FIXME: LP 966670 fixes this in keystone
-        sh.unlink(sh.joinpths(self.app_dir, 'etc', fn))
         return contents
 
     def _config_adjust_catalog(self, contents, fn):

devstack/components/nova.py

@@ -261,6 +261,7 @@ class NovaInstaller(NovaMixin, comp.PythonInstallComponent):
         self.volume_configurator = None
         self.volumes_enabled = NVOL in self.desired_subsystems
         self.xvnc_enabled = NXVNC in self.desired_subsystems
+        self.root_wrap_bin = sh.joinpths(self.distro.get_command_config('bin_dir'), 'nova-rootwrap')
         self.volume_maker = None
         if self.volumes_enabled:
             self.volume_maker = NovaVolumeConfigurator(self)
@@ -327,10 +328,10 @@ class NovaInstaller(NovaMixin, comp.PythonInstallComponent):
         db.drop_db(self.cfg, self.pw_gen, self.distro, DB_NAME)
         db.create_db(self.cfg, self.pw_gen, self.distro, DB_NAME)
 
-    def _generate_nova_conf(self):
+    def _generate_nova_conf(self, root_wrapped):
         conf_fn = self._get_target_config_name(API_CONF)
         LOG.info("Generating dynamic content for nova: %r" % (conf_fn))
-        nova_conf_contents = self.conf_maker.configure()
+        nova_conf_contents = self.conf_maker.configure(root_wrapped)
         self.tracewriter.dirs_made(*sh.mkdirslist(sh.dirname(conf_fn)))
         self.tracewriter.cfg_file_written(sh.write_file(conf_fn, nova_conf_contents))
 
@@ -395,9 +396,27 @@ class NovaInstaller(NovaMixin, comp.PythonInstallComponent):
             mp['FIXED_RANGE'] = self.cfg.getdefaulted('nova', 'fixed_range', '10.0.0.0/24')
         return mp
 
+    def _generate_root_wrap(self):
+        if not self.cfg.getboolean('nova', 'do_root_wrap'):
+            return False
+        else:
+            lines = list()
+            lines.append("%s ALL=(root) NOPASSWD: %s" % (sh.getuser(), self.root_wrap_bin))
+            fc = utils.joinlinesep(*lines)
+            root_wrap_fn = sh.joinpths(self.distro.get_command_config('sudoers_dir'), 'nova-rootwrap')
+            self.tracewriter.file_touched(root_wrap_fn)
+            with sh.Rooted(True):
+                sh.write_file(root_wrap_fn, fc)
+                sh.chmod(root_wrap_fn, 0440)
+                sh.chown(root_wrap_fn, sh.getuid(sh.ROOT_USER), sh.getgid(sh.ROOT_GROUP))
+            return True
+
     def configure(self):
         configs_made = comp.PythonInstallComponent.configure(self)
-        self._generate_nova_conf()
+        root_wrapped = self._generate_root_wrap()
+        if root_wrapped:
+            configs_made += 1
+        self._generate_nova_conf(root_wrapped)
         configs_made += 1
         return configs_made
 
@@ -589,7 +608,7 @@ class NovaConfConfigurator(object):
                 msg = "Libvirt flat interface %s is not a known interface" % (flat_interface)
                 raise exceptions.ConfigException(msg)
 
-    def configure(self):
+    def configure(self, root_wrapped):
         # Everything built goes in here
         nova_conf = NovaConf()
 
@@ -687,6 +706,10 @@ class NovaConfConfigurator(object):
         # Handle any virt driver specifics
         self._configure_virt_driver(nova_conf)
 
+        # Setup our root wrap helper that will limit our sudo ability
+        if root_wrapped:
+            self._configure_root_wrap(nova_conf)
+
         # Annnnnd extract to finish
         return self._get_content(nova_conf)
 
@@ -733,6 +756,9 @@ class NovaConfConfigurator(object):
             generated_content = utils.joinlinesep(*new_contents)
         return generated_content
 
+    def _configure_root_wrap(self, nova_conf):
+        nova_conf.add('root_helper', 'sudo %s' % (self.install.root_wrap_bin))
+
     def _configure_image_service(self, nova_conf, hostip):
         # What image service we will u be using sir?
         img_service = self._getstr('img_service', DEF_IMAGE_SERVICE)

devstack/image/uploader.py

@@ -338,21 +338,13 @@ class Service:
         LOG.debug("With headers %s" % (headers))
 
         response = urllib2.urlopen(request)
-
+        token = utils.get_from_path(json.loads(response.read()), "access/token/id")
-        token = json.loads(response.read())
+        if not token:
-
-        # TODO is there a better way to validate???
-        if (not token or not type(token) is dict or
-            not token.get('access') or not type(token.get('access')) is dict or
-            not token.get('access').get('token') or not type(token.get('access').get('token')) is dict or
-            not token.get('access').get('token').get('id')):
             msg = "Response from url %r did not match expected json format." % (keystone_token_url)
             raise IOError(msg)
 
-        # Basic checks passed, extract it!
+        LOG.debug("Got token %r" % (token))
-        tok = token['access']['token']['id']
+        return token
-        LOG.debug("Got token %r" % (tok))
-        return tok
 
     def install(self):
         LOG.info("Setting up any specified images in glance.")

devstack/progs/actions.py

@@ -163,17 +163,15 @@ class InstallRunner(ActionRunner):
         return False
 
     def _write_rc_file(self, root_dir):
+        fn = sh.abspth(settings.gen_rc_filename('core'))
         writer = env_rc.RcWriter(self.cfg, self.pw_gen, root_dir)
-        if not sh.isfile(settings.OSRC_FN):
+        if not sh.isfile(fn):
-            LOG.info("Generating a file at %r that will contain your environment settings.",
+            LOG.info("Generating a file at %r that will contain your environment settings.", fn)
-                     settings.OSRC_FN)
+            writer.write(fn)
-            writer.write(settings.OSRC_FN)
         else:
-            LOG.info("Updating a file at %r that contains your environment settings.",
+            LOG.info("Updating a file at %r that contains your environment settings.", fn)
-                     settings.OSRC_FN)
+            am_upd = writer.update(fn)
-            am_upd = writer.update(settings.OSRC_FN)
+            LOG.info("Updated %s settings in rc file %r", am_upd, fn)
-            LOG.info("Updated %s settings in rc file %r",
-                     am_upd, settings.OSRC_FN)
 
     def _run(self, persona, root_dir, component_order, instances):
         self._write_rc_file(root_dir)

devstack/settings.py

@@ -31,7 +31,6 @@ COMPONENT_CONFIG_DIR = "config"
 
 # RC files generated / used
 RC_FN_TEMPL = "os-%s.rc"
-OSRC_FN = RC_FN_TEMPL % ('core')
 
 # Where the configs and templates should be at.
 STACK_BIN_DIR = os.path.abspath(os.path.dirname(sys.argv[0]))
@@ -39,3 +38,7 @@ STACK_CONFIG_DIR = os.path.join(STACK_BIN_DIR, "conf")
 STACK_DISTRO_DIR = os.path.join(STACK_CONFIG_DIR, "distros")
 STACK_TEMPLATE_DIR = os.path.join(STACK_CONFIG_DIR, "templates")
 STACK_CONFIG_LOCATION = os.path.join(STACK_CONFIG_DIR, "stack.ini")
+
+
+def gen_rc_filename(root_name):
+    return RC_FN_TEMPL % (root_name)

devstack/shell.py

@@ -30,6 +30,7 @@ from devstack import log as logging
 
 LOG = logging.getLogger("devstack.shell")
 ROOT_USER = "root"
+ROOT_GROUP = ROOT_USER
 ROOT_USER_UID = 0
 SUDO_UID = 'SUDO_UID'
 SUDO_GID = 'SUDO_GID'
@@ -69,6 +70,7 @@ class Rooted(object):
 
     def __enter__(self):
         if self.root_mode and not got_root():
+            LOG.debug("Engaging root mode")
             root_mode()
             self.engaged = True
         return self.engaged
@@ -76,6 +78,7 @@ class Rooted(object):
     def __exit__(self, type, value, traceback):
         if self.root_mode and self.engaged:
             user_mode()
+            LOG.debug("Disengaging root mode")
             self.engaged = False
 
 
@@ -139,6 +142,22 @@ def execute(*cmd, **kwargs):
         LOG.audit("With additional environment overrides: %s" % (env_overrides))
         for (k, v) in env_overrides.items():
             process_env[k] = str(v)
+    else:
+        process_env = env.get()
+
+    LOG.debug("With environment %s", process_env)
+    demoter = None
+    def demoter_functor(user_uid, user_gid):
+        def doit():
+            os.setregid(user_gid, user_gid)
+            os.setreuid(user_uid, user_uid)
+        return doit
+
+    if not run_as_root:
+        (user_uid, user_gid) = get_suids()
+        if user_uid and user_gid:
+            LOG.debug("Not running as root, we will run with real & effective gid:uid --> %s:%s", user_gid, user_uid)
+            demoter = demoter_functor(user_uid=user_uid, user_gid=user_gid)
 
     rc = None
     result = None
@@ -155,6 +174,7 @@ def execute(*cmd, **kwargs):
                                        close_fds=close_file_descriptors,
                                        cwd=cwd,
                                        shell=shell,
+                                       preexec_fn=demoter,
                                        env=process_env)
                 if process_input is not None:
                     result = obj.communicate(str(process_input))
@@ -242,7 +262,7 @@ def joinpths(*paths):
     return os.path.join(*paths)
 
 
-def _get_suids():
+def get_suids():
     uid = env.get_key(SUDO_UID)
     if uid is not None:
         uid = int(uid)
@@ -252,21 +272,27 @@ def _get_suids():
     return (uid, gid)
 
 
+def chown(path, uid, gid, run_as_root=True):
+    LOG.audit("Changing ownership of %r to %s:%s" % (path, uid, gid))
+    with Rooted(run_as_root):
+        if not DRYRUN_MODE:
+            os.chown(path, uid, gid)
+    return 1
+
+
 def chown_r(path, uid, gid, run_as_root=True):
+    changed = 0
     with Rooted(run_as_root):
         if isdir(path):
-            LOG.audit("Changing ownership of %r to %s:%s" % (path, uid, gid))
             for root, dirs, files in os.walk(path):
-                os.chown(root, uid, gid)
+                changed += chown(root, uid, gid)
-                LOG.audit("Changing ownership of %r to %s:%s" % (root, uid, gid))
                 for d in dirs:
                     dir_pth = joinpths(root, d)
-                    os.chown(dir_pth, uid, gid)
+                    changed += chown(dir_pth, uid, gid)
-                    LOG.audit("Changing ownership of %r to %s:%s" % (dir_pth, uid, gid))
                 for f in files:
                     fn_pth = joinpths(root, f)
-                    os.chown(fn_pth, uid, gid)
+                    changed += chown(fn_pth, uid, gid)
-                    LOG.audit("Changing ownership of %r to %s:%s" % (fn_pth, uid, gid))
+    return changed
 
 
 def _explode_path(path):
@@ -476,7 +502,7 @@ def group_exists(grpname):
 
 
 def getuser():
-    (uid, _) = _get_suids()
+    (uid, gid) = get_suids()
     if uid is None:
         return getpass.getuser()
     return pwd.getpwuid(uid).pw_name
@@ -486,8 +512,12 @@ def getuid(username):
     return pwd.getpwnam(username).pw_uid
 
 
-def gethomedir():
+def gethomedir(user=None):
-    return os.path.expanduser("~")
+    if not user:
+        user = getuser()
+    home_dir = os.path.expanduser("~%s" % (user))
+    LOG.audit("Fetching homedir of %r => %r", user, home_dir)
+    return home_dir
 
 
 def getgid(groupname):
@@ -495,10 +525,9 @@ def getgid(groupname):
 
 
 def getgroupname():
-    (_, gid) = _get_suids()
+    (uid, gid) = get_suids()
     if gid is None:
-        return os.getgid()
+        gid = os.getgid()
-    else:
     return grp.getgrgid(gid).gr_name
 
 
@@ -621,7 +650,7 @@ def root_mode(quiet=True):
 
 
 def user_mode(quiet=True):
-    (sudo_uid, sudo_gid) = _get_suids()
+    (sudo_uid, sudo_gid) = get_suids()
     if sudo_uid is not None and sudo_gid is not None:
         try:
             LOG.debug("Dropping permissions to (user=%s, group=%s)" % (sudo_uid, sudo_gid))

devstack/utils.py

@@ -94,6 +94,41 @@ def make_bool(val):
     return False
 
 
+def get_from_path(items, path, quiet=True):
+
+    (first_token, sep, remainder) = path.partition('/')
+
+    if len(path) == 0:
+        return items
+
+    LOG.debug("Looking up %r in %s" % (path, items))
+
+    if len(first_token) == 0:
+        if not quiet:
+            raise RuntimeError("Invalid first token found in %s" % (path))
+        else:
+            return None
+
+    if isinstance(items, list):
+        index = int(first_token)
+        ok_use = (index < len(items) and index >= 0)
+        if quiet and not ok_use:
+            return None
+        else:
+            LOG.debug("Looking up index %s in list %s" % (index, items))
+            return get_from_path(items[index], remainder)
+    else:
+        get_method = getattr(items, 'get', None)
+        if not get_method:
+            if not quiet:
+                raise RuntimeError("Can not figure out how to extract an item from %s" % (items))
+            else:
+                return None
+        else:
+            LOG.debug("Looking up %r in object %s with method %s" % (first_token, items, get_method))
+            return get_from_path(get_method(first_token), remainder)
+
+
 def configure_logging(log_level, cli_args):
     root_logger = logging.getLogger().logger
     console_logger = logging.StreamHandler(sys.stdout)

stack

@@ -83,7 +83,8 @@ def dump_config(config_cache):
 
 
 def load_rc_files():
-    fns = [settings.OSRC_FN]
+    fns = list()
+    fns.append(sh.abspth(settings.gen_rc_filename('core')))
     for fn in fns:
         try:
             LOG.info("Attempting to load file %r which has your environment settings." % (fn))
@@ -199,7 +200,7 @@ def main():
 
     try:
         # Drop to usermode
-        sh.user_mode(False)
+        sh.user_mode(quiet=False)
         started_ok = run(args)
         if not started_ok:
             me = utils.color_text(prog_name, "red", True)