anvil/devstack/shell.py

# vim: tabstop=4 shiftwidth=4 softtabstop=4

#    Copyright (C) 2012 Yahoo! Inc. All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import fileinput
import getpass
import grp
import os
import pwd
import shutil
import subprocess
import sys
import time

from devstack import env
from devstack import exceptions as excp
from devstack import log as logging

LOG = logging.getLogger("devstack.shell")
ROOT_USER = "root"
ROOT_USER_UID = 0
SUDO_UID = 'SUDO_UID'
SUDO_GID = 'SUDO_GID'
SHELL_QUOTE_REPLACERS = {
    "\"": "\\\"",
    "(": "\\(",
    ")": "\\)",
    "$": '\$',
    '`': '\`',
}
SHELL_WRAPPER = "\"%s\""
ROOT_PATH = os.sep
DRYRUN_MODE = False
DRY_RC = 0
DRY_STDOUT_ERR = ("", "")


def set_dryrun(val):
    global DRYRUN_MODE
    if val:
        LOG.debug("Setting dryrun to: %s" % (True))
        DRYRUN_MODE = True
    else:
        LOG.debug("Resetting dryrun to: %s" % (False))
        DRYRUN_MODE = False


#root context guard
class Rooted(object):
    def __init__(self, run_as_root):
        self.root_mode = run_as_root
        self.engaged = False

    def __enter__(self):
        if self.root_mode and not got_root():
            root_mode()
            self.engaged = True
        return self.engaged

    def __exit__(self, type, value, traceback):
        if self.root_mode and self.engaged:
            user_mode()
            self.engaged = False


def execute(*cmd, **kwargs):
    process_input = kwargs.pop('process_input', None)
    check_exit_code = kwargs.pop('check_exit_code', [0])
    cwd = kwargs.pop('cwd', None)
    env_overrides = kwargs.pop('env_overrides', None)
    close_stdin = kwargs.pop('close_stdin', False)
    ignore_exit_code = kwargs.pop('ignore_exit_code', False)

    if isinstance(check_exit_code, bool):
        ignore_exit_code = not check_exit_code
        check_exit_code = [0]
    elif isinstance(check_exit_code, int):
        check_exit_code = [check_exit_code]

    run_as_root = kwargs.pop('run_as_root', False)
    shell = kwargs.pop('shell', False)

    execute_cmd = list()
    for c in cmd:
        execute_cmd.append(str(c))

    str_cmd = " ".join(execute_cmd)
    if shell:
        execute_cmd = str_cmd.strip()

    if not shell:
        LOG.audit('Running cmd: %s' % (execute_cmd))
    else:
        LOG.audit('Running shell cmd: %s' % (execute_cmd))

    if process_input is not None:
        LOG.audit('With stdin: %s' % (process_input))

    if cwd:
        LOG.audit("In working directory: %s" % (cwd))

    stdin_fh = subprocess.PIPE
    stdout_fh = subprocess.PIPE
    stderr_fh = subprocess.PIPE
    close_file_descriptors = True

    if 'stdout_fh' in kwargs.keys():
        stdout_fh = kwargs.get('stdout_fh')
        LOG.debug("Redirecting stdout to file handle: %s" % (stdout_fh))

    if 'stdin_fh' in kwargs.keys():
        stdin_fh = kwargs.get('stdin_fh')
        LOG.debug("Redirecting stdin to file handle: %s" % (stdin_fh))
        process_input = None

    if 'stderr_fh' in kwargs.keys():
        stderr_fh = kwargs.get('stderr_fh')
        LOG.debug("Redirecting stderr to file handle: %s" % (stderr_fh))

    process_env = None
    if env_overrides and len(env_overrides):
        process_env = env.get()
        LOG.audit("With additional environment overrides: %s" % (env_overrides))
        for (k, v) in env_overrides.items():
            process_env[k] = str(v)

    rc = None
    result = None
    with Rooted(run_as_root):
        if DRYRUN_MODE:
            rc = DRY_RC
            result = DRY_STDOUT_ERR
        else:
            try:
                obj = subprocess.Popen(execute_cmd,
                                       stdin=stdin_fh,
                                       stdout=stdout_fh,
                                       stderr=stderr_fh,
                                       close_fds=close_file_descriptors,
                                       cwd=cwd,
                                       shell=shell,
                                       env=process_env)
                if process_input is not None:
                    result = obj.communicate(str(process_input))
                else:
                    result = obj.communicate()
            except OSError as e:
                error_description = "%s: [%s, %s]" % (e.message, e.errno, e.strerror)
                raise excp.ProcessExecutionError(description=error_description, cmd=str_cmd)
            if (stdin_fh != subprocess.PIPE
                and obj.stdin and close_stdin):
                obj.stdin.close()
            rc = obj.returncode
        LOG.audit('Cmd result had exit code: %s' % rc)

    if not result:
        result = ("", "")

    (stdout, stderr) = result
    if stdout is None:
        stdout = ''
    if stderr is None:
        stderr = ''

    if (not ignore_exit_code) and (rc not in check_exit_code):
        raise excp.ProcessExecutionError(exit_code=rc, stdout=stdout, \
                                         stderr=stderr, cmd=str_cmd)
    else:
        #log it anyway
        if rc not in check_exit_code:
            LOG.debug("A failure may of just happened when running command \"%s\" [%s] (%s, %s)", \
                str_cmd, rc, stdout.strip(), stderr.strip())
        #log for debugging figuring stuff out
        LOG.debug("Received stdout: %s" % (stdout.strip()))
        LOG.debug("Received stderr: %s" % (stderr.strip()))
        return (stdout, stderr)


def abspth(path):
    if not path:
        path = ROOT_PATH
    if path == "~":
        path = gethomedir()
    return os.path.abspath(path)


def shellquote(text):
    #TODO since there doesn't seem to be a standard lib that actually works use this way...
    do_adjust = False
    for srch in SHELL_QUOTE_REPLACERS.keys():
        if text.find(srch) != -1:
            do_adjust = True
            break
    if do_adjust:
        for (srch, replace) in SHELL_QUOTE_REPLACERS.items():
            text = text.replace(srch, replace)
    if do_adjust or \
        text.startswith((" ", "\t")) or \
        text.endswith((" ", "\t")) or \
        text.find("'") != -1:
        text = SHELL_WRAPPER % (text)
    return text


def listdir(path):
    return os.listdir(path)


def isfile(fn):
    return os.path.isfile(fn)


def isdir(path):
    return os.path.isdir(path)


def islink(path):
    return os.path.islink(path)


def joinpths(*paths):
    return os.path.join(*paths)


def _get_suids():
    uid = env.get_key(SUDO_UID)
    if uid is not None:
        uid = int(uid)
    gid = env.get_key(SUDO_GID)
    if gid is not None:
        gid = int(gid)
    return (uid, gid)


def chown_r(path, uid, gid, run_as_root=True):
    with Rooted(run_as_root):
        if isdir(path):
            LOG.audit("Changing ownership of %s to %s:%s" % (path, uid, gid))
            for root, dirs, files in os.walk(path):
                os.chown(root, uid, gid)
                LOG.audit("Changing ownership of %s to %s:%s" % (root, uid, gid))
                for d in dirs:
                    os.chown(joinpths(root, d), uid, gid)
                    LOG.audit("Changing ownership of %s to %s:%s" % (joinpths(root, d), uid, gid))
                for f in files:
                    os.chown(joinpths(root, f), uid, gid)
                    LOG.audit("Changing ownership of %s to %s:%s" % (joinpths(root, f), uid, gid))


def _explode_path(path):
    parts = list()
    path = abspth(path)
    while path != ROOT_PATH and path:
        (path, name) = os.path.split(path)
        parts.append(name)
    parts.reverse()
    return parts


def _explode_form_path(path):
    ret_paths = list()
    ret_paths.append(ROOT_PATH)
    ex_path = _explode_path(path)
    for i in range(len(ex_path)):
        to_make = [ROOT_PATH] + ex_path[0:i] + [ex_path[i]]
        ret_paths.append(joinpths(*to_make))
    return ret_paths


def in_terminal(check_both=False):
    if check_both:
        return sys.stdout.isatty() and sys.stderr.isatty()
    else:
        return sys.stdout.isatty()


def remove_parents(child_path, paths):
    if not paths:
        return list()
    cleaned_paths = [abspth(p) for p in paths]
    cleaned_child_path = abspth(child_path)
    LOG.audit("Removing parents of [%s] from input [%s]" % (cleaned_child_path, ",".join(cleaned_paths)))
    to_check_paths = [_explode_path(p) for p in cleaned_paths]
    check_path = _explode_path(cleaned_child_path)
    new_paths = list()
    for p in to_check_paths:
        if _array_begins_with(p, check_path):
            pass
        else:
            new_paths.append(p)
    ret_paths = list()
    for p in new_paths:
        ret_paths.append(abspth(os.sep + os.sep.join(p)))
    LOG.debug("Removal resulted in [%s]", ",".join(ret_paths))
    return ret_paths


def _array_begins_with(haystack, needle):
    if len(haystack) > len(needle):
        return False
    for i in range(len(haystack)):
        if haystack[i] != needle[i]:
            return False
    return True


def mkdirslist(path):
    LOG.debug("Determining potential paths to create for target path \"%s\"" % (path))
    dirs_possible = _explode_form_path(path)
    dirs_made = list()
    for check_path in dirs_possible:
        if not isdir(check_path):
            mkdir(check_path, False)
            dirs_made.append(check_path)
    return dirs_made


def append_file(fn, text, flush=True, quiet=False):
    if not quiet:
        LOG.audit("Appending to file %s (%d bytes) (flush=%s)", fn, len(text), flush)
        LOG.audit(">> %s" % (text))
    if not DRYRUN_MODE:
        with open(fn, "a") as f:
            f.write(text)
            if flush:
                f.flush()
    return fn


def write_file(fn, text, flush=True, quiet=False):
    if not quiet:
        LOG.audit("Writing to file %s (%d bytes) (flush=%s)", fn, len(text), flush)
        LOG.audit("> %s" % (text))
    if not DRYRUN_MODE:
        with open(fn, "w") as f:
            f.write(text)
            if flush:
                f.flush()
    return fn


def touch_file(fn, die_if_there=True, quiet=False, file_size=0):
    if not isfile(fn):
        if not quiet:
            LOG.audit("Touching and truncating file %s (truncate size=%s)", fn, file_size)
        if not DRYRUN_MODE:
            with open(fn, "w") as f:
                f.truncate(file_size)
    else:
        if die_if_there:
            msg = "Can not touch & truncate file %s since it already exists" % (fn)
            raise excp.FileException(msg)
    return fn


def load_file(fn, quiet=False):
    if not quiet:
        LOG.audit("Loading data from file %s", fn)
    data = ""
    try:
        with open(fn, "r") as f:
            data = f.read()
    except IOError as e:
        if DRYRUN_MODE:
            # We still need to actually load something (ie the json install files so thats)
            # Why this is in the exception path.
            LOG.audit("Passing on load exception since in dry-run mode")
        else:
            raise e
    if not quiet:
        LOG.audit("Loaded (%d) bytes from file %s", len(data), fn)
    return data


def mkdir(path, recurse=True):
    if not isdir(path):
        if recurse:
            LOG.audit("Recursively creating directory \"%s\"" % (path))
            if not DRYRUN_MODE:
                os.makedirs(path)
        else:
            LOG.audit("Creating directory \"%s\"" % (path))
            if not DRYRUN_MODE:
                os.mkdir(path)


def deldir(path, run_as_root=False):
    with Rooted(run_as_root):
        if isdir(path):
            LOG.audit("Recursively deleting directory tree starting at \"%s\"" % (path))
            if not DRYRUN_MODE:
                shutil.rmtree(path)


def rmdir(path, quiet=True, run_as_root=False):
    if not isdir(path):
        return
    try:
        with Rooted(run_as_root):
            LOG.audit("Deleting directory \"%s\" with the cavet that we will fail if it's not empty." % (path))
            if not DRYRUN_MODE:
                os.rmdir(path)
            LOG.audit("Deleted directory \"%s\"" % (path))
    except OSError:
        if not quiet:
            raise
        else:
            pass


def symlink(source, link, force=True, run_as_root=True):
    with Rooted(run_as_root):
        LOG.audit("Creating symlink from %s => %s" % (link, source))
        path = dirname(link)
        needed_pths = mkdirslist(path)
        if not DRYRUN_MODE:
            if force and (exists(link) or islink(link)):
                unlink(link, True)
            os.symlink(source, link)
        return needed_pths


def exists(path):
    return os.path.exists(path)


def basename(path):
    return os.path.basename(path)


def dirname(path):
    return os.path.dirname(path)


def canon_path(path):
    return os.path.realpath(path)


def prompt(prompt_str):
    return raw_input(prompt_str)


def user_exists(username):
    all_users = pwd.getpwall()
    for info in all_users:
        if info.pw_name == username:
            return True
    return False


def group_exists(grpname):
    all_grps = grp.getgrall()
    for info in all_grps:
        if info.gr_name == grpname:
            return True
    return False


def getuser():
    (uid, _) = _get_suids()
    if uid is None:
        return getpass.getuser()
    return pwd.getpwuid(uid).pw_name


def getuid(username):
    return pwd.getpwnam(username).pw_uid


def gethomedir():
    return os.path.expanduser("~")


def getgid(groupname):
    return grp.getgrnam(groupname).gr_gid


def getgroupname():
    (_, gid) = _get_suids()
    if gid is None:
        return os.getgid()
    else:
        return grp.getgrgid(gid).gr_name


def create_loopback_file(fname, size, bsize=1024, fs_type='ext3', run_as_root=False):
    dd_cmd = ['dd', 'if=/dev/zero', 'of=%s' % fname, 'bs=%d' % bsize,
              'count=0', 'seek=%d' % size]
    mkfs_cmd = ['mkfs.%s' % fs_type, '-f', '-i', 'size=%d' % bsize, fname]

    # make sure folder exists
    files = mkdirslist(dirname(fname))

    # create file
    touch_file(fname)

    # fill with zeroes
    execute(*dd_cmd, run_as_root=run_as_root)

    # create fs on the file
    execute(*mkfs_cmd, run_as_root=run_as_root)

    return files


def mount_loopback_file(fname, device_name, fs_type='ext3'):
    mount_cmd = ['mount', '-t', fs_type, '-o',
                 'loop,noatime,nodiratime,nobarrier,logbufs=8', fname,
                 device_name]

    files = mkdirslist(device_name)

    execute(*mount_cmd, run_as_root=True)

    return files


def umount(dev_name, ignore_errors=True):
    try:
        execute('umount', dev_name, run_as_root=True)
    except excp.ProcessExecutionError:
        if not ignore_errors:
            raise
        else:
            pass


def unlink(path, ignore_errors=True, run_as_root=False):
    LOG.audit("Unlinking (removing) %s" % (path))
    if not DRYRUN_MODE:
        try:
            with Rooted(run_as_root):
                os.unlink(path)
        except OSError:
            if not ignore_errors:
                raise
            else:
                pass


def move(src, dst):
    LOG.audit("Moving: %s => %s" % (src, dst))
    if not DRYRUN_MODE:
        shutil.move(src, dst)
    return dst


def chmod(fname, mode):
    LOG.audit("Applying chmod: %s to %o" % (fname, mode))
    if not DRYRUN_MODE:
        os.chmod(fname, mode)
    return fname


def replace_in(fn, search, replace, run_as_root=False):
    with Rooted(run_as_root):
        contents = load_file(fn)

        def replacer(match):
            return replace

        (contents, num_changed) = search.subn(replacer, contents)
        if num_changed:
            write_file(fn, contents)


def copy_replace_file(fsrc, fdst, linemap):
    files = mkdirslist(dirname(fdst))
    LOG.audit("Copying and replacing file: %s => %s" % (fsrc, fdst))
    if not DRYRUN_MODE:
        with open(fdst, 'w') as fh:
            for line in fileinput.input(fsrc):
                for (k, v) in linemap.items():
                    line = line.replace(k, v)
                fh.write(line)
    return files


def got_root():
    return os.geteuid() == ROOT_USER_UID


def root_mode(quiet=True):
    root_uid = getuid(ROOT_USER)
    root_gid = getgid(ROOT_USER)
    if root_uid is None or root_gid is None:
        msg = "Cannot escalate permissions to (user=%s) - does that user exist??" % (ROOT_USER)
        if quiet:
            LOG.warn(msg)
        else:
            raise excp.StackException(msg)
    else:
        try:
            LOG.debug("Escalating permissions to (user=%s, group=%s)" % (root_uid, root_gid))
            os.setreuid(0, root_uid)
            os.setregid(0, root_gid)
        except OSError:
            if quiet:
                LOG.warn("Cannot escalate permissions to (user=%s, group=%s)" % (root_uid, root_gid))
            else:
                raise


def user_mode(quiet=True):
    (sudo_uid, sudo_gid) = _get_suids()
    if sudo_uid is not None and sudo_gid is not None:
        try:
            LOG.debug("Dropping permissions to (user=%s, group=%s)" % (sudo_uid, sudo_gid))
            os.setregid(0, sudo_gid)
            os.setreuid(0, sudo_uid)
        except OSError:
            if quiet:
                LOG.warn("Cannot drop permissions to (user=%s, group=%s)" % (sudo_uid, sudo_gid))
            else:
                raise
    else:
        msg = "Can not switch to user mode, no suid user id or group id"
        if quiet:
            LOG.warn(msg)
        else:
            raise excp.StackException(msg)


def geteuid():
    return os.geteuid()


def getegid():
    return os.getegid()


def sleep(winks):
    if DRYRUN_MODE:
        LOG.audit("Not really sleeping for: %s seconds" % (winks))
    else:
        time.sleep(winks)