devstack/functions

#!/bin/bash
#
# functions - DevStack-specific functions
#
# The following variables are assumed to be defined by certain functions:
#
# - ``DATABASE_BACKENDS``
# - ``ENABLED_SERVICES``
# - ``FILES``
# - ``GLANCE_HOSTPORT``
#

# ensure we don't re-source this in the same environment
[[ -z "$_DEVSTACK_FUNCTIONS" ]] || return 0
declare -r -g _DEVSTACK_FUNCTIONS=1

# Include the common functions
FUNC_DIR=$(cd $(dirname "${BASH_SOURCE:-$0}") && pwd)
source ${FUNC_DIR}/functions-common
source ${FUNC_DIR}/inc/ini-config
source ${FUNC_DIR}/inc/meta-config
source ${FUNC_DIR}/inc/python
source ${FUNC_DIR}/inc/rootwrap

# Save trace setting
_XTRACE_FUNCTIONS=$(set +o | grep xtrace)
set +o xtrace

# Check if a function already exists
function function_exists {
    declare -f -F $1 > /dev/null
}

# short_source prints out the current location of the caller in a way
# that strips redundant directories. This is useful for PS4 usage.
function short_source {
    saveIFS=$IFS
    IFS=" "
    called=($(caller 0))
    IFS=$saveIFS
    file=${called[2]}
    file=${file#$RC_DIR/}
    printf "%-40s " "$file:${called[1]}:${called[0]}"
}
# PS4 is exported to child shells and uses the 'short_source' function, so
# export it so child shells have access to the 'short_source' function also.
export -f short_source

# Download a file from a URL
#
# Will check cache (in $FILES) or download given URL.
#
# Argument is the URL to the remote file
#
# Will echo the local path to the file as the output.  Will die on
# failure to download.
#
# Files can be pre-cached for CI environments, see EXTRA_CACHE_URLS
# and tools/image_list.sh
function get_extra_file {
    local file_url=$1

    file_name=$(basename "$file_url")
    if [[ $file_url != file* ]]; then
        # If the file isn't cache, download it
        if [[ ! -f $FILES/$file_name ]]; then
            wget --progress=dot:giga -t 2 -c $file_url -O $FILES/$file_name
            if [[ $? -ne 0 ]]; then
                die "$file_url could not be downloaded"
            fi
        fi
        echo "$FILES/$file_name"
        return
    else
        # just strip the file:// bit and that's the path to the file
        echo $file_url | sed 's/$file:\/\///g'
    fi
}


# Retrieve an image from a URL and upload into Glance.
# Uses the following variables:
#
# - ``FILES`` must be set to the cache dir
# - ``GLANCE_HOSTPORT``
#
# upload_image image-url
function upload_image {
    local image_url=$1

    local image image_fname image_name

    # Create a directory for the downloaded image tarballs.
    mkdir -p $FILES/images
    image_fname=`basename "$image_url"`
    if [[ $image_url != file* ]]; then
        # Downloads the image (uec ami+akistyle), then extracts it.
        if [[ ! -f $FILES/$image_fname || "$(stat -c "%s" $FILES/$image_fname)" = "0" ]]; then
            wget --progress=dot:giga -c $image_url -O $FILES/$image_fname
            if [[ $? -ne 0 ]]; then
                echo "Not found: $image_url"
                return
            fi
        fi
        image="$FILES/${image_fname}"
    else
        # File based URL (RFC 1738): ``file://host/path``
        # Remote files are not considered here.
        # unix: ``file:///home/user/path/file``
        # windows: ``file:///C:/Documents%20and%20Settings/user/path/file``
        image=$(echo $image_url | sed "s/^file:\/\///g")
        if [[ ! -f $image || "$(stat -c "%s" $image)" == "0" ]]; then
            echo "Not found: $image_url"
            return
        fi
    fi

    # OpenVZ-format images are provided as .tar.gz, but not decompressed prior to loading
    if [[ "$image_url" =~ 'openvz' ]]; then
        image_name="${image_fname%.tar.gz}"
        openstack --os-cloud=devstack-admin --os-region-name="$REGION_NAME" image create "$image_name" --public --container-format ami --disk-format ami < "${image}"
        return
    fi

    # vmdk format images
    if [[ "$image_url" =~ '.vmdk' ]]; then
        image_name="${image_fname%.vmdk}"

        # Before we can upload vmdk type images to glance, we need to know it's
        # disk type, storage adapter, and networking adapter. These values are
        # passed to glance as custom properties.
        # We take these values from the vmdk file if populated. Otherwise, we use
        # vmdk filename, which is expected in the following format:
        #
        #     <name>-<disk type>;<storage adapter>;<network adapter>
        #
        # If the filename does not follow the above format then the vsphere
        # driver will supply default values.

        local vmdk_disktype=""
        local vmdk_net_adapter="e1000"
        local path_len

        # vmdk adapter type
        local vmdk_adapter_type
        vmdk_adapter_type="$(head -25 $image | { grep -a -F -m 1 'ddb.adapterType =' $image || true; })"
        vmdk_adapter_type="${vmdk_adapter_type#*\"}"
        vmdk_adapter_type="${vmdk_adapter_type%?}"

        # vmdk disk type
        local vmdk_create_type
        vmdk_create_type="$(head -25 $image | { grep -a -F -m 1 'createType=' $image || true; })"
        vmdk_create_type="${vmdk_create_type#*\"}"
        vmdk_create_type="${vmdk_create_type%\"*}"

        descriptor_data_pair_msg="Monolithic flat and VMFS disks "`
                                    `"should use a descriptor-data pair."
        if [[ "$vmdk_create_type" = "monolithicSparse" ]]; then
            vmdk_disktype="sparse"
        elif [[ "$vmdk_create_type" = "monolithicFlat" || "$vmdk_create_type" = "vmfs" ]]; then
            # Attempt to retrieve the ``*-flat.vmdk``
            local flat_fname
            flat_fname="$(head -25 $image | { grep -G 'RW\|RDONLY [0-9]+ FLAT\|VMFS' $image || true; })"
            flat_fname="${flat_fname#*\"}"
            flat_fname="${flat_fname%?}"
            if [[ -z "$flat_fname" ]]; then
                flat_fname="$image_name-flat.vmdk"
            fi
            path_len=`expr ${#image_url} - ${#image_fname}`
            local flat_url="${image_url:0:$path_len}$flat_fname"
            warn $LINENO "$descriptor_data_pair_msg"`
                            `" Attempt to retrieve the *-flat.vmdk: $flat_url"
            if [[ $flat_url != file* ]]; then
                if [[ ! -f $FILES/$flat_fname || \
                "$(stat -c "%s" $FILES/$flat_fname)" = "0" ]]; then
                    wget --progress=dot:giga -c $flat_url -O $FILES/$flat_fname
                fi
                image="$FILES/${flat_fname}"
            else
                image=$(echo $flat_url | sed "s/^file:\/\///g")
                if [[ ! -f $image || "$(stat -c "%s" $image)" == "0" ]]; then
                    echo "Flat disk not found: $flat_url"
                    return 1
                fi
            fi
            image_name="${flat_fname}"
            vmdk_disktype="preallocated"
        elif [[ "$vmdk_create_type" = "streamOptimized" ]]; then
            vmdk_disktype="streamOptimized"
        elif [[ -z "$vmdk_create_type" ]]; then
            # *-flat.vmdk provided: attempt to retrieve the descriptor (*.vmdk)
            # to retrieve appropriate metadata
            if [[ ${image_name: -5} != "-flat" ]]; then
                warn $LINENO "Expected filename suffix: '-flat'."`
                            `" Filename provided: ${image_name}"
            else
                descriptor_fname="${image_name:0:${#image_name} - 5}.vmdk"
                path_len=`expr ${#image_url} - ${#image_fname}`
                local flat_path="${image_url:0:$path_len}"
                local descriptor_url=$flat_path$descriptor_fname
                warn $LINENO "$descriptor_data_pair_msg"`
                                `" Attempt to retrieve the descriptor *.vmdk: $descriptor_url"
                if [[ $flat_path != file* ]]; then
                    if [[ ! -f $FILES/$descriptor_fname || \
                    "$(stat -c "%s" $FILES/$descriptor_fname)" = "0" ]]; then
                        wget -c $descriptor_url -O $FILES/$descriptor_fname
                    fi
                    descriptor_url="$FILES/$descriptor_fname"
                else
                    descriptor_url=$(echo $descriptor_url | sed "s/^file:\/\///g")
                    if [[ ! -f $descriptor_url || \
                    "$(stat -c "%s" $descriptor_url)" == "0" ]]; then
                        echo "Descriptor not found: $descriptor_url"
                        return 1
                    fi
                fi
                vmdk_adapter_type="$(head -25 $descriptor_url | { grep -a -F -m 1 'ddb.adapterType =' $descriptor_url || true; })"
                vmdk_adapter_type="${vmdk_adapter_type#*\"}"
                vmdk_adapter_type="${vmdk_adapter_type%?}"
            fi
            vmdk_disktype="preallocated"
        else
            vmdk_disktype="preallocated"
        fi

        # NOTE: For backwards compatibility reasons, colons may be used in place
        # of semi-colons for property delimiters but they are not permitted
        # characters in NTFS filesystems.
        property_string=`echo "$image_name" | { grep -oP '(?<=-)(?!.*-).*[:;].*[:;].*$' || true; }`
        IFS=':;' read -a props <<< "$property_string"
        vmdk_disktype="${props[0]:-$vmdk_disktype}"
        vmdk_adapter_type="${props[1]:-$vmdk_adapter_type}"
        vmdk_net_adapter="${props[2]:-$vmdk_net_adapter}"

        openstack --os-cloud=devstack-admin --os-region-name="$REGION_NAME" image create "$image_name" --public --container-format bare --disk-format vmdk --property vmware_disktype="$vmdk_disktype" --property vmware_adaptertype="$vmdk_adapter_type" --property hw_vif_model="$vmdk_net_adapter" < "${image}"
        return
    fi

    # XenServer-vhd-ovf-format images are provided as .vhd.tgz
    # and should not be decompressed prior to loading
    if [[ "$image_url" =~ '.vhd.tgz' ]]; then
        image_name="${image_fname%.vhd.tgz}"
        local force_vm_mode=""
        if [[ "$image_name" =~ 'cirros' ]]; then
            # Cirros VHD image currently only boots in PV mode.
            # Nova defaults to PV for all VHD images, but
            # the glance setting is needed for booting
            # directly from volume.
            force_vm_mode="--property vm_mode=xen"
        fi
        openstack \
            --os-cloud=devstack-admin --os-region-name="$REGION_NAME" \
            image create \
            "$image_name" --public \
            --container-format=ovf --disk-format=vhd \
            $force_vm_mode < "${image}"
        return
    fi

    # .xen-raw.tgz suggests a Xen capable raw image inside a tgz.
    # and should not be decompressed prior to loading.
    # Setting metadata, so PV mode is used.
    if [[ "$image_url" =~ '.xen-raw.tgz' ]]; then
        image_name="${image_fname%.xen-raw.tgz}"
        openstack \
            --os-cloud=devstack-admin --os-region-name="$REGION_NAME" \
            image create \
            "$image_name" --public \
            --container-format=tgz --disk-format=raw \
            --property vm_mode=xen < "${image}"
        return
    fi

    if [[ "$image_url" =~ '.hds' ]]; then
        image_name="${image_fname%.hds}"
        vm_mode=${image_name##*-}
        if [[ $vm_mode != 'exe' && $vm_mode != 'hvm' ]]; then
            die $LINENO "Unknown vm_mode=${vm_mode} for Virtuozzo image"
        fi

        openstack \
            --os-cloud=devstack-admin --os-region-name="$REGION_NAME" \
            image create \
            "$image_name" --public \
            --container-format=bare --disk-format=ploop \
            --property vm_mode=$vm_mode < "${image}"
        return
    fi

    local kernel=""
    local ramdisk=""
    local disk_format=""
    local container_format=""
    local unpack=""
    local img_property="--property hw_rng_model=virtio"
    case "$image_fname" in
        *.tar.gz|*.tgz)
            # Extract ami and aki files
            [ "${image_fname%.tar.gz}" != "$image_fname" ] &&
                image_name="${image_fname%.tar.gz}" ||
                image_name="${image_fname%.tgz}"
            local xdir="$FILES/images/$image_name"
            rm -Rf "$xdir";
            mkdir "$xdir"
            tar -zxf $image -C "$xdir"
            kernel=$(for f in "$xdir/"*-vmlinuz* "$xdir/"aki-*/image; do
                [ -f "$f" ] && echo "$f" && break; done; true)
            ramdisk=$(for f in "$xdir/"*-initrd* "$xdir/"ari-*/image; do
                [ -f "$f" ] && echo "$f" && break; done; true)
            image=$(for f in "$xdir/"*.img "$xdir/"ami-*/image; do
                [ -f "$f" ] && echo "$f" && break; done; true)
            if [[ -z "$image_name" ]]; then
                image_name=$(basename "$image" ".img")
            fi
            ;;
        *.img)
            image_name=$(basename "$image" ".img")
            local format
            format=$(qemu-img info ${image} | awk '/^file format/ { print $3; exit }')
            if [[ ",qcow2,raw,vdi,vmdk,vpc," =~ ",$format," ]]; then
                disk_format=$format
            else
                disk_format=raw
            fi
            container_format=bare
            ;;
        *.img.gz)
            image_name=$(basename "$image" ".img.gz")
            disk_format=raw
            container_format=bare
            unpack=zcat
            ;;
        *.img.bz2)
            image_name=$(basename "$image" ".img.bz2")
            disk_format=qcow2
            container_format=bare
            unpack=bunzip2
            ;;
        *.qcow2)
            image_name=$(basename "$image" ".qcow2")
            disk_format=qcow2
            container_format=bare
            ;;
        *.qcow2.xz)
            image_name=$(basename "$image" ".qcow2.xz")
            disk_format=qcow2
            container_format=bare
            unpack=unxz
            ;;
        *.raw)
            image_name=$(basename "$image" ".raw")
            disk_format=raw
            container_format=bare
            ;;
        *.iso)
            image_name=$(basename "$image" ".iso")
            disk_format=iso
            container_format=bare
            ;;
        *.vhd|*.vhdx|*.vhd.gz|*.vhdx.gz)
            local extension="${image_fname#*.}"
            image_name=$(basename "$image" ".$extension")
            disk_format=$(echo $image_fname | grep -oP '(?<=\.)vhdx?(?=\.|$)')
            container_format=bare
            if [ "${image_fname##*.}" == "gz" ]; then
                unpack=zcat
            fi
            ;;
        *) echo "Do not know what to do with $image_fname"; false;;
    esac

    if is_arch "ppc64le" || is_arch "ppc64" || is_arch "ppc"; then
        img_property="$img_property --property hw_cdrom_bus=scsi --property os_command_line=console=hvc0"
    fi

    if is_arch "aarch64"; then
        img_property="$img_property --property hw_machine_type=virt --property hw_cdrom_bus=scsi --property hw_scsi_model=virtio-scsi --property os_command_line='console=ttyAMA0'"
    fi

    if [ "$container_format" = "bare" ]; then
        if [ "$unpack" = "zcat" ]; then
            openstack --os-cloud=devstack-admin --os-region-name="$REGION_NAME" image create "$image_name" $img_property --public --container-format=$container_format --disk-format $disk_format < <(zcat --force "${image}")
        elif [ "$unpack" = "bunzip2" ]; then
            openstack --os-cloud=devstack-admin --os-region-name="$REGION_NAME" image create "$image_name" $img_property --public --container-format=$container_format --disk-format $disk_format < <(bunzip2 -cdk "${image}")
        elif [ "$unpack" = "unxz" ]; then
            # NOTE(brtknr): unxz the file first and cleanup afterwards to
            # prevent timeout while Glance tries to upload image (e.g. to Swift).
            local tmp_dir
            local image_path
            tmp_dir=$(mktemp -d)
            image_path="$tmp_dir/$image_name"
            unxz -cv "${image}" > "$image_path"
            openstack --os-cloud=devstack-admin --os-region-name="$REGION_NAME" image create "$image_name" $img_property --public --container-format=$container_format --disk-format $disk_format --file "$image_path"
            rm -rf $tmp_dir
        else
            openstack --os-cloud=devstack-admin --os-region-name="$REGION_NAME" image create "$image_name" $img_property --public --container-format=$container_format --disk-format $disk_format < "${image}"
        fi
    else
        # Use glance client to add the kernel the root filesystem.
        # We parse the results of the first upload to get the glance ID of the
        # kernel for use when uploading the root filesystem.
        local kernel_id="" ramdisk_id="";
        if [ -n "$kernel" ]; then
            kernel_id=$(openstack --os-cloud=devstack-admin --os-region-name="$REGION_NAME" image create "$image_name-kernel" $img_property --public --container-format aki --disk-format aki < "$kernel" | grep ' id ' | get_field 2)
        fi
        if [ -n "$ramdisk" ]; then
            ramdisk_id=$(openstack --os-cloud=devstack-admin --os-region-name="$REGION_NAME" image create "$image_name-ramdisk" $img_property --public --container-format ari --disk-format ari < "$ramdisk" | grep ' id ' | get_field 2)
        fi
        openstack --os-cloud=devstack-admin --os-region-name="$REGION_NAME" image create "${image_name%.img}" $img_property --public --container-format ami --disk-format ami ${kernel_id:+--property kernel_id=$kernel_id} ${ramdisk_id:+--property ramdisk_id=$ramdisk_id} < "${image}"
    fi
}


# Set the database backend to use
# When called from stackrc/localrc DATABASE_BACKENDS has not been
# initialized yet, just save the configuration selection and call back later
# to validate it.
#
# ``$1`` - the name of the database backend to use (mysql, postgresql, ...)
function use_database {
    if [[ -z "$DATABASE_BACKENDS" ]]; then
        # No backends registered means this is likely called from ``localrc``
        # This is now deprecated usage
        DATABASE_TYPE=$1
        deprecated "The database backend needs to be properly set in ENABLED_SERVICES; use_database is deprecated localrc"
    else
        # This should no longer get called...here for posterity
        use_exclusive_service DATABASE_BACKENDS DATABASE_TYPE $1
    fi
}

#Macro for curl statements. curl requires -g option for literal IPv6 addresses.
CURL_GET="${CURL_GET:-curl -g}"

# Wait for an HTTP server to start answering requests
# wait_for_service timeout url
#
# If the service we want is behind a proxy, the proxy may be available
# before the service. Compliant proxies will return a 503 in this case
# Loop until we get something else.
# Also check for the case where there is no proxy and the service just
# hasn't started yet. curl returns 7 for Failed to connect to host.
function wait_for_service {
    local timeout=$1
    local url=$2
    local rval=0
    time_start "wait_for_service"
    timeout $timeout bash -x <<EOF || rval=$?
        while [[ \$( ${CURL_GET} -k --noproxy '*' -s -o /dev/null -w '%{http_code}' ${url} ) == 503 || \$? -eq 7 ]]; do
            sleep 1
        done
EOF
    time_stop "wait_for_service"
    return $rval
}

function wait_for_compute {
    local timeout=$1
    local rval=0
    local compute_hostname
    time_start "wait_for_service"
    compute_hostname=$(iniget $NOVA_CONF DEFAULT host)
    if [[ -z $compute_hostname ]]; then
        compute_hostname=$(hostname)
    fi
    timeout $timeout bash -x <<EOF || rval=$?
        ID=""
        while [[ "\$ID" == "" ]]; do
            sleep 1
            if [[ "$VIRT_DRIVER" = 'fake' ]]; then
                # When using the fake driver the compute hostnames have a suffix of 1 to NUMBER_FAKE_NOVA_COMPUTE
                ID=\$(openstack --os-cloud devstack-admin --os-region "$REGION_NAME" compute service list --host `hostname`1 --service nova-compute -c ID -f value)
            else
                ID=\$(openstack --os-cloud devstack-admin --os-region "$REGION_NAME" compute service list --host "$compute_hostname" --service nova-compute -c ID -f value)
            fi
        done
EOF
    time_stop "wait_for_service"
    # Figure out what's happening on platforms where this doesn't work
    if [[ "$rval" != 0 ]]; then
        echo "Didn't find service registered by hostname after $timeout seconds"
        openstack --os-cloud devstack-admin --os-region "$REGION_NAME" compute service list
    fi
    return $rval
}


# ping check
# Uses globals ``ENABLED_SERVICES``, ``TOP_DIR``, ``PRIVATE_NETWORK``
# ping_check <ip> [boot-timeout] [from_net] [expected]
function ping_check {
    local ip=$1
    local timeout=${2:-30}
    local from_net=${3:-""}
    local expected=${4:-True}
    local op="!"
    local failmsg="[Fail] Couldn't ping server"
    local ping_cmd="ping"

    # if we don't specify a from_net we're expecting things to work
    # fine from our local box.
    if [[ -n "$from_net" ]]; then
        # TODO(stephenfin): Is there any way neutron could be disabled now?
        if is_service_enabled neutron; then
            ping_cmd="$TOP_DIR/tools/ping_neutron.sh $from_net"
        fi
    fi

    # inverse the logic if we're testing no connectivity
    if [[ "$expected" != "True" ]]; then
        op=""
        failmsg="[Fail] Could ping server"
    fi

    # Because we've transformed this command so many times, print it
    # out at the end.
    local check_command="while $op $ping_cmd -c1 -w1 $ip; do sleep 1; done"
    echo "Checking connectivity with $check_command"

    if ! timeout $timeout sh -c "$check_command"; then
        die $LINENO $failmsg
    fi
}

# Get ip of instance
function get_instance_ip {
    local vm_id=$1
    local network_name=$2
    local addresses
    local ip

    addresses=$(openstack server show -c addresses -f value "$vm_id")
    ip=$(echo $addresses | sed -n "s/^.*$network_name=\([0-9\.]*\).*$/\1/p")
    if [[ $ip = "" ]];then
        echo "addresses of server $vm_id : $addresses"
        die $LINENO "[Fail] Couldn't get ipaddress of VM"
    fi
    echo $ip
}

# ssh check

# ssh_check net-name key-file floating-ip default-user active-timeout
function ssh_check {
    if is_service_enabled neutron; then
        _ssh_check_neutron  "$1" $2 $3 $4 $5
        return
    fi
    _ssh_check_novanet "$1" $2 $3 $4 $5
}

function _ssh_check_novanet {
    local NET_NAME=$1
    local KEY_FILE=$2
    local FLOATING_IP=$3
    local DEFAULT_INSTANCE_USER=$4
    local ACTIVE_TIMEOUT=$5
    local probe_cmd=""
    if ! timeout $ACTIVE_TIMEOUT sh -c "while ! ssh -o StrictHostKeyChecking=no -i $KEY_FILE ${DEFAULT_INSTANCE_USER}@$FLOATING_IP echo success; do sleep 1; done"; then
        die $LINENO "server didn't become ssh-able!"
    fi
}


# Get the location of the $module-rootwrap executables, where module is cinder
# or nova.
# get_rootwrap_location module
function get_rootwrap_location {
    local module=$1

    echo "$(get_python_exec_prefix)/$module-rootwrap"
}


# Path permissions sanity check
# check_path_perm_sanity path
function check_path_perm_sanity {
    # Ensure no element of the path has 0700 permissions, which is very
    # likely to cause issues for daemons.  Inspired by default 0700
    # homedir permissions on RHEL and common practice of making DEST in
    # the stack user's homedir.

    local real_path
    real_path=$(readlink -f $1)
    local rebuilt_path=""
    for i in $(echo ${real_path} | tr "/" " "); do
        rebuilt_path=$rebuilt_path"/"$i

        if [[ $(stat -c '%a' ${rebuilt_path}) = 700 ]]; then
            echo "*** DEST path element"
            echo "***    ${rebuilt_path}"
            echo "*** appears to have 0700 permissions."
            echo "*** This is very likely to cause fatal issues for DevStack daemons."

            if [[ -n "$SKIP_PATH_SANITY" ]]; then
                return
            else
                echo "*** Set SKIP_PATH_SANITY to skip this check"
                die $LINENO "Invalid path permissions"
            fi
        fi
    done
}


# vercmp ver1 op ver2
#  Compare VER1 to VER2
#   - op is one of < <= == >= >
#   - returns true if satisified
#  e.g.
#  if vercmp 1.0 "<" 2.0; then
#    ...
#  fi
function vercmp {
    local v1=$1
    local op=$2
    local v2=$3
    local result

    # sort the two numbers with sort's "-V" argument.  Based on if v2
    # swapped places with v1, we can determine ordering.
    result=$(echo -e "$v1\n$v2" | sort -V | head -1)

    case $op in
        "==")
            [ "$v1" = "$v2" ]
            return
            ;;
        ">")
            [ "$v1" != "$v2" ] && [ "$result" = "$v2" ]
            return
            ;;
        "<")
            [ "$v1" != "$v2" ] && [ "$result" = "$v1" ]
            return
            ;;
        ">=")
            [ "$result" = "$v2" ]
            return
            ;;
        "<=")
            [ "$result" = "$v1" ]
            return
            ;;
        *)
            die $LINENO "unrecognised op: $op"
            ;;
    esac
}

# This sets up defaults we like in devstack for logging for tracking
# down issues, and makes sure everything is done the same between
# projects.
function setup_logging {
    local conf_file=$1
    local other_cond=${2:-"False"}
    if [[ "$USE_SYSTEMD" == "True" ]]; then
        setup_systemd_logging $conf_file
    elif [ "$LOG_COLOR" == "True" ] && [ "$SYSLOG" == "False" ] && [ "$other_cond" == "False" ]; then
        setup_colorized_logging $conf_file
    else
        setup_standard_logging_identity $conf_file
    fi
}

# This function sets log formatting options for colorizing log
# output to stdout. It is meant to be called by lib modules.
# The last two parameters are optional and can be used to specify
# non-default value for project and user format variables.
# Defaults are respectively 'project_name' and 'user_name'
#
# setup_colorized_logging something.conf SOMESECTION
function setup_colorized_logging {
    local conf_file=$1
    local conf_section="DEFAULT"
    local project_var="project_name"
    local user_var="user_name"
    # Add color to logging output
    iniset $conf_file $conf_section logging_context_format_string "%(asctime)s.%(msecs)03d %(color)s%(levelname)s %(name)s [%(request_id)s %("$project_var")s %("$user_var")s%(color)s] %(instance)s%(color)s%(message)s"
    iniset $conf_file $conf_section logging_default_format_string "%(asctime)s.%(msecs)03d %(color)s%(levelname)s %(name)s [-%(color)s] %(instance)s%(color)s%(message)s"
    iniset $conf_file $conf_section logging_debug_format_suffix "from (pid=%(process)d) %(funcName)s %(pathname)s:%(lineno)d"
    iniset $conf_file $conf_section logging_exception_prefix "%(color)s%(asctime)s.%(msecs)03d TRACE %(name)s %(instance)s"
}

function setup_systemd_logging {
    local conf_file=$1
    local conf_section="DEFAULT"
    # NOTE(sdague): this is a nice to have, and means we're using the
    # native systemd path, which provides for things like search on
    # request-id. However, there may be an eventlet interaction here,
    # so going off for now.
    USE_JOURNAL=$(trueorfalse False USE_JOURNAL)
    local pidstr=""
    if [[ "$USE_JOURNAL" == "True" ]]; then
        iniset $conf_file $conf_section use_journal "True"
        # if we are using the journal directly, our process id is already correct
    else
        pidstr="(pid=%(process)d) "
    fi
    iniset $conf_file $conf_section logging_debug_format_suffix "{{${pidstr}%(funcName)s %(pathname)s:%(lineno)d}}"

    iniset $conf_file $conf_section logging_context_format_string "%(color)s%(levelname)s %(name)s [%(global_request_id)s %(request_id)s %(project_name)s %(user_name)s%(color)s] %(instance)s%(color)s%(message)s"
    iniset $conf_file $conf_section logging_default_format_string "%(color)s%(levelname)s %(name)s [-%(color)s] %(instance)s%(color)s%(message)s"
    iniset $conf_file $conf_section logging_exception_prefix "ERROR %(name)s %(instance)s"
}

function setup_standard_logging_identity {
    local conf_file=$1
    iniset $conf_file DEFAULT logging_user_identity_format "%(project_name)s %(user_name)s"
}

# These functions are provided for basic fall-back functionality for
# projects that include parts of DevStack (Grenade).  stack.sh will
# override these with more specific versions for DevStack (with fancy
# spinners, etc).  We never override an existing version
if ! function_exists echo_summary; then
    function echo_summary {
        echo $@
    }
fi
if ! function_exists echo_nolog; then
    function echo_nolog {
        echo $@
    }
fi


# create_disk - Create backing disk
function create_disk {
    local node_number
    local disk_image=${1}
    local storage_data_dir=${2}
    local loopback_disk_size=${3}

    # Create a loopback disk and format it to XFS.
    if [[ -e ${disk_image} ]]; then
        if egrep -q ${storage_data_dir} /proc/mounts; then
            sudo umount ${storage_data_dir}
            sudo rm -f ${disk_image}
        fi
    fi

    sudo mkdir -p ${storage_data_dir}/drives/images

    sudo truncate -s ${loopback_disk_size} ${disk_image}

    # Make a fresh XFS filesystem. Use bigger inodes so xattr can fit in
    # a single inode. Keeping the default inode size (256) will result in multiple
    # inodes being used to store xattr. Retrieving the xattr will be slower
    # since we have to read multiple inodes. This statement is true for both
    # Swift and Ceph.
    sudo mkfs.xfs -f -i size=1024 ${disk_image}

    # Mount the disk with mount options to make it as efficient as possible
    if ! egrep -q ${storage_data_dir} /proc/mounts; then
        sudo mount -t xfs -o loop,noatime,nodiratime,logbufs=8  \
            ${disk_image} ${storage_data_dir}
    fi
}


# set_mtu - Set MTU on a device
function set_mtu {
    local dev=$1
    local mtu=$2
    sudo ip link set mtu $mtu dev $dev
}


# running_in_container - Returns true otherwise false
function running_in_container {
    [[ $(systemd-detect-virt --container) != 'none' ]]
}


# enable_kernel_bridge_firewall - Enable kernel support for bridge firewalling
function enable_kernel_bridge_firewall {
    # Load bridge module. This module provides access to firewall for bridged
    # frames; and also on older kernels (pre-3.18) it provides sysctl knobs to
    # enable/disable bridge firewalling
    sudo modprobe bridge
    # For newer kernels (3.18+), those sysctl settings are split into a separate
    # kernel module (br_netfilter). Load it too, if present.
    sudo modprobe br_netfilter 2>> /dev/null || :
    # Enable bridge firewalling in case it's disabled in kernel (upstream
    # default is enabled, but some distributions may decide to change it).
    # This is at least needed for RHEL 7.2 and earlier releases.
    for proto in ip ip6; do
        sudo sysctl -w net.bridge.bridge-nf-call-${proto}tables=1
    done
}


# Set a systemd system override
#
# This sets a system-side override in system.conf. A per-service
# override would be /etc/systemd/system/${service}.service/override.conf
function set_systemd_override {
    local key="$1"
    local value="$2"

    local sysconf="/etc/systemd/system.conf"
    iniset -sudo "${sysconf}" "Manager" "$key" "$value"
    echo "Set systemd system override for ${key}=${value}"

    sudo systemctl daemon-reload
}

# Get a random port from the local port range
#
# This function returns an available port in the local port range. The search
# order is not truly random, but should be considered a random value by the
# user because it depends on the state of your local system.
function get_random_port {
    read lower_port upper_port < /proc/sys/net/ipv4/ip_local_port_range
    while true; do
        for (( port = upper_port ; port >= lower_port ; port-- )); do
            sudo lsof -i ":$port" &> /dev/null
            if [[ $? > 0 ]] ; then
                break 2
            fi
        done
    done
    echo $port
}

# Save some state information
#
# Write out various useful state information to /etc/devstack-version
function write_devstack_version {
    cat - <<EOF | sudo tee /etc/devstack-version >/dev/null
DevStack Version: ${DEVSTACK_SERIES}
Change: $(git log --format="%H %s %ci" -1)
OS Version: ${os_VENDOR} ${os_RELEASE} ${os_CODENAME}
EOF
}

# Restore xtrace
$_XTRACE_FUNCTIONS

# Local variables:
# mode: shell-script
# End: