From 67787e6b4c6f31388cbee6d83b67371b31c443d4 Mon Sep 17 00:00:00 2001 From: Dean Troyer Date: Wed, 2 May 2012 11:48:15 -0500 Subject: [PATCH] Add cinder support * using the new functional pattern: cinder_XXX functions are in lib/cinder * enable with 'c-api,c-sch,c-vol' in ENABLED_SERVICES, n-vol is still the default * exercises/volumes.sh runs for cinder and n-vol * move config to /etc/cinder * change volume_group to stack-volumes; this also renames the backing file to /opt/stack/data/stack-volumes-backing-file. * removes osapi_volume from nova.conf enabled_apis * integrates cinder + keystone * launches c-sch * tweaks for multi node * move enabled_apis substitution to init_cinder 18Jun2010 * restored & rebased * update setup.py to use setup_develop() in lib/cinder Change-Id: I1e1aa4387031c56e4fa239eb73bea2af8cef0e38 --- exerciserc | 6 ++ exercises/euca.sh | 3 + exercises/volumes.sh | 4 ++ files/apts/cinder | 2 + files/keystone_data.sh | 11 +++ files/rpms/cinder | 2 + functions | 1 + lib/cinder | 154 +++++++++++++++++++++++++++++++++++++++++ stack.sh | 62 ++++++++++++++--- stackrc | 11 +++ unstack.sh | 2 +- 11 files changed, 248 insertions(+), 10 deletions(-) create mode 100644 files/apts/cinder create mode 100644 files/rpms/cinder create mode 100644 lib/cinder diff --git a/exerciserc b/exerciserc index b41714da49..82c74b7fe4 100644 --- a/exerciserc +++ b/exerciserc @@ -20,3 +20,9 @@ export RUNNING_TIMEOUT=${RUNNING_TIMEOUT:-$(($BOOT_TIMEOUT + $ACTIVE_TIMEOUT))} # Max time to wait for a vm to terminate export TERMINATE_TIMEOUT=${TERMINATE_TIMEOUT:-30} + +# Max time to wait for a euca-volume command to propogate +export VOLUME_TIMEOUT=${VOLUME_TIMEOUT:-30} + +# Max time to wait for a euca-delete command to propogate +export VOLUME_DELETE_TIMEOUT=${SNAPSHOT_DELETE_TIMEOUT:-60} diff --git a/exercises/euca.sh b/exercises/euca.sh index 76e5202abc..4a538c6359 100755 --- a/exercises/euca.sh +++ b/exercises/euca.sh @@ -24,6 +24,9 @@ set -o xtrace # Keep track of the current directory EXERCISE_DIR=$(cd $(dirname "$0") && pwd) TOP_DIR=$(cd $EXERCISE_DIR/..; pwd) +VOLUME_ZONE=cinder +VOLUME_SIZE=1 +ATTACH_DEVICE=/dev/vdc # Import common functions source $TOP_DIR/functions diff --git a/exercises/volumes.sh b/exercises/volumes.sh index 6749558a21..0f25355f62 100755 --- a/exercises/volumes.sh +++ b/exercises/volumes.sh @@ -33,6 +33,10 @@ source $TOP_DIR/openrc # Import exercise configuration source $TOP_DIR/exerciserc +# If cinder or n-vol are not enabled we exit with exitcode 55 which mean +# exercise is skipped. +is_service_enabled cinder n-vol || exit 55 + # Instance type to create DEFAULT_INSTANCE_TYPE=${DEFAULT_INSTANCE_TYPE:-m1.tiny} diff --git a/files/apts/cinder b/files/apts/cinder new file mode 100644 index 0000000000..5db06eac99 --- /dev/null +++ b/files/apts/cinder @@ -0,0 +1,2 @@ +tgt +lvm2 diff --git a/files/keystone_data.sh b/files/keystone_data.sh index 2cdc2fa9d1..5aea82ea34 100755 --- a/files/keystone_data.sh +++ b/files/keystone_data.sh @@ -9,6 +9,7 @@ # service nova admin, [ResellerAdmin (swift only)] # service quantum admin # if enabled # service swift admin # if enabled +# service cinder admin # if enabled # demo admin admin # demo demo Member, anotherrole # invisible_to_admin demo Member @@ -128,3 +129,13 @@ if [[ "$ENABLED_SERVICES" =~ "tempest" ]]; then --email=alt_demo@example.com) keystone user-role-add --user $ALT_DEMO_USER --role $MEMBER_ROLE --tenant_id $ALT_DEMO_TENANT fi + +if [[ "$ENABLED_SERVICES" =~ "cinder" ]]; then + CINDER_USER=$(get_id keystone user-create --name=cinder \ + --pass="$SERVICE_PASSWORD" \ + --tenant_id $SERVICE_TENANT \ + --email=cinder@example.com) + keystone user-role-add --tenant_id $SERVICE_TENANT \ + --user_id $CINDER_USER \ + --role_id $ADMIN_ROLE +fi diff --git a/files/rpms/cinder b/files/rpms/cinder new file mode 100644 index 0000000000..df861aade0 --- /dev/null +++ b/files/rpms/cinder @@ -0,0 +1,2 @@ +lvm2 +scsi-target-utils diff --git a/functions b/functions index 32427a4df3..a80d06d471 100644 --- a/functions +++ b/functions @@ -253,6 +253,7 @@ function is_service_enabled() { for service in ${services}; do [[ ,${ENABLED_SERVICES}, =~ ,${service}, ]] && return 0 [[ ${service} == "nova" && ${ENABLED_SERVICES} =~ "n-" ]] && return 0 + [[ ${service} == "cinder" && ${ENABLED_SERVICES} =~ "c-" ]] && return 0 [[ ${service} == "glance" && ${ENABLED_SERVICES} =~ "g-" ]] && return 0 [[ ${service} == "quantum" && ${ENABLED_SERVICES} =~ "q-" ]] && return 0 done diff --git a/lib/cinder b/lib/cinder new file mode 100644 index 0000000000..f0715a4d9f --- /dev/null +++ b/lib/cinder @@ -0,0 +1,154 @@ +# lib/cinder +# Install and start Cinder volume service + +# Dependencies: +# - functions +# - KEYSTONE_AUTH_* must be defined +# SERVICE_{TENANT_NAME|PASSWORD} must be defined + +# stack.sh +# --------- +# install_XXX +# configure_XXX +# init_XXX +# start_XXX +# stop_XXX +# cleanup_XXX + +# Print the commands being run so that we can see the command that triggers +# an error. It is also useful for following along as the install occurs. +set -o xtrace + + +# Defaults +# -------- + +# set up default directories +CINDER_DIR=$DEST/cinder +CINDERCLIENT_DIR=$DEST/python-cinderclient +CINDER_CONF_DIR=/etc/cinder +CINDER_CONF=$CINDER_CONF_DIR/cinder.conf + +# Name of the lvm volume group to use/create for iscsi volumes +VOLUME_GROUP=${VOLUME_GROUP:-stack-volumes} +VOLUME_NAME_PREFIX=${VOLUME_NAME_PREFIX:-volume-} + +# cleanup_cinder() - Remove residual data files, anything left over from previous +# runs that a clean run would need to clean up +function cleanup_cinder() { + # This function intentionally left blank + : +} + +# configure_cinder() - Set config files, create data dirs, etc +function configure_cinder() { + setup_develop $CINDER_DIR + setup_develop $CINDERCLIENT_DIR + + if [[ ! -d $CINDER_CONF_DIR ]]; then + sudo mkdir -p $CINDER_CONF_DIR + fi + sudo chown `whoami` $CINDER_CONF_DIR + + cp -p $CINDER_DIR/etc/cinder/policy.json $CINDER_CONF_DIR + + CINDER_API_PASTE_INI=$CINDER_CONF_DIR/api-paste.ini + cp $CINDER_DIR/etc/cinder/api-paste.ini $CINDER_API_PASTE_INI + iniset $CINDER_API_PASTE_INI filter:authtoken auth_host $KEYSTONE_AUTH_HOST + iniset $CINDER_API_PASTE_INI filter:authtoken auth_port $KEYSTONE_AUTH_PORT + iniset $CINDER_API_PASTE_INI filter:authtoken auth_protocol $KEYSTONE_AUTH_PROTOCOL + iniset $CINDER_API_PASTE_INI filter:authtoken admin_tenant_name $SERVICE_TENANT_NAME + iniset $CINDER_API_PASTE_INI filter:authtoken admin_user cinder + iniset $CINDER_API_PASTE_INI filter:authtoken admin_password $SERVICE_PASSWORD + + cp $CINDER_DIR/etc/cinder/cinder.conf.sample $CINDER_CONF + iniset $CINDER_CONF DEFAULT auth_strategy keystone + iniset $CINDER_CONF DEFAULT verbose True + iniset $CINDER_CONF DEFAULT volume_group $VOLUME_GROUP + iniset $CINDER_CONF DEFAULT volume_name_template ${VOLUME_NAME_PREFIX}%s + iniset $CINDER_CONF DEFAULT iscsi_helper tgtadm + iniset $CINDER_CONF DEFAULT sql_connection $BASE_SQL_CONN/cinder?charset=utf8 + iniset $CINDER_CONF DEFAULT rabbit_host $RABBIT_HOST + iniset $CINDER_CONF DEFAULT rabbit_password $RABBIT_PASSWORD + iniset $CINDER_CONF DEFAULT api_paste_config $CINDER_API_PASTE_INI +} + +# init_cinder() - Initialize database and volume group +function init_cinder() { + # Force nova volumes off + NOVA_ENABLED_APIS=$(echo $NOVA_ENABLED_APIS | sed "s/osapi_volume,//") + + if is_service_enabled mysql; then + # (re)create cinder database + mysql -u$MYSQL_USER -p$MYSQL_PASSWORD -e 'DROP DATABASE IF EXISTS cinder;' + mysql -u$MYSQL_USER -p$MYSQL_PASSWORD -e 'CREATE DATABASE cinder;' + + # (re)create cinder database + $CINDER_DIR/bin/cinder-manage db sync + fi + + if is_service_enabled c-vol; then + # Configure a default volume group called '`stack-volumes`' for the volume + # service if it does not yet exist. If you don't wish to use a file backed + # volume group, create your own volume group called ``stack-volumes`` before + # invoking ``stack.sh``. + # + # By default, the backing file is 2G in size, and is stored in ``/opt/stack/data``. + + if ! sudo vgs $VOLUME_GROUP; then + VOLUME_BACKING_FILE=${VOLUME_BACKING_FILE:-$DATA_DIR/${VOLUME_GROUP}-backing-file} + VOLUME_BACKING_FILE_SIZE=${VOLUME_BACKING_FILE_SIZE:-2052M} + # Only create if the file doesn't already exists + [[ -f $VOLUME_BACKING_FILE ]] || truncate -s $VOLUME_BACKING_FILE_SIZE $VOLUME_BACKING_FILE + DEV=`sudo losetup -f --show $VOLUME_BACKING_FILE` + # Only create if the loopback device doesn't contain $VOLUME_GROUP + if ! sudo vgs $VOLUME_GROUP; then sudo vgcreate $VOLUME_GROUP $DEV; fi + fi + + if sudo vgs $VOLUME_GROUP; then + # Remove iscsi targets + sudo tgtadm --op show --mode target | grep $VOLUME_NAME_PREFIX | grep Target | cut -f3 -d ' ' | sudo xargs -n1 tgt-admin --delete || true + # Clean out existing volumes + for lv in `sudo lvs --noheadings -o lv_name $VOLUME_GROUP`; do + # VOLUME_NAME_PREFIX prefixes the LVs we want + if [[ "${lv#$VOLUME_NAME_PREFIX}" != "$lv" ]]; then + sudo lvremove -f $VOLUME_GROUP/$lv + fi + done + fi + fi +} + +# install_cinder() - Collect source and prepare +function install_cinder() { + git_clone $CINDER_REPO $CINDER_DIR $CINDER_BRANCH + git_clone $CINDERCLIENT_REPO $CINDERCLIENT_DIR $CINDERCLIENT_BRANCH +} + +# start_cinder() - Start running processes, including screen +function start_cinder() { + if is_service_enabled c-vol; then + if [[ "$os_PACKAGE" = "deb" ]]; then + # tgt in oneiric doesn't restart properly if tgtd isn't running + # do it in two steps + sudo stop tgt || true + sudo start tgt + else + # bypass redirection to systemctl during restart + sudo /sbin/service --skip-redirect tgtd restart + fi + fi + + screen_it c-api "cd $CINDER_DIR && $CINDER_DIR/bin/cinder-api --config-file $CINDER_CONF" + screen_it c-vol "cd $CINDER_DIR && $CINDER_DIR/bin/cinder-volume --config-file $CINDER_CONF" + screen_it c-sch "cd $CINDER_DIR && $CINDER_DIR/bin/cinder-scheduler --config-file $CINDER_CONF" +} + +# stop_cinder() - Stop running processes (non-screen) +function stop_cinder() { + # FIXME(dtroyer): stop only the cinder screen window? + + if is_service_enabled c-vol; then + stop_service tgt + fi +} diff --git a/stack.sh b/stack.sh index 126ff37551..793ded99e5 100755 --- a/stack.sh +++ b/stack.sh @@ -112,6 +112,13 @@ else NOVA_ROOTWRAP=/usr/bin/nova-rootwrap fi +# ``stack.sh`` keeps function libraries here +# Make sure ``$TOP_DIR/lib`` directory is present +if [ ! -d $TOP_DIR/lib ]; then + echo "ERROR: missing devstack/lib - did you grab more than just stack.sh?" + exit 1 +fi + # stack.sh keeps the list of ``apt`` and ``pip`` dependencies in external # files, along with config templates and other useful files. You can find these # in the ``files`` directory (next to this script). We will reference this @@ -130,6 +137,12 @@ if type -p screen >/dev/null && screen -ls | egrep -q "[0-9].stack"; then exit 1 fi +# Make sure we only have one volume service enabled. +if is_service_enabled cinder && is_service_enabled n-vol; then + echo "ERROR: n-vol and cinder must not be enabled at the same time" + exit 1 +fi + # OpenStack is designed to be run as a regular user (Horizon will fail to run # as root, since apache refused to startup serve content from root user). If # ``stack.sh`` is run as **root**, it automatically creates a **stack** user with @@ -201,6 +214,19 @@ fi # prerequisites and initialize ``$DEST``. OFFLINE=`trueorfalse False $OFFLINE` +# Destination path for service data +DATA_DIR=${DATA_DIR:-${DEST}/data} +sudo mkdir -p $DATA_DIR +sudo chown `whoami` $DATA_DIR + + +# Projects +# -------- + +# Get project function libraries +source $TOP_DIR/lib/cinder + + # Set the destination directories for openstack projects NOVA_DIR=$DEST/nova HORIZON_DIR=$DEST/horizon @@ -234,7 +260,7 @@ M_HOST=${M_HOST:-localhost} M_MAC_RANGE=${M_MAC_RANGE:-FE-EE-DD-00-00-00/24} # Name of the lvm volume group to use/create for iscsi volumes -VOLUME_GROUP=${VOLUME_GROUP:-nova-volumes} +VOLUME_GROUP=${VOLUME_GROUP:-stack-volumes} VOLUME_NAME_PREFIX=${VOLUME_NAME_PREFIX:-volume-} INSTANCE_NAME_PREFIX=${INSTANCE_NAME_PREFIX:-instance-} @@ -607,6 +633,10 @@ function get_packages() { if [[ ! $file_to_parse =~ glance ]]; then file_to_parse="${file_to_parse} glance" fi + elif [[ $service == c-* ]]; then + if [[ ! $file_to_parse =~ cinder ]]; then + file_to_parse="${file_to_parse} cinder" + fi elif [[ $service == n-* ]]; then if [[ ! $file_to_parse =~ nova ]]; then file_to_parse="${file_to_parse} nova" @@ -704,10 +734,12 @@ if is_service_enabled m-svc; then # melange git_clone $MELANGE_REPO $MELANGE_DIR $MELANGE_BRANCH fi - if is_service_enabled melange; then git_clone $MELANGECLIENT_REPO $MELANGECLIENT_DIR $MELANGECLIENT_BRANCH fi +if is_service_enabled cinder; then + install_cinder +fi # Initialization @@ -743,6 +775,9 @@ fi if is_service_enabled melange; then setup_develop $MELANGECLIENT_DIR fi +if is_service_enabled cinder; then + configure_cinder +fi # Do this _after_ glance is installed to override the old binary setup_develop $GLANCECLIENT_DIR @@ -1643,17 +1678,18 @@ fi # Volume Service # -------------- -if is_service_enabled n-vol; then - # - # Configure a default volume group called 'nova-volumes' for the nova-volume +if is_service_enabled cinder; then + init_cinder +elif is_service_enabled n-vol; then + # Configure a default volume group called '`stack-volumes`' for the volume # service if it does not yet exist. If you don't wish to use a file backed - # volume group, create your own volume group called 'nova-volumes' before - # invoking stack.sh. + # volume group, create your own volume group called ``stack-volumes`` before + # invoking ``stack.sh``. # - # By default, the backing file is 2G in size, and is stored in /opt/stack. + # By default, the backing file is 2G in size, and is stored in ``/opt/stack/data``. if ! sudo vgs $VOLUME_GROUP; then - VOLUME_BACKING_FILE=${VOLUME_BACKING_FILE:-$DEST/nova-volumes-backing-file} + VOLUME_BACKING_FILE=${VOLUME_BACKING_FILE:-$DATA_DIR/${VOLUME_GROUP}-backing-file} VOLUME_BACKING_FILE_SIZE=${VOLUME_BACKING_FILE_SIZE:-2052M} # Only create if the file doesn't already exists [[ -f $VOLUME_BACKING_FILE ]] || truncate -s $VOLUME_BACKING_FILE_SIZE $VOLUME_BACKING_FILE @@ -1801,6 +1837,10 @@ else add_nova_opt "logging_context_format_string=%(asctime)s %(levelname)s %(name)s [%(request_id)s %(user_name)s %(project_name)s] %(instance)s%(message)s" fi +# If cinder is enabled, use the cinder volume driver +if is_service_enabled cinder; then + add_nova_opt "volume_api_class=nova.volume.cinder.API" +fi # Provide some transition from EXTRA_FLAGS to EXTRA_OPTS if [[ -z "$EXTRA_OPTS" && -n "$EXTRA_FLAGS" ]]; then @@ -1968,6 +2008,7 @@ fi # launch the nova-api and wait for it to answer before continuing if is_service_enabled n-api; then + add_nova_opt "enabled_apis=$NOVA_ENABLED_APIS" screen_it n-api "cd $NOVA_DIR && $NOVA_DIR/bin/nova-api" echo "Waiting for nova-api to start..." if ! timeout $SERVICE_TIMEOUT sh -c "while ! http_proxy= wget -q -O- http://127.0.0.1:8774; do sleep 1; done"; then @@ -2003,6 +2044,9 @@ screen_it n-sch "cd $NOVA_DIR && $NOVA_DIR/bin/nova-scheduler" screen_it n-novnc "cd $NOVNC_DIR && ./utils/nova-novncproxy --config-file $NOVA_CONF_DIR/$NOVA_CONF --web ." screen_it n-xvnc "cd $NOVA_DIR && ./bin/nova-xvpvncproxy --config-file $NOVA_CONF_DIR/$NOVA_CONF" screen_it n-cauth "cd $NOVA_DIR && ./bin/nova-consoleauth" +if is_service_enabled cinder; then + start_cinder +fi screen_it horizon "cd $HORIZON_DIR && sudo tail -f /var/log/$APACHE_NAME/horizon_error.log" screen_it swift "cd $SWIFT_DIR && $SWIFT_DIR/bin/swift-proxy-server ${SWIFT_CONFIG_DIR}/proxy-server.conf -v" diff --git a/stackrc b/stackrc index 3387d43e86..cc07906d7f 100644 --- a/stackrc +++ b/stackrc @@ -9,6 +9,17 @@ RC_DIR=$(cd $(dirname "$BASH_SOURCE") && pwd) # ENABLED_SERVICES="$ENABLED_SERVICES,swift" ENABLED_SERVICES=g-api,g-reg,key,n-api,n-crt,n-obj,n-cpu,n-net,n-vol,n-sch,n-novnc,n-xvnc,n-cauth,horizon,mysql,rabbit +# Set the default Nova APIs to enable +NOVA_ENABLED_APIS=ec2,osapi_compute,osapi_volume,metadata + +# volume service +CINDER_REPO=https://github.com/openstack/cinder +CINDER_BRANCH=master + +# volume client +CINDERCLIENT_REPO=https://github.com/openstack/python-cinderclient +CINDERCLIENT_BRANCH=master + # compute service NOVA_REPO=https://github.com/openstack/nova.git NOVA_BRANCH=master diff --git a/unstack.sh b/unstack.sh index 341270d602..879f842463 100755 --- a/unstack.sh +++ b/unstack.sh @@ -44,7 +44,7 @@ if is_service_enabled horizon; then fi # Get the iSCSI volumes -if is_service_enabled n-vol; then +if is_service_enabled cinder n-vol; then TARGETS=$(sudo tgtadm --op show --mode target) if [[ -n "$TARGETS" ]]; then # FIXME(dtroyer): this could very well require more here to