diff --git a/devstack/plugin.sh b/devstack/plugin.sh index 0b4e9041f9..6102e9c66b 100644 --- a/devstack/plugin.sh +++ b/devstack/plugin.sh @@ -281,14 +281,25 @@ function configure_trove { iniset $TROVE_GUESTAGENT_CONF DEFAULT remote_swift_client trove.common.clients_admin.swift_client_trove_admin iniset $TROVE_GUESTAGENT_CONF DEFAULT remote_glance_client trove.common.clients_admin.glance_client_trove_admin - # To avoid 'Connection timed out' error of sudo command inside the guest agent - CLOUDINIT_PATH=/etc/trove/cloudinit/${TROVE_DATASTORE_TYPE}.cloudinit - sudo mkdir -p $(dirname "$CLOUDINIT_PATH") - sudo touch "$CLOUDINIT_PATH" - sudo tee $CLOUDINIT_PATH >/dev/null <<'EOF' + # 1. To avoid 'Connection timed out' error of sudo command inside the guest agent + # 2. Config the controller IP address used by guest-agent to download Trove code during initialization (only valid for dev_mode=true). + common_cloudinit=/etc/trove/cloudinit/common.cloudinit + sudo mkdir -p $(dirname ${common_cloudinit}) + sudo touch ${common_cloudinit} + sudo tee ${common_cloudinit} >/dev/null </dev/null <<'EOF' -#cloud-config -manage_etc_hosts: "localhost" -EOF } # Set up Trove management network and make configuration change. diff --git a/integration/README.md b/integration/README.md index 2717958a74..02dcaf35b8 100644 --- a/integration/README.md +++ b/integration/README.md @@ -135,9 +135,8 @@ PATH_DEVSTACK_OUTPUT=/opt/stack \ - if `dev_mode=false`, the trove code for guest agent is injected into the image at the building time. Now `dev_mode=false` is still in experimental and not considered production ready yet. -- If `dev_mode=true`, the script assumes to be running on the host of - trove-taskmanager, otherwise, `CONTROLLER_IP` needs to be specified - explicitly. +- If `dev_mode=true`, no Trove code is injected into the guest image. The guest + agent will download Trove code during the service initialization. For example, build a Mysql image for Ubuntu Xenial operating system: diff --git a/integration/scripts/files/elements/ubuntu-trusty-cassandra/element-deps b/integration/scripts/files/elements/ubuntu-trusty-cassandra/element-deps deleted file mode 100644 index 28898cf7c3..0000000000 --- a/integration/scripts/files/elements/ubuntu-trusty-cassandra/element-deps +++ /dev/null @@ -1 +0,0 @@ -ubuntu-cassandra diff --git a/integration/scripts/files/elements/ubuntu-trusty-couchbase/element-deps b/integration/scripts/files/elements/ubuntu-trusty-couchbase/element-deps deleted file mode 100644 index fa85fc7e76..0000000000 --- a/integration/scripts/files/elements/ubuntu-trusty-couchbase/element-deps +++ /dev/null @@ -1 +0,0 @@ -ubuntu-couchbase diff --git a/integration/scripts/files/elements/ubuntu-trusty-couchdb/element-deps b/integration/scripts/files/elements/ubuntu-trusty-couchdb/element-deps deleted file mode 100644 index abd5561b71..0000000000 --- a/integration/scripts/files/elements/ubuntu-trusty-couchdb/element-deps +++ /dev/null @@ -1 +0,0 @@ -ubuntu-couchdb diff --git a/integration/scripts/files/elements/ubuntu-trusty-db2/element-deps b/integration/scripts/files/elements/ubuntu-trusty-db2/element-deps deleted file mode 100644 index c5a439cdb6..0000000000 --- a/integration/scripts/files/elements/ubuntu-trusty-db2/element-deps +++ /dev/null @@ -1 +0,0 @@ -ubuntu-db2 diff --git a/integration/scripts/files/elements/ubuntu-trusty-guest/element-deps b/integration/scripts/files/elements/ubuntu-trusty-guest/element-deps deleted file mode 100644 index eaa808e186..0000000000 --- a/integration/scripts/files/elements/ubuntu-trusty-guest/element-deps +++ /dev/null @@ -1 +0,0 @@ -ubuntu-guest diff --git a/integration/scripts/files/elements/ubuntu-trusty-guest/extra-data.d/20-guest-upstart b/integration/scripts/files/elements/ubuntu-trusty-guest/extra-data.d/20-guest-upstart deleted file mode 100755 index 5d06bb0b7b..0000000000 --- a/integration/scripts/files/elements/ubuntu-trusty-guest/extra-data.d/20-guest-upstart +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash - -set -e -set -o xtrace - -# CONTEXT: HOST prior to IMAGE BUILD as SCRIPT USER -# PURPOSE: stages the bootstrap file and upstart conf file while replacing variables so that guest image is properly -# configured - -source $_LIB/die - -[ -n "$TMP_HOOKS_PATH" ] || die "Temp hook path not set" - -[ -n "${GUEST_USERNAME}" ] || die "GUEST_USERNAME needs to be set to the user for the guest image" -[ -n "${HOST_SCP_USERNAME}" ] || die "HOST_SCP_USERNAME needs to be set to the user for the host instance" -[ -n "${CONTROLLER_IP}" ] || die "CONTROLLER_IP needs to be set to the ip address that guests will use to contact the controller" -[ -n "${ESCAPED_PATH_TROVE}" ] || die "ESCAPED_PATH_TROVE needs to be set to the path to the trove directory on the trovestack host" -[ -n "${TROVESTACK_SCRIPTS}" ] || die "TROVESTACK_SCRIPTS needs to be set to the trove/integration/scripts dir" -[ -n "${ESCAPED_GUEST_LOGDIR}" ] || die "ESCAPED_GUEST_LOGDIR must be set to the escaped guest log dir" - -sed "s/GUEST_USERNAME/${GUEST_USERNAME}/g;s/GUEST_LOGDIR/${ESCAPED_GUEST_LOGDIR}/g;s/HOST_SCP_USERNAME/${HOST_SCP_USERNAME}/g;s/CONTROLLER_IP/${CONTROLLER_IP}/g;s/PATH_TROVE/${ESCAPED_PATH_TROVE}/g" ${TROVESTACK_SCRIPTS}/files/trove-guest.upstart.conf > ${TMP_HOOKS_PATH}/trove-guest.conf diff --git a/integration/scripts/files/elements/ubuntu-trusty-guest/install.d/20-etc b/integration/scripts/files/elements/ubuntu-trusty-guest/install.d/20-etc deleted file mode 100755 index 8ac19f7cdd..0000000000 --- a/integration/scripts/files/elements/ubuntu-trusty-guest/install.d/20-etc +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh - -# CONTEXT: GUEST during CONSTRUCTION as ROOT -# PURPOSE: take "staged" trove-guest.conf file and put it in the init directory on guest image - -dd if=/tmp/in_target.d/trove-guest.conf of=/etc/init/trove-guest.conf - - diff --git a/integration/scripts/files/elements/ubuntu-trusty-guest/pre-install.d/01-trim-pkgs b/integration/scripts/files/elements/ubuntu-trusty-guest/pre-install.d/01-trim-pkgs deleted file mode 100755 index 8787df7b48..0000000000 --- a/integration/scripts/files/elements/ubuntu-trusty-guest/pre-install.d/01-trim-pkgs +++ /dev/null @@ -1,117 +0,0 @@ -#!/bin/bash - -# CONTEXT: GUEST during CONSTRUCTION as ROOT -# PURPOSE: Install basic services and applications - -set -e -set -o xtrace - -export DEBIAN_FRONTEND=noninteractive -apt-get -y purge acpid\ - apport\ - apport-symptoms\ - apt-transport-https\ - apt-xapian-index\ - aptitude\ - at\ - bash-completion\ - bc\ - bind9-host\ - bsdmainutils\ - busybox-static\ - byobu\ - command-not-found\ - command-not-found-data\ - curl\ - dbus\ - dmidecode\ - dosfstools\ - ed\ - fonts-ubuntu-font-family-console\ - friendly-recovery\ - ftp\ - fuse\ - geoip-database\ - groff-base\ - hdparm\ - info\ - install-info\ - iptables\ - iputils-tracepath\ - irqbalance\ - landscape-client\ - landscape-common\ - language-selector-common\ - laptop-detect\ - libaccountsservice0\ - libbind9-90\ - libclass-accessor-perl\ - libcwidget3\ - libdns100\ - libept1.4.12\ - libevent-2.0-5\ - libgc1c2\ - libgeoip1\ - libio-string-perl\ - libisc95\ - liblwres90\ - libnfnetlink0\ - libparse-debianchangelog-perl\ - libparted0debian1\ - libpcap0.8\ - libpci3\ - libpipeline1\ - libpolkit-gobject-1-0\ - libsasl2-modules\ - libsigc++-2.0-0c2a\ - libsub-name-perl\ - libusb-1.0-0\ - libxapian22\ - lshw\ - lsof\ - ltrace\ - man-db\ - mlocate\ - mtr-tiny\ - nano\ - ntfs-3g\ - parted\ - patch\ - plymouth-theme-ubuntu-text\ - popularity-contest\ - powermgmt-base\ - ppp\ - pppoeconf\ - python-debian\ - python-gdbm\ - python-pam\ - python-twisted-bin\ - python-xapian\ - python-zope.interface\ - screen\ - shared-mime-info\ - strace\ - tasksel\ - tcpdump\ - telnet\ - time\ - tmux\ - ubuntu-standard\ - ufw\ - update-manager-core\ - update-notifier-common\ - usbutils\ - uuid-runtime\ - w3m - -# The following packages cannot be removed as they cause cloud-init to be -# uninstalled in Ubuntu 14.04 -# gir1.2-glib-2.0 -# libdbus-glib-1-2 -# libgirepository-1.0-1 -# python-chardet -# python-serial -# xz-utils - -apt-get -y autoremove - diff --git a/integration/scripts/files/elements/ubuntu-trusty-mariadb/element-deps b/integration/scripts/files/elements/ubuntu-trusty-mariadb/element-deps deleted file mode 100644 index 5d7756f933..0000000000 --- a/integration/scripts/files/elements/ubuntu-trusty-mariadb/element-deps +++ /dev/null @@ -1 +0,0 @@ -ubuntu-mariadb diff --git a/integration/scripts/files/elements/ubuntu-trusty-mariadb/install.d/30-mariadb b/integration/scripts/files/elements/ubuntu-trusty-mariadb/install.d/30-mariadb deleted file mode 100755 index e8b7dd760f..0000000000 --- a/integration/scripts/files/elements/ubuntu-trusty-mariadb/install.d/30-mariadb +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash - -# CONTEXT: GUEST during CONSTRUCTION as ROOT -# PURPOSE: Install controller base required packages - -set -e -set -o xtrace - -export DEBIAN_FRONTEND=noninteractive - -# NOTE(vkmc): Using MariaDB repositories is required -# https://mariadb.com/kb/en/mariadb/installing-mariadb-deb-files/ -apt-get --allow-unauthenticated -y install software-properties-common - -get_key_robust 0xcbcb082a1bb943db - -add-apt-repository 'deb http://ftp.osuosl.org/pub/mariadb/repo/10.1/ubuntu trusty main' - -# Pin MariaDB repository -sudo echo -e "Package: *\nPin: origin ftp.osuosl.org\nPin-Priority: 1000" > /etc/apt/preferences.d/mariadb.pref - -apt-get -y update -# The fix to make versions of percona-xtrabackup > v2.2 work with Trove -# was put into the mysql guestagent code for Mitaka. There are no current -# plans to backport so we need to make sure the guest generated when the -# tests are run for Kilo or Liberty get the 2.2 verson of PXB -if [[ $BRANCH_OVERRIDE == "stable/kilo" || $BRANCH_OVERRIDE == "stable/liberty" ]]; then - PXB_VERSION_OVERRIDE="-22" -fi -apt-get --allow-unauthenticated -y install socat percona-xtrabackup${PXB_VERSION_OVERRIDE} -apt-get --allow-unauthenticated -y install libmariadbclient18 mariadb-server - -cat >/etc/mysql/conf.d/no_perf_schema.cnf <<_EOF_ -[mysqld] -performance_schema = off -_EOF_ diff --git a/integration/scripts/files/elements/ubuntu-trusty-mongodb/element-deps b/integration/scripts/files/elements/ubuntu-trusty-mongodb/element-deps deleted file mode 100644 index 6a5964ec67..0000000000 --- a/integration/scripts/files/elements/ubuntu-trusty-mongodb/element-deps +++ /dev/null @@ -1 +0,0 @@ -ubuntu-mongodb diff --git a/integration/scripts/files/elements/ubuntu-trusty-mongodb/install.d/10-mongodb-thp b/integration/scripts/files/elements/ubuntu-trusty-mongodb/install.d/10-mongodb-thp deleted file mode 100755 index 48d6c8bf76..0000000000 --- a/integration/scripts/files/elements/ubuntu-trusty-mongodb/install.d/10-mongodb-thp +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/sh - -set -e -set -o xtrace - -export DEBIAN_FRONTEND=noninteractive - -cat > /etc/init.d/disable-transparent-hugepages << '_EOF_' -#!/bin/sh -### BEGIN INIT INFO -# Provides: disable-transparent-hugepages -# Required-Start: $local_fs -# Required-Stop: -# X-Start-Before: mongod mongodb-mms-automation-agent -# Default-Start: 2 3 4 5 -# Default-Stop: 0 1 6 -# Short-Description: Disable Linux transparent huge pages -# Description: Disable Linux transparent huge pages, to improve -# database performance. -### END INIT INFO - -case $1 in - start) - if [ -d /sys/kernel/mm/transparent_hugepage ]; then - thp_path=/sys/kernel/mm/transparent_hugepage - elif [ -d /sys/kernel/mm/redhat_transparent_hugepage ]; then - thp_path=/sys/kernel/mm/redhat_transparent_hugepage - else - return 0 - fi - - echo 'never' > ${thp_path}/enabled - echo 'never' > ${thp_path}/defrag - - unset thp_path - ;; -esac -_EOF_ - -chmod 755 /etc/init.d/disable-transparent-hugepages - -update-rc.d disable-transparent-hugepages defaults diff --git a/integration/scripts/files/elements/ubuntu-trusty-mongodb/install.d/20-mongodb b/integration/scripts/files/elements/ubuntu-trusty-mongodb/install.d/20-mongodb deleted file mode 100755 index 6320964fdd..0000000000 --- a/integration/scripts/files/elements/ubuntu-trusty-mongodb/install.d/20-mongodb +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh - -set -e -set -o xtrace - -export DEBIAN_FRONTEND=noninteractive - -apt-get -y install mongodb-org=3.2.11 diff --git a/integration/scripts/files/elements/ubuntu-trusty-mongodb/install.d/25-trove-mongo-dep b/integration/scripts/files/elements/ubuntu-trusty-mongodb/install.d/25-trove-mongo-dep deleted file mode 100755 index c0488bd0a5..0000000000 --- a/integration/scripts/files/elements/ubuntu-trusty-mongodb/install.d/25-trove-mongo-dep +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -# CONTEXT: GUEST during CONSTRUCTION as ROOT -# PURPOSE: Install trove guest python dependencies - see trovestack functions_qemu - -set -e -set -o xtrace - -pip3 install pymongo>=3.0.2,!=3.1 diff --git a/integration/scripts/files/elements/ubuntu-trusty-mongodb/install.d/30-mongodb-conf b/integration/scripts/files/elements/ubuntu-trusty-mongodb/install.d/30-mongodb-conf deleted file mode 100755 index 5ccf32591c..0000000000 --- a/integration/scripts/files/elements/ubuntu-trusty-mongodb/install.d/30-mongodb-conf +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/sh - -set -e -set -o xtrace - -# Remove the default pid file -rm -f /var/run/mongodb.pid - - -cat > /etc/mongod.conf << '_EOF_' -storage.dbPath: /var/lib/mongodb -security.authorization: enabled -storage.engine: wiredTiger -storage.journal.enabled: true -systemLog.destination: file -systemLog.logAppend: true -systemLog.path: /var/log/mongodb/mongod.log -_EOF_ - - -cat > /etc/mongos.conf << '_EOF_' -systemLog.destination: file -systemLog.logAppend: true -systemLog.path: /var/log/mongodb/mongos.log -_EOF_ diff --git a/integration/scripts/files/elements/ubuntu-trusty-mongodb/install.d/41-mongod-init b/integration/scripts/files/elements/ubuntu-trusty-mongodb/install.d/41-mongod-init deleted file mode 100755 index 77a1292723..0000000000 --- a/integration/scripts/files/elements/ubuntu-trusty-mongodb/install.d/41-mongod-init +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/sh - -set -e -set -o xtrace - -cat > /etc/init/mongod.conf << '_EOF_' -limit fsize unlimited unlimited -limit cpu unlimited unlimited -limit as unlimited unlimited -limit nofile 64000 64000 -limit rss unlimited unlimited -limit nproc 64000 64000 - -kill timeout 300 # wait 300s between SIGTERM and SIGKILL. - -pre-start script - mkdir -p /var/run/mongodb/ - touch /var/run/mongodb/mongod.pid - chown mongodb -R /var/run/mongodb/ -end script - -start on runlevel [2345] -stop on runlevel [06] - -script - CONF=/etc/mongod.conf - - # Handle NUMA access to CPUs (SERVER-3574) - # This verifies the existence of numactl as well as testing that the command works - NUMACTL_ARGS="--interleave=all" - if which numactl >/dev/null 2>/dev/null && numactl $NUMACTL_ARGS ls / >/dev/null 2>/dev/null - then - NUMACTL="$(which numactl) -- $NUMACTL_ARGS" - DAEMON_OPTS=${DAEMON_OPTS:-"--config $CONF"} - else - NUMACTL="" - DAEMON_OPTS="-- "${DAEMON_OPTS:-"--config $CONF"} - fi - - exec start-stop-daemon --start \ - --chuid mongodb \ - --pidfile /var/run/mongod.pid \ - --make-pidfile \ - --exec $NUMACTL /usr/bin/mongod $DAEMON_OPTS -end script -_EOF_ diff --git a/integration/scripts/files/elements/ubuntu-trusty-mongodb/install.d/42-mongos-init b/integration/scripts/files/elements/ubuntu-trusty-mongodb/install.d/42-mongos-init deleted file mode 100755 index dc91fdbac8..0000000000 --- a/integration/scripts/files/elements/ubuntu-trusty-mongodb/install.d/42-mongos-init +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/sh - -set -e -set -o xtrace - -cat > /etc/init/mongos.conf << '_EOF_' -limit fsize unlimited unlimited -limit cpu unlimited unlimited -limit as unlimited unlimited -limit nofile 64000 64000 -limit rss unlimited unlimited -limit nproc 64000 64000 - -pre-start script - mkdir -p /var/run/mongodb/ - touch /var/run/mongodb/mongos.pid - chown mongodb -R /var/run/mongodb/ -end script - -start on runlevel [2345] -stop on runlevel [06] - -script - exec start-stop-daemon --start \ - --chuid mongodb \ - --pidfile /var/run/mongos.pid \ - --make-pidfile \ - --exec /usr/bin/mongos -- --config /etc/mongos.conf -end script -_EOF_ diff --git a/integration/scripts/files/elements/ubuntu-trusty-mysql/element-deps b/integration/scripts/files/elements/ubuntu-trusty-mysql/element-deps deleted file mode 100644 index bd3447a6ec..0000000000 --- a/integration/scripts/files/elements/ubuntu-trusty-mysql/element-deps +++ /dev/null @@ -1 +0,0 @@ -ubuntu-mysql diff --git a/integration/scripts/files/elements/ubuntu-trusty-mysql/install.d/30-mysql b/integration/scripts/files/elements/ubuntu-trusty-mysql/install.d/30-mysql deleted file mode 100755 index 887bda2f9d..0000000000 --- a/integration/scripts/files/elements/ubuntu-trusty-mysql/install.d/30-mysql +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash - -# CONTEXT: GUEST during CONSTRUCTION as ROOT -# PURPOSE: Install controller base required packages - -set -e -set -o xtrace - -export DEBIAN_FRONTEND=noninteractive - -# The fix to make versions of percona-xtrabackup > v2.2 work with Trove -# was put into the mysql guestagent code for Mitaka. There are no current -# plans to backport so we need to make sure the guest generated when the -# tests are run for Kilo or Liberty get the 2.2 verson of PXB -if [[ $BRANCH_OVERRIDE == "stable/kilo" || $BRANCH_OVERRIDE == "stable/liberty" ]]; then - PXB_VERSION_OVERRIDE="-22" -fi -apt-get --allow-unauthenticated -y install libmysqlclient18 mysql-server-5.6 percona-xtrabackup${PXB_VERSION_OVERRIDE} - -cat >/etc/mysql/conf.d/no_perf_schema.cnf <<_EOF_ -[mysqld] -performance_schema = off -_EOF_ diff --git a/integration/scripts/files/elements/ubuntu-trusty-percona/element-deps b/integration/scripts/files/elements/ubuntu-trusty-percona/element-deps deleted file mode 100644 index bc5f9af6d0..0000000000 --- a/integration/scripts/files/elements/ubuntu-trusty-percona/element-deps +++ /dev/null @@ -1 +0,0 @@ -ubuntu-percona diff --git a/integration/scripts/files/elements/ubuntu-trusty-postgresql/element-deps b/integration/scripts/files/elements/ubuntu-trusty-postgresql/element-deps deleted file mode 100644 index 98e1bc19ee..0000000000 --- a/integration/scripts/files/elements/ubuntu-trusty-postgresql/element-deps +++ /dev/null @@ -1 +0,0 @@ -ubuntu-postgresql diff --git a/integration/scripts/files/elements/ubuntu-trusty-pxc/element-deps b/integration/scripts/files/elements/ubuntu-trusty-pxc/element-deps deleted file mode 100644 index 7b1a84c93a..0000000000 --- a/integration/scripts/files/elements/ubuntu-trusty-pxc/element-deps +++ /dev/null @@ -1 +0,0 @@ -ubuntu-pxc diff --git a/integration/scripts/files/elements/ubuntu-trusty-redis/element-deps b/integration/scripts/files/elements/ubuntu-trusty-redis/element-deps deleted file mode 100644 index 030d85baf2..0000000000 --- a/integration/scripts/files/elements/ubuntu-trusty-redis/element-deps +++ /dev/null @@ -1 +0,0 @@ -ubuntu-redis diff --git a/integration/scripts/files/elements/ubuntu-trusty-vertica/element-deps b/integration/scripts/files/elements/ubuntu-trusty-vertica/element-deps deleted file mode 100644 index 575094f02c..0000000000 --- a/integration/scripts/files/elements/ubuntu-trusty-vertica/element-deps +++ /dev/null @@ -1 +0,0 @@ -ubuntu-vertica diff --git a/integration/scripts/files/elements/ubuntu-xenial-guest/extra-data.d/20-guest-systemd b/integration/scripts/files/elements/ubuntu-xenial-guest/extra-data.d/20-guest-systemd index d541078c02..030efc8927 100755 --- a/integration/scripts/files/elements/ubuntu-xenial-guest/extra-data.d/20-guest-systemd +++ b/integration/scripts/files/elements/ubuntu-xenial-guest/extra-data.d/20-guest-systemd @@ -13,10 +13,9 @@ source $_LIB/die [ -n "${GUEST_USERNAME}" ] || die "GUEST_USERNAME needs to be set to the user for the guest image" [ -n "${HOST_SCP_USERNAME}" ] || die "HOST_SCP_USERNAME needs to be set to the user for the host instance" -[ -n "${CONTROLLER_IP}" ] || die "CONTROLLER_IP needs to be set to the ip address that guests will use to contact the controller" [ -n "${ESCAPED_PATH_TROVE}" ] || die "ESCAPED_PATH_TROVE needs to be set to the path to the trove directory on the trovestack host" [ -n "${TROVESTACK_SCRIPTS}" ] || die "TROVESTACK_SCRIPTS needs to be set to the trove/integration/scripts dir" [ -n "${ESCAPED_GUEST_LOGDIR}" ] || die "ESCAPED_GUEST_LOGDIR must be set to the escaped guest log dir" -sed "s/GUEST_USERNAME/${GUEST_USERNAME}/g;s/GUEST_LOGDIR/${ESCAPED_GUEST_LOGDIR}/g;s/HOST_SCP_USERNAME/${HOST_SCP_USERNAME}/g;s/CONTROLLER_IP/${CONTROLLER_IP}/g;s/PATH_TROVE/${ESCAPED_PATH_TROVE}/g" ${TROVESTACK_SCRIPTS}/files/trove-guest.systemd.conf > ${TMP_HOOKS_PATH}/trove-guest.service +sed "s/GUEST_USERNAME/${GUEST_USERNAME}/g;s/GUEST_LOGDIR/${ESCAPED_GUEST_LOGDIR}/g;s/HOST_SCP_USERNAME/${HOST_SCP_USERNAME}/g;s/PATH_TROVE/${ESCAPED_PATH_TROVE}/g" ${TROVESTACK_SCRIPTS}/files/trove-guest.systemd.conf > ${TMP_HOOKS_PATH}/trove-guest.service diff --git a/integration/scripts/files/trove-guest.systemd.conf b/integration/scripts/files/trove-guest.systemd.conf index b9ab6c7925..ac4182ecdc 100644 --- a/integration/scripts/files/trove-guest.systemd.conf +++ b/integration/scripts/files/trove-guest.systemd.conf @@ -8,15 +8,20 @@ Type=simple User=GUEST_USERNAME Group=GUEST_USERNAME +# This script is only for testing purpose for dev_mode=true, the controller +# IP address should be defined in /etc/trove/controller.conf, e.g. +# CONTROLLER=192.168.32.151 +EnvironmentFile=/etc/trove/controller.conf + ExecStartPre=/bin/bash -c "sudo mkdir -p GUEST_LOGDIR ; sudo chown GUEST_USERNAME:root GUEST_LOGDIR" # If ~/trove-installed does not exist, copy the trove source from # the user's development environment, then touch the sentinel file -ExecStartPre=/bin/bash -c "test -e /home/GUEST_USERNAME/trove-installed || sudo -u GUEST_USERNAME rsync -e 'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' -avz --exclude='.*' HOST_SCP_USERNAME@CONTROLLER_IP:PATH_TROVE/ /home/GUEST_USERNAME/trove && touch /home/GUEST_USERNAME/trove-installed" +ExecStartPre=/bin/bash -c "test -e /home/GUEST_USERNAME/trove-installed || sudo -u GUEST_USERNAME rsync -e 'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' -avz --exclude='.*' HOST_SCP_USERNAME@$CONTROLLER:PATH_TROVE/ /home/GUEST_USERNAME/trove && touch /home/GUEST_USERNAME/trove-installed" # If /etc/trove does not exist, create it and then copy the trove-guestagent.conf # from /etc/trove on the user's development environment, -ExecStartPre=/bin/bash -c "test -d /etc/trove/conf.d || sudo mkdir -p /etc/trove/conf.d && sudo -u GUEST_USERNAME rsync -e 'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' -avz --exclude='.*' HOST_SCP_USERNAME@CONTROLLER_IP:/etc/trove/trove-guestagent.conf ~GUEST_USERNAME/ && sudo mv ~GUEST_USERNAME/trove-guestagent.conf /etc/trove/conf.d/trove-guestagent.conf" +ExecStartPre=/bin/bash -c "test -d /etc/trove/conf.d || sudo mkdir -p /etc/trove/conf.d && sudo -u GUEST_USERNAME rsync -e 'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' -avz --exclude='.*' HOST_SCP_USERNAME@$CONTROLLER:/etc/trove/trove-guestagent.conf ~GUEST_USERNAME/ && sudo mv ~GUEST_USERNAME/trove-guestagent.conf /etc/trove/conf.d/trove-guestagent.conf" ExecStartPre=/bin/bash -c "sudo chown -R GUEST_USERNAME:root /etc/trove" diff --git a/integration/scripts/functions_qemu b/integration/scripts/functions_qemu index acc624401e..b59c4ad290 100644 --- a/integration/scripts/functions_qemu +++ b/integration/scripts/functions_qemu @@ -26,8 +26,6 @@ function build_vm() { # In dev mode, the trove guest agent needs to download trove code from # trove-taskmanager host during service initialization. if [[ "${dev_mode}" == "true" ]]; then - host_ip=$(ip route get 8.8.8.8 | head -1 | awk '{print $7}') - export CONTROLLER_IP=${CONTROLLER_IP:-${host_ip}} export PATH_TROVE=${PATH_TROVE} export ESCAPED_PATH_TROVE=$(echo ${PATH_TROVE} | sed 's/\//\\\//g') export GUEST_LOGDIR=${GUEST_LOGDIR:-"/var/log/trove/"} diff --git a/integration/scripts/trovestack b/integration/scripts/trovestack index ecc88f0ff7..e48d648d19 100755 --- a/integration/scripts/trovestack +++ b/integration/scripts/trovestack @@ -1050,6 +1050,7 @@ function cmd_int_tests() { args="$@" fi + # Referenced in test script export TROVE_TEST_SSH_USER=${TROVE_TEST_SSH_USER:-"ubuntu"} export TROVE_TEST_SSH_KEY_FILE=${TROVE_TEST_SSH_KEY_FILE:-"$HOME/.ssh/id_rsa"} @@ -1285,48 +1286,17 @@ function cmd_kick_start() { } function cmd_gate_tests() { - ACTUAL_HOSTNAME=$(hostname -I | sed 's/[0-9a-z][0-9a-z]*:.*:[0-9a-z][0-9a-z]*//g' | sed 's/[0-9]*\.[0-9]*\.[0-9]*\.1\b//g' | sed 's/ /\n/g' | sed '/^$/d' | sort -bu | head -1) - local DATASTORE_TYPE=${1:-'mysql'} local TEST_GROUP=${2:-${DATASTORE_TYPE}} local HOST_SCP_USERNAME=${3:-$(whoami)} local GUEST_USERNAME=${4:-'ubuntu'} - local CONTROLLER_IP=${5:-$ACTUAL_HOSTNAME} # We're not using devstack-gate in Zuul v3 job if [[ $GATE_JOB_VER == "v2" ]]; then - local ESCAPED_PATH_TROVE=${6:-'\/opt\/stack\/new\/trove'} + local ESCAPED_PATH_TROVE=${5:-'\/opt\/stack\/new\/trove'} fi exclaim "Running cmd_gate_tests ..." - # Sometimes in the gate the ACTUAL_HOSTNAME is blank - this code attempts to debug it - if [[ -z "${CONTROLLER_IP// }" ]]; then - echo "*** CONTROLLER_IP is blank, trying to determine actual hostname" - local hostname_part=$(hostname -I) - echo "Hostname pass 1: '$hostname_part'" - hostname_part=$(echo $hostname_part | sed 's/[0-9a-z][0-9a-z]*:.*:[0-9a-z][0-9a-z]*//g') - echo "Hostname pass 2: '$hostname_part'" - hostname_part_no_ip6=$hostname_part - hostname_part=$(echo $hostname_part | sed 's/[0-9]*\.[0-9]*\.[0-9]*\.1\b//g') - echo "Hostname pass 3: '$hostname_part'" - if [[ -z "${hostname_part// }" ]]; then - # This seems to occur when the actual hostname ends with '.1' - # If this happens, take the first one that doesn't start with '192' or '172' - hostname_part=$(echo $hostname_part_no_ip6 | sed 's/1[79]2\.[0-9]*\.[0-9]*\.1\b//g') - echo "Hostname pass 3a: '$hostname_part'" - fi - hostname_part=$(echo $hostname_part | sed 's/ /\n/g') - echo "Hostname pass 4: '$hostname_part'" - hostname_part=$(echo $hostname_part | sed '/^$/d') - echo "Hostname pass 5: '$hostname_part'" - hostname_part=$(echo $hostname_part | sort -bu) - echo "Hostname pass 6: '$hostname_part'" - hostname_part=$(echo $hostname_part | head -1) - echo "Hostname pass 7: '$hostname_part'" - CONTROLLER_IP=$hostname_part - echo "*** CONTROLLER_IP was blank (CONTROLLER_IP now set to '$CONTROLLER_IP')" - fi - export REPORT_DIRECTORY=${REPORT_DIRECTORY:=$HOME/gate-tests-report/} export TROVE_REPORT_DIR=$HOME/gate-tests-report/ TROVESTACK_DUMP_ENV=true @@ -1337,38 +1307,14 @@ function cmd_gate_tests() { # This var is used to ssh into the db instance during the test. export TROVE_TEST_SSH_KEY_FILE=${SSH_DIR}/id_rsa - # To avoid 'Connection timed out' error of sudo command - CLOUDINIT_PATH=/etc/trove/cloudinit/${DATASTORE_TYPE}.cloudinit - sudo mkdir -p $(dirname "$CLOUDINIT_PATH") - sudo touch "$CLOUDINIT_PATH" - sudo tee $CLOUDINIT_PATH >/dev/null <<'EOF' -#cloud-config -manage_etc_hosts: "localhost" -EOF - # Fix iptables rules that prevent amqp connections from the devstack box to the guests sudo iptables -D openstack-INPUT -j REJECT --reject-with icmp-host-prohibited || true sudo chown -R $(whoami) /etc/trove - iniset $TROVE_GUESTAGENT_CONF DEFAULT rabbit_host $CONTROLLER_IP - cd $TROVESTACK_SCRIPTS - sudo -H $HTTP_PROXY pip install --upgrade pip dib-utils local RESTART_TROVE=false cmd_kick_start "${DATASTORE_TYPE}" "${RESTART_TROVE}" - # TODO(zhaochao): in fact, devstack also use CONTROLLER_IP to construct the endpoint urls - # so we could safely remove the following lines in the future(they're kept now for not - # introducing too many changes. - if [[ $GATE_JOB_VER == "v2" ]]; then - # Update the local swift endpoint in the catalog to use the CONTROLLER_IP instead of 127.0.0.1 - SWIFT_ENDPOINTS=$(openstack $CLOUD_ADMIN_ARG endpoint list --service swift -c ID -f value) - openstack $CLOUD_ADMIN_ARG endpoint create swift public 'http://'$CONTROLLER_IP':8080/v1/AUTH_$(tenant_id)s' --region RegionOne - openstack $CLOUD_ADMIN_ARG endpoint create swift internal 'http://'$CONTROLLER_IP':8080/v1/AUTH_$(tenant_id)s' --region RegionOne - openstack $CLOUD_ADMIN_ARG endpoint create swift admin 'http://'$CONTROLLER_IP':8080' --region RegionOne - echo $SWIFT_ENDPOINTS | xargs -n 1 openstack $CLOUD_ADMIN_ARG endpoint delete - fi - cmd_int_tests --group=$TEST_GROUP } diff --git a/trove/guestagent/api.py b/trove/guestagent/api.py index 1f4a1d30d7..b199ce985a 100644 --- a/trove/guestagent/api.py +++ b/trove/guestagent/api.py @@ -101,7 +101,7 @@ class API(object): raise exception.GuestTimeout() def _cast(self, method_name, version, **kwargs): - LOG.debug("Casting %s", method_name) + LOG.debug("Calling %s asynchronously", method_name) try: with NotificationCastWrapper(self.context, 'guest'): cctxt = self.client.prepare(version=version) diff --git a/trove/instance/models.py b/trove/instance/models.py index 356fcc7d93..7ad357cb94 100644 --- a/trove/instance/models.py +++ b/trove/instance/models.py @@ -24,6 +24,7 @@ import six from novaclient import exceptions as nova_exceptions from oslo_config.cfg import NoSuchOptError from oslo_log import log as logging +from oslo_utils import encodeutils from sqlalchemy import func from trove.backup.models import Backup @@ -890,11 +891,10 @@ class BaseInstance(SimpleInstance): instance_key = get_instance_encryption_key(self.id) if instance_key: - files = {guest_info_file: ( - "%s" - "instance_rpc_encr_key=%s\n" % ( - files.get(guest_info_file), - instance_key))} + files = { + guest_info_file: ("%sinstance_rpc_encr_key=%s\n" % + (files.get(guest_info_file), instance_key)) + } if os.path.isfile(CONF.get('guest_config')): with open(CONF.get('guest_config'), "r") as f: @@ -1811,6 +1811,10 @@ class instance_encryption_key_cache(object): if val is None: return val + # We need string anyway + if isinstance(val, six.binary_type): + val = encodeutils.safe_decode(val) + if len(self._lru) == self._lru_cache_size: tail = self._lru.pop() del self._table[tail]