From b098d117b49774a34b9708d86047485265d6aacc Mon Sep 17 00:00:00 2001 From: Wenzhi Yu Date: Mon, 5 Sep 2016 18:00:18 +0800 Subject: [PATCH] Set up etcd in devstack We intend to use etcd as data store for Zun, so we need to set up etcd server and client in devstack. Part of blueprint etcd-db-driver Change-Id: I63bf91bc0cdb894b53c4f8385c7d3d651035831a --- devstack/etcd.conf | 22 ++++++++ devstack/etcd.service | 15 +++++ devstack/etcd.service.conf | 41 ++++++++++++++ devstack/lib/zun | 113 +++++++++++++++++++++++++++++++++++-- devstack/settings | 1 + 5 files changed, 186 insertions(+), 6 deletions(-) create mode 100644 devstack/etcd.conf create mode 100644 devstack/etcd.service create mode 100644 devstack/etcd.service.conf diff --git a/devstack/etcd.conf b/devstack/etcd.conf new file mode 100644 index 000000000..fbc2b38b3 --- /dev/null +++ b/devstack/etcd.conf @@ -0,0 +1,22 @@ +description "etcd distributed key-value store" +author "Scott Lowe " + +start on (net-device-up + and local-filesystems + and runlevel [2345]) +stop on runlevel [016] + +respawn +respawn limit 10 5 + +script + if [ -f "/etc/default/etcd" ]; then + . /etc/default/etcd + fi + +if [ ! -d "/var/etcd" ]; then + mkdir /var/etcd +fi +cd /var/etcd +exec /usr/local/bin/etcd >>/var/log/etcd.log 2>&1 +end script diff --git a/devstack/etcd.service b/devstack/etcd.service new file mode 100644 index 000000000..0444feed3 --- /dev/null +++ b/devstack/etcd.service @@ -0,0 +1,15 @@ +[Unit] +Description=Etcd Server +After=network.target + +[Service] +Type=notify +WorkingDirectory=/var/lib/etcd/ +EnvironmentFile=-/etc/etcd/etcd.conf +#User=etcd +ExecStart=/usr/local/bin/etcd +Restart=on-failure +LimitNOFILE=65536 + +[Install] +WantedBy=multi-user.target diff --git a/devstack/etcd.service.conf b/devstack/etcd.service.conf new file mode 100644 index 000000000..0b1e4b8ad --- /dev/null +++ b/devstack/etcd.service.conf @@ -0,0 +1,41 @@ +# [member] +#ETCD_NAME=default +#ETCD_DATA_DIR="/var/lib/etcd/default.etcd" +#ETCD_SNAPSHOT_COUNTER="10000" +#ETCD_HEARTBEAT_INTERVAL="100" +#ETCD_ELECTION_TIMEOUT="1000" +#ETCD_LISTEN_PEER_URLS="http://localhost:2380" +#ETCD_LISTEN_CLIENT_URLS="http://localhost:2379" +#ETCD_MAX_SNAPSHOTS="5" +#ETCD_MAX_WALS="5" +#ETCD_CORS="" +# +#[cluster] +#ETCD_INITIAL_ADVERTISE_PEER_URLS="http://localhost:2380" +# if you use different ETCD_NAME (e.g. test), set ETCD_INITIAL_CLUSTER value for this name, i.e. "test=http://..." +#ETCD_INITIAL_CLUSTER="default=http://localhost:2380" +#ETCD_INITIAL_CLUSTER_STATE="new" +#ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster" +#ETCD_ADVERTISE_CLIENT_URLS="http://localhost:2379" +#ETCD_DISCOVERY="" +#ETCD_DISCOVERY_SRV="" +#ETCD_DISCOVERY_FALLBACK="proxy" +#ETCD_DISCOVERY_PROXY="" +# +#[proxy] +#ETCD_PROXY="off" +# +#[security] +#ETCD_CERT_FILE="" +#ETCD_KEY_FILE="" +#ETCD_CLIENT_CERT_AUTH="false" +#ETCD_TRUSTED_CA_FILE="" +#ETCD_PEER_CERT_FILE="" +#ETCD_PEER_KEY_FILE="" +#ETCD_PEER_CLIENT_CERT_AUTH="false" +#ETCD_PEER_TRUSTED_CA_FILE="" +# +#[logging] +#ETCD_DEBUG="false" +# examples for -log-package-levels etcdserver=WARNING,security=DEBUG +#ETCD_LOG_PACKAGE_LEVELS="" diff --git a/devstack/lib/zun b/devstack/lib/zun index 93138378c..5a7bfd311 100644 --- a/devstack/lib/zun +++ b/devstack/lib/zun @@ -66,6 +66,12 @@ fi DOCKER_GROUP=docker DEFAULT_CONTAINER_DRIVER=docker + +ETCD_VERSION=v3.0.7 +if is_ubuntu; then + UBUNTU_RELEASE_BASE_NUM=`lsb_release -r | awk '{print $2}' | cut -d '.' -f 1` +fi + # Functions # --------- @@ -218,14 +224,21 @@ function create_zun_cache_dir { # init_zun() - Initialize databases, etc. function init_zun { # Only do this step once on the API node for an entire cluster. - if is_service_enabled $DATABASE_BACKENDS && is_service_enabled zun-api; then - # (Re)create zun database - recreate_database zun + if is_service_enabled zun-api; then + if is_service_enabled $DATABASE_BACKENDS; then + # (Re)create zun database + recreate_database zun - # Migrate zun database - $ZUN_BIN_DIR/zun-db-manage upgrade + # Migrate zun database + $ZUN_BIN_DIR/zun-db-manage upgrade + fi + + if is_service_enabled zun-etcd; then + install_etcd_server + install_etcd_client + fi + create_zun_cache_dir fi - create_zun_cache_dir } # install_zunclient() - Collect source and prepare @@ -242,6 +255,82 @@ function install_zun { setup_develop $ZUN_DIR } +function configure_etcd_systemd_service { + ETCD_CONF_DIR=/etc/etcd + ETCD_CONF=$ETCD_CONF_DIR/etcd.conf + ETCD_DATA_DIR="/var/lib/etcd" + # Delete any existing etcd database: + sudo rm -rf $ETCD_DATA_DIR + sudo mkdir -p $ETCD_DATA_DIR + + sudo rm -rf $ETCD_CONF_DIR + sudo install -d -o $STACK_USER $ETCD_CONF_DIR + cp $ZUN_DIR/devstack/etcd.service.conf $ETCD_CONF + + iniset $ETCD_CONF DEFAULT ETCD_INITIAL_CLUSTER "$HOSTNAME=http://$HOST_IP:2380" + iniset $ETCD_CONF DEFAULT ETCD_INITIAL_CLUSTER_STATE "new" + iniset $ETCD_CONF DEFAULT ETCD_INITIAL_CLUSTER_TOKEN "etcd-cluster-01" + iniset $ETCD_CONF DEFAULT ETCD_INITIAL_ADVERTISE_PEER_URLS "http://$HOST_IP:2380" + iniset $ETCD_CONF DEFAULT ETCD_DATA_DIR "$ETCD_DATA_DIR" + iniset $ETCD_CONF DEFAULT ETCD_LISTEN_PEER_URLS "http://0.0.0.0:2380" + iniset $ETCD_CONF DEFAULT ETCD_LISTEN_CLIENT_URLS "http://$HOST_IP:2379" + iniset $ETCD_CONF DEFAULT ETCD_ADVERTISE_CLIENT_URLS "http://$HOST_IP:2379" + iniset $ETCD_CONF DEFAULT ETCD_NAME "$HOSTNAME" + + sudo cp $ZUN_DIR/devstack/etcd.service /lib/systemd/system/ + sudo systemctl enable etcd +} + +function configure_etcd { + sudo cp $ZUN_DIR/devstack/etcd.conf /etc/init/etcd.conf + # Delete any existing etcd database: + sudo rm -rf /var/etcd + + OVERRIDE_FILE=$ZUN_DIR/devstack/etcd.override + sudo rm -f $OVERRIDE_FILE + +cat < $OVERRIDE_FILE +# Override file for etcd Upstart script providing some environment variables +env ETCD_INITIAL_CLUSTER="$HOSTNAME=http://$HOST_IP:2380" +env ETCD_INITIAL_CLUSTER_STATE="new" +env ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster-01" +env ETCD_INITIAL_ADVERTISE_PEER_URLS="http://$HOST_IP:2380" +env ETCD_DATA_DIR="/var/etcd" +env ETCD_LISTEN_PEER_URLS="http://0.0.0.0:2380" +env ETCD_LISTEN_CLIENT_URLS="http://$HOST_IP:2379" +env ETCD_ADVERTISE_CLIENT_URLS="http://$HOST_IP:2379" +env ETCD_NAME="$HOSTNAME" +EOF + + sudo cp $OVERRIDE_FILE /etc/init/etcd.override +} + +function install_etcd_server { + echo "Installing etcd" + if [ ! -f "$ZUN_DIR/etcd/etcd-$ETCD_VERSION-linux-amd64/etcd" ]; then + mkdir -p $ZUN_DIR/etcd + curl -L https://github.com/coreos/etcd/releases/download/$ETCD_VERSION/etcd-$ETCD_VERSION-linux-amd64.tar.gz -o $ZUN_DIR/etcd/etcd-$ETCD_VERSION-linux-amd64.tar.gz + tar xzvf $ZUN_DIR/etcd/etcd-$ETCD_VERSION-linux-amd64.tar.gz -C $ZUN_DIR/etcd + sudo cp $ZUN_DIR/etcd/etcd-$ETCD_VERSION-linux-amd64/etcd /usr/local/bin/etcd + fi + if [ ! -f "/usr/local/bin/etcd" ]; then + sudo cp $DEST/etcd/etcd-$ETCD_VERSION-linux-amd64/etcd /usr/local/bin/etcd + fi + if is_ubuntu; then + if [ $UBUNTU_RELEASE_BASE_NUM -ge 16 ] ; then + configure_etcd_systemd_service + else + configure_etcd + fi + elif is_fedora; then + configure_etcd_systemd_service + fi +} + +function install_etcd_client { + sudo pip install python-etcd +} + # start_zun_api() - Start the API process ahead of other things function start_zun_api { # Get right service port for testing @@ -273,12 +362,24 @@ function start_zun_compute { run_process zun-compute "$ZUN_BIN_DIR/zun-compute" ${DOCKER_GROUP} } +function start_zun_etcd { + echo "Start zun etcd..." + if is_ubuntu; then + sudo service etcd start || true + elif is_fedora; then + sudo systemctl start etcd || true + fi +} + # start_zun() - Start running processes, including screen function start_zun { # ``run_process`` checks ``is_service_enabled``, it is not needed here start_zun_api start_zun_compute + if is_service_enabled zun-etcd; then + start_zun_etcd + fi } # stop_zun() - Stop running processes (non-screen) diff --git a/devstack/settings b/devstack/settings index 036c72190..751ff5180 100644 --- a/devstack/settings +++ b/devstack/settings @@ -18,3 +18,4 @@ # Enable Zun services enable_service zun-api enable_service zun-compute +enable_service zun-etcd