From ff6e6d19f9f882776773bc8532c008c2e6c98f4f Mon Sep 17 00:00:00 2001 From: Pete Birley Date: Thu, 10 Aug 2017 19:05:36 -0500 Subject: [PATCH] Gate: Heat functional test This PS adds a functional test of heat in the OSH gates, it also moves most params to a seperate file - making config of the gate in non-zuul environments simpler. Change-Id: I37a1bc0dcc8000c5da8067a8d376c78f7cd6f7ab --- tools/gate/dump_logs.sh | 2 + .../gate/files/heat-basic-vm-deployment.yaml | 82 ++++++++++++ .../files/heat-public-net-deployment.yaml | 43 ++++++ .../files/heat-subnet-pool-deployment.yaml | 24 ++++ tools/gate/funcs/openstack.sh | 24 ++++ tools/gate/helm_dry_run.sh | 3 +- tools/gate/kubeadm_aio.sh | 3 +- .../armada.sh} | 30 +---- .../{basic_launch.sh => launch-osh/basic.sh} | 21 +-- tools/gate/launch-osh/common.sh | 37 ++++++ tools/gate/openstack/network_launch.sh | 78 +++++++++++ tools/gate/openstack/vm_cli_launch.sh | 91 +++++++++++++ tools/gate/openstack/vm_heat_launch.sh | 67 ++++++++++ tools/gate/openstack_aio_launch.sh | 124 ------------------ tools/gate/provision_gate_worker_node.sh | 5 +- tools/gate/setup_gate.sh | 75 ++++------- tools/gate/setup_gate_worker_nodes.sh | 6 +- tools/gate/vars.sh | 77 +++++++++++ 18 files changed, 566 insertions(+), 226 deletions(-) create mode 100644 tools/gate/files/heat-basic-vm-deployment.yaml create mode 100644 tools/gate/files/heat-public-net-deployment.yaml create mode 100644 tools/gate/files/heat-subnet-pool-deployment.yaml rename tools/gate/{armada_launch.sh => launch-osh/armada.sh} (69%) mode change 100644 => 100755 rename tools/gate/{basic_launch.sh => launch-osh/basic.sh} (88%) create mode 100755 tools/gate/launch-osh/common.sh create mode 100755 tools/gate/openstack/network_launch.sh create mode 100755 tools/gate/openstack/vm_cli_launch.sh create mode 100755 tools/gate/openstack/vm_heat_launch.sh delete mode 100644 tools/gate/openstack_aio_launch.sh create mode 100755 tools/gate/vars.sh diff --git a/tools/gate/dump_logs.sh b/tools/gate/dump_logs.sh index 4c1ce3cbf8..4f7f492c3e 100755 --- a/tools/gate/dump_logs.sh +++ b/tools/gate/dump_logs.sh @@ -86,6 +86,7 @@ echo "${NODE_NAME}" > ${LOGS_DIR}/nodes/master.txt sudo docker logs kubelet 2> ${LOGS_DIR}/nodes/${NODE_NAME}/kubelet.txt sudo docker logs kubeadm-aio 2>&1 > ${LOGS_DIR}/nodes/${NODE_NAME}/kubeadm-aio.txt sudo docker images --digests --no-trunc --all > ${LOGS_DIR}/nodes/${NODE_NAME}/images.txt +sudo du -h --max-depth=1 /var/lib/docker | sort -hr > ${LOGS_DIR}/nodes/${NODE_NAME}/docker-size.txt sudo iptables-save > ${LOGS_DIR}/nodes/${NODE_NAME}/iptables.txt sudo ip a > ${LOGS_DIR}/nodes/${NODE_NAME}/ip.txt sudo route -n > ${LOGS_DIR}/nodes/${NODE_NAME}/routes.txt @@ -102,6 +103,7 @@ if [ "x$INTEGRATION" == "xmulti" ]; then ssh -i ${SSH_PRIVATE_KEY} $(whoami)@${NODE_IP} sudo docker logs kubelet 2> ${LOGS_DIR}/nodes/${NODE_NAME}/kubelet.txt ssh -i ${SSH_PRIVATE_KEY} $(whoami)@${NODE_IP} sudo docker logs kubeadm-aio 2>&1 > ${LOGS_DIR}/nodes/${NODE_NAME}/kubeadm-aio.txt ssh -i ${SSH_PRIVATE_KEY} $(whoami)@${NODE_IP} sudo docker images --digests --no-trunc --all > ${LOGS_DIR}/nodes/${NODE_NAME}/images.txt + ssh -i ${SSH_PRIVATE_KEY} $(whoami)@${NODE_IP} sudo du -h --max-depth=1 /var/lib/docker | sort -hr > ${LOGS_DIR}/nodes/${NODE_NAME}/docker-size.txt ssh -i ${SSH_PRIVATE_KEY} $(whoami)@${NODE_IP} sudo iptables-save > ${LOGS_DIR}/nodes/${NODE_NAME}/iptables.txt ssh -i ${SSH_PRIVATE_KEY} $(whoami)@${NODE_IP} sudo ip a > ${LOGS_DIR}/nodes/${NODE_NAME}/ip.txt ssh -i ${SSH_PRIVATE_KEY} $(whoami)@${NODE_IP} sudo route -n > ${LOGS_DIR}/nodes/${NODE_NAME}/routes.txt diff --git a/tools/gate/files/heat-basic-vm-deployment.yaml b/tools/gate/files/heat-basic-vm-deployment.yaml new file mode 100644 index 0000000000..de60e0a45c --- /dev/null +++ b/tools/gate/files/heat-basic-vm-deployment.yaml @@ -0,0 +1,82 @@ +heat_template_version: 2016-10-14 + +parameters: + public_net: + type: string + default: public + image: + type: string + default: Cirros 0.3.5 64-bit + flavor: + type: string + default: m1.tiny + ssh_key: + type: string + default: heat-vm-key + cidr: + type: string + default: 10.11.11.0/24 + +resources: + server: + type: OS::Nova::Server + properties: + image: {get_param: image} + flavor: {get_param: flavor} + key_name: {get_param: ssh_key} + networks: + - port: { get_resource: server_port } + user_data_format: RAW + + router: + type: OS::Neutron::Router + properties: + external_gateway_info: + network: {get_param: public_net} + + router_interface: + type: OS::Neutron::RouterInterface + properties: + router_id: { get_resource: router } + subnet_id: { get_resource: private_subnet } + + private_net: + type: OS::Neutron::Net + + private_subnet: + type: OS::Neutron::Subnet + properties: + network: { get_resource: private_net } + cidr: {get_param: cidr} + dns_nameservers: + - 8.8.8.8 + - 8.8.4.4 + + port_security_group: + type: OS::Neutron::SecurityGroup + properties: + name: default_port_security_group + description: > + Default security group assigned to port. + rules: [ + {remote_ip_prefix: 0.0.0.0/0, + protocol: tcp, + port_range_min: 22, + port_range_max: 22}, + {remote_ip_prefix: 0.0.0.0/0, + protocol: icmp}] + + server_port: + type: OS::Neutron::Port + properties: + network: {get_resource: private_net} + fixed_ips: + - subnet: { get_resource: private_subnet } + security_groups: + - { get_resource: port_security_group } + + server_floating_ip: + type: OS::Neutron::FloatingIP + properties: + floating_network: {get_param: public_net} + port_id: { get_resource: server_port } diff --git a/tools/gate/files/heat-public-net-deployment.yaml b/tools/gate/files/heat-public-net-deployment.yaml new file mode 100644 index 0000000000..386de5722a --- /dev/null +++ b/tools/gate/files/heat-public-net-deployment.yaml @@ -0,0 +1,43 @@ +heat_template_version: 2016-10-14 + +parameters: + network_name: + type: string + default: public + + physical_network_name: + type: string + default: public + + subnet_name: + type: string + default: public + + subnet_cidr: + type: string + default: 172.24.4.0/24 + + subnet_gateway: + type: string + default: 172.24.4.1 + +resources: + public_net: + type: OS::Neutron::ProviderNet + properties: + name: {get_param: network_name} + router_external: true + physical_network: {get_param: physical_network_name} + network_type: flat + + private_subnet: + type: OS::Neutron::Subnet + properties: + name: {get_param: subnet_name} + network: { get_resource: public_net } + cidr: {get_param: subnet_cidr} + gateway_ip: {get_param: subnet_gateway} + enable_dhcp: false + dns_nameservers: + - 8.8.8.8 + - 8.8.4.4 diff --git a/tools/gate/files/heat-subnet-pool-deployment.yaml b/tools/gate/files/heat-subnet-pool-deployment.yaml new file mode 100644 index 0000000000..69cdf729c6 --- /dev/null +++ b/tools/gate/files/heat-subnet-pool-deployment.yaml @@ -0,0 +1,24 @@ +heat_template_version: 2016-10-14 + +parameters: + subnet_pool_name: + type: string + default: shared-default-subnetpool + + subnet_pool_prefixes: + type: comma_delimited_list + default: ["10.0.0.0/8"] + + subnet_pool_default_prefix_length: + type: number + default: 24 + +resources: + public_net: + type: OS::Neutron::SubnetPool + properties: + name: {get_param: subnet_pool_name} + shared: true + is_default: true + default_prefixlen: {get_param: subnet_pool_default_prefix_length} + prefixes: {get_param: subnet_pool_prefixes} diff --git a/tools/gate/funcs/openstack.sh b/tools/gate/funcs/openstack.sh index fee2282d30..56560d607b 100755 --- a/tools/gate/funcs/openstack.sh +++ b/tools/gate/funcs/openstack.sh @@ -26,6 +26,9 @@ KEYSTONE_CREDS="--os-username ${KS_USER} \ --os-project-domain-name ${KS_PROJECT_DOMAIN} \ --os-user-domain-name ${KS_USER_DOMAIN} \ --os-password ${KS_PASSWORD}" + +HEAT_POD=$(kubectl get -n openstack pods -l application=heat,component=engine --no-headers -o name | awk -F '/' '{ print $NF; exit }') +HEAT="kubectl exec -n openstack ${HEAT_POD} -- heat ${KEYSTONE_CREDS}" NEUTRON_POD=$(kubectl get -n openstack pods -l application=heat,component=engine --no-headers -o name | awk -F '/' '{ print $NF; exit }') NEUTRON="kubectl exec -n openstack ${NEUTRON_POD} -- neutron ${KEYSTONE_CREDS}" NOVA_POD=$(kubectl get -n openstack pods -l application=heat,component=engine --no-headers -o name | awk -F '/' '{ print $NF; exit }') @@ -94,3 +97,24 @@ function wait_for_ssh_port { done set -x } + +function openstack_wait_for_stack { + # Default wait timeout is 180 seconds + set +x + end=$(date +%s) + if ! [ -z $2 ]; then + end=$((end + $2)) + else + end=$((end + 180)) + fi + while true; do + STATUS=$($OPENSTACK stack show $1 -f value -c stack_status) + [ $STATUS == "CREATE_COMPLETE" ] && \ + break || true + sleep 1 + now=$(date +%s) + [ $now -gt $end ] && echo Stack failed to start. && \ + $OPENSTACK stack show $1 && exit -1 + done + set -x +} diff --git a/tools/gate/helm_dry_run.sh b/tools/gate/helm_dry_run.sh index f6cbb899ec..3021286a59 100755 --- a/tools/gate/helm_dry_run.sh +++ b/tools/gate/helm_dry_run.sh @@ -12,7 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. set -ex - +: ${WORK_DIR:="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"} +source ${WORK_DIR}/tools/gate/vars.sh source ${WORK_DIR}/tools/gate/funcs/helm.sh helm_build diff --git a/tools/gate/kubeadm_aio.sh b/tools/gate/kubeadm_aio.sh index a9d937ff16..97c8b74e47 100755 --- a/tools/gate/kubeadm_aio.sh +++ b/tools/gate/kubeadm_aio.sh @@ -12,7 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. set -ex - +: ${WORK_DIR:="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"} +source ${WORK_DIR}/tools/gate/vars.sh source ${WORK_DIR}/tools/gate/funcs/network.sh source ${WORK_DIR}/tools/gate/funcs/kube.sh diff --git a/tools/gate/armada_launch.sh b/tools/gate/launch-osh/armada.sh old mode 100644 new mode 100755 similarity index 69% rename from tools/gate/armada_launch.sh rename to tools/gate/launch-osh/armada.sh index 890dc6016c..b657fd9226 --- a/tools/gate/armada_launch.sh +++ b/tools/gate/launch-osh/armada.sh @@ -12,38 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. set -ex -: ${WORK_DIR:="$(pwd)"} +: ${WORK_DIR:="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/.."} +source ${WORK_DIR}/tools/gate/vars.sh source ${WORK_DIR}/tools/gate/funcs/helm.sh source ${WORK_DIR}/tools/gate/funcs/kube.sh source ${WORK_DIR}/tools/gate/funcs/network.sh -# NOTE(portdirect): Temp workaround until module loading is supported by -# OpenStack-Helm in Fedora -if [ "x$HOST_OS" == "xfedora" ]; then - sudo modprobe openvswitch - sudo modprobe gre - sudo modprobe vxlan - sudo modprobe ip6_tables -fi - -helm install --namespace=openstack ${WORK_DIR}/dns-helper --name=dns-helper -kube_wait_for_pods openstack 180 - -if ! [ "x$PVC_BACKEND" == "xceph" ]; then - echo "ARMADA LAUNCH only supports ceph currently" - exit 1 -fi - -kubectl label nodes ceph-mon=enabled --all -kubectl label nodes ceph-osd=enabled --all -kubectl label nodes ceph-mds=enabled --all -CONTROLLER_MANAGER_POD=$(kubectl get -n kube-system pods -l component=kube-controller-manager --no-headers -o name | awk -F '/' '{ print $NF; exit }') -kubectl exec -n kube-system ${CONTROLLER_MANAGER_POD} -- sh -c "cat > /etc/resolv.conf < /proc/sys/net/ipv4/ip_forward" +fi + +# Assign IP address to br-ex +sudo ip addr add ${OSH_BR_EX_ADDR} dev br-ex +sudo ip link set br-ex up +# Setup masquerading on default route dev to public subnet +sudo iptables -t nat -A POSTROUTING -o $(net_default_iface) -s ${OSH_EXT_SUBNET} -j MASQUERADE + +# Disable In-Band rules on br-ex bridge to ease debugging +OVS_VSWITCHD_POD=$(kubectl get -n openstack pods -l application=neutron,component=ovs-vswitchd --no-headers -o name | head -1 | awk -F '/' '{ print $NF }') +kubectl exec -n openstack ${OVS_VSWITCHD_POD} -- ovs-vsctl set Bridge br-ex other_config:disable-in-band=true + + +if ! $OPENSTACK service list -f value -c Type | grep -q orchestration; then + echo "No orchestration service active: creating public network via CLI" + $NEUTRON net-create ${OSH_EXT_NET_NAME} -- --is-default \ + --router:external \ + --provider:network_type=flat \ + --provider:physical_network=public + $NEUTRON subnet-create \ + --name ${OSH_EXT_SUBNET_NAME} \ + --ip-version 4 \ + $($NEUTRON net-show ${OSH_EXT_NET_NAME} -f value -c id) ${OSH_EXT_SUBNET} -- \ + --enable_dhcp=False + + # Create default subnet pool + $NEUTRON subnetpool-create \ + ${OSH_PRIVATE_SUBNET_POOL_NAME} \ + --default-prefixlen ${OSH_PRIVATE_SUBNET_POOL_DEF_PREFIX} \ + --pool-prefix ${OSH_PRIVATE_SUBNET_POOL} \ + --shared \ + --is-default=True +else + echo "Orchestration service active: creating public network via Heat" + HEAT_TEMPLATE=$(cat ${WORK_DIR}/tools/gate/files/${OSH_PUB_NET_STACK}.yaml | base64 -w 0) + kubectl exec -n openstack ${OPENSTACK_POD} -- bash -c "echo $HEAT_TEMPLATE | base64 -d > /tmp/${OSH_PUB_NET_STACK}.yaml" + $OPENSTACK stack create \ + --parameter network_name=${OSH_EXT_NET_NAME} \ + --parameter physical_network_name=public \ + --parameter subnet_name=${OSH_EXT_SUBNET_NAME} \ + --parameter subnet_cidr=${OSH_EXT_SUBNET} \ + --parameter subnet_gateway=${OSH_BR_EX_ADDR%/*} \ + -t /tmp/${OSH_PUB_NET_STACK}.yaml \ + ${OSH_PUB_NET_STACK} + openstack_wait_for_stack ${OSH_PUB_NET_STACK} + + HEAT_TEMPLATE=$(cat ${WORK_DIR}/tools/gate/files/${OSH_SUBNET_POOL_STACK}.yaml | base64 -w 0) + kubectl exec -n openstack ${OPENSTACK_POD} -- bash -c "echo $HEAT_TEMPLATE | base64 -d > /tmp/${OSH_SUBNET_POOL_STACK}.yaml" + $OPENSTACK stack create \ + --parameter subnet_pool_name=${OSH_PRIVATE_SUBNET_POOL_NAME} \ + --parameter subnet_pool_prefixes=${OSH_PRIVATE_SUBNET_POOL} \ + --parameter subnet_pool_default_prefix_length=${OSH_PRIVATE_SUBNET_POOL_DEF_PREFIX} \ + -t /tmp/${OSH_SUBNET_POOL_STACK}.yaml \ + ${OSH_SUBNET_POOL_STACK} + openstack_wait_for_stack ${OSH_SUBNET_POOL_STACK} +fi diff --git a/tools/gate/openstack/vm_cli_launch.sh b/tools/gate/openstack/vm_cli_launch.sh new file mode 100755 index 0000000000..29dec0edca --- /dev/null +++ b/tools/gate/openstack/vm_cli_launch.sh @@ -0,0 +1,91 @@ +#!/bin/bash +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -xe +: ${WORK_DIR:="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/.."} +source ${WORK_DIR}/tools/gate/vars.sh +source ${WORK_DIR}/tools/gate/funcs/network.sh +source ${WORK_DIR}/tools/gate/funcs/openstack.sh + +# Create default private network +$NEUTRON net-create ${OSH_PRIVATE_NET_NAME} +$NEUTRON subnet-create \ + --name ${OSH_PRIVATE_SUBNET_NAME} \ + --ip-version 4 \ + --dns-nameserver ${OSH_EXT_DNS} \ + $($NEUTRON net-show private -f value -c id) \ + ${OSH_PRIVATE_SUBNET} + +# Create default router and link networks +$NEUTRON router-create ${OSH_ROUTER} +$NEUTRON router-interface-add \ + $($NEUTRON router-show ${OSH_ROUTER} -f value -c id) \ + $($NEUTRON subnet-show private-subnet -f value -c id) +$NEUTRON router-gateway-set \ + $($NEUTRON router-show ${OSH_ROUTER} -f value -c id) \ + $($NEUTRON net-show ${OSH_EXT_NET_NAME} -f value -c id) + +ROUTER_PUBLIC_IP=$($NEUTRON router-show ${OSH_ROUTER} -f value -c external_gateway_info | jq -r '.external_fixed_ips[].ip_address') +wait_for_ping ${ROUTER_PUBLIC_IP} + +# Loosen up security group to allow access to the VM +PROJECT=$($OPENSTACK project show admin -f value -c id) +SECURITY_GROUP=$($OPENSTACK security group list -f csv | grep ${PROJECT} | grep "default" | awk -F "," '{ print $1 }' | tr -d '"') +$OPENSTACK security group rule create ${SECURITY_GROUP} \ + --protocol icmp \ + --src-ip 0.0.0.0/0 +$OPENSTACK security group rule create ${SECURITY_GROUP} \ + --protocol tcp \ + --dst-port 22:22 \ + --src-ip 0.0.0.0/0 + +# Setup SSH Keypair in Nova +KEYPAIR_LOC="$(mktemp).pem" +$OPENSTACK keypair create ${OSH_VM_KEY_CLI} > ${KEYPAIR_LOC} +chmod 600 ${KEYPAIR_LOC} + +# Boot a vm and wait for it to become active +FLAVOR=$($OPENSTACK flavor show "${OSH_VM_FLAVOR}" -f value -c id) +IMAGE=$($OPENSTACK image list -f csv | awk -F ',' '{ print $2 "," $1 }' | grep "^\"Cirros" | head -1 | awk -F ',' '{ print $2 }' | tr -d '"') +NETWORK=$($NEUTRON net-show ${OSH_PRIVATE_NET_NAME} -f value -c id) +$NOVA boot \ + --nic net-id=${NETWORK} \ + --flavor=${FLAVOR} \ + --image=${IMAGE} \ + --key-name=${OSH_VM_KEY_CLI} \ + --security-groups="default" \ + ${OSH_VM_NAME_CLI} +openstack_wait_for_vm ${OSH_VM_NAME_CLI} + +# Assign a floating IP to the VM +FLOATING_IP=$($OPENSTACK floating ip create ${OSH_EXT_NET_NAME} -f value -c floating_ip_address) +$OPENSTACK server add floating ip ${OSH_VM_NAME_CLI} ${FLOATING_IP} + +# Ping our VM +wait_for_ping ${FLOATING_IP} ${SERVICE_TEST_TIMEOUT} + +# Wait for SSH to come up +wait_for_ssh_port ${FLOATING_IP} ${SERVICE_TEST_TIMEOUT} + +# SSH into the VM and check it can reach the outside world +ssh-keyscan "$FLOATING_IP" >> ~/.ssh/known_hosts +ssh -i ${KEYPAIR_LOC} cirros@${FLOATING_IP} ping -q -c 1 -W 2 ${OSH_BR_EX_ADDR%/*} + +# SSH into the VM and check it can reach the metadata server +ssh -i ${KEYPAIR_LOC} cirros@${FLOATING_IP} curl -sSL 169.254.169.254 + +# Bonus round - display a Unicorn +ssh -i ${KEYPAIR_LOC} cirros@${FLOATING_IP} curl http://artscene.textfiles.com/asciiart/unicorn || true + +# Remove the test vm +$NOVA delete ${OSH_VM_NAME_CLI} diff --git a/tools/gate/openstack/vm_heat_launch.sh b/tools/gate/openstack/vm_heat_launch.sh new file mode 100755 index 0000000000..1e85591fd1 --- /dev/null +++ b/tools/gate/openstack/vm_heat_launch.sh @@ -0,0 +1,67 @@ +#!/bin/bash +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -xe +: ${WORK_DIR:="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/.."} +source ${WORK_DIR}/tools/gate/vars.sh +source ${WORK_DIR}/tools/gate/funcs/network.sh +source ${WORK_DIR}/tools/gate/funcs/openstack.sh + +# Setup SSH Keypair in Nova +KEYPAIR_LOC="$(mktemp).pem" +$OPENSTACK keypair create ${OSH_VM_KEY_STACK} > ${KEYPAIR_LOC} +chmod 600 ${KEYPAIR_LOC} + +# NOTE(portdirect): We do this fancy, and seemingly pointless, footwork to get +# the full image name for the cirros Image without having to be explicit. +IMAGE_NAME=$($OPENSTACK image show -f value -c name \ + $($OPENSTACK image list -f csv | awk -F ',' '{ print $2 "," $1 }' | \ + grep "^\"Cirros" | head -1 | awk -F ',' '{ print $2 }' | tr -d '"')) + +HEAT_TEMPLATE=$(cat ${WORK_DIR}/tools/gate/files/${OSH_BASIC_VM_STACK}.yaml | base64 -w 0) +kubectl exec -n openstack ${OPENSTACK_POD} -- bash -c "echo $HEAT_TEMPLATE | base64 -d > /tmp/${OSH_BASIC_VM_STACK}.yaml" +$OPENSTACK stack create \ + --parameter public_net=${OSH_EXT_NET_NAME} \ + --parameter image="${IMAGE_NAME}" \ + --parameter flavor=${OSH_VM_FLAVOR} \ + --parameter ssh_key=${OSH_VM_KEY_STACK} \ + --parameter cidr=${OSH_PRIVATE_SUBNET} \ + -t /tmp/${OSH_BASIC_VM_STACK}.yaml \ + ${OSH_BASIC_VM_STACK} +openstack_wait_for_stack ${OSH_BASIC_VM_STACK} ${SERVICE_TEST_TIMEOUT} + +FLOATING_IP=$($OPENSTACK floating ip show \ + $($OPENSTACK stack resource show \ + ${OSH_BASIC_VM_STACK} \ + server_floating_ip \ + -f value -c physical_resource_id) \ + -f value -c floating_ip_address) + +# Ping our VM +wait_for_ping ${FLOATING_IP} ${SERVICE_TEST_TIMEOUT} + +# Wait for SSH to come up +wait_for_ssh_port ${FLOATING_IP} ${SERVICE_TEST_TIMEOUT} + +# SSH into the VM and check it can reach the outside world +ssh-keyscan "$FLOATING_IP" >> ~/.ssh/known_hosts +ssh -i ${KEYPAIR_LOC} cirros@${FLOATING_IP} ping -q -c 1 -W 2 ${OSH_BR_EX_ADDR%/*} + +# SSH into the VM and check it can reach the metadata server +ssh -i ${KEYPAIR_LOC} cirros@${FLOATING_IP} curl -sSL 169.254.169.254 + +# Bonus round - display a Unicorn +ssh -i ${KEYPAIR_LOC} cirros@${FLOATING_IP} curl http://artscene.textfiles.com/asciiart/unicorn || true + +# Remove the test stack +$OPENSTACK stack delete ${OSH_BASIC_VM_STACK} diff --git a/tools/gate/openstack_aio_launch.sh b/tools/gate/openstack_aio_launch.sh deleted file mode 100644 index a1dfa74c34..0000000000 --- a/tools/gate/openstack_aio_launch.sh +++ /dev/null @@ -1,124 +0,0 @@ -#!/bin/bash -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -set -xe - -: ${OSH_BR_EX_ADDR:="172.24.4.1/24"} -: ${OSH_EXT_SUBNET:="172.24.4.0/24"} -: ${OSH_EXT_DNS:="8.8.8.8"} -: ${OSH_EXT_NET_NAME:="public"} -: ${OSH_EXT_SUBNET_NAME:="public-subnet"} -: ${OSH_ROUTER:="router1"} -: ${OSH_PRIVATE_NET_NAME:="private"} -: ${OSH_PRIVATE_SUBNET:="10.0.0.0/24"} -: ${OSH_PRIVATE_SUBNET_NAME:="private-subnet"} -: ${OSH_PRIVATE_SUBNET_POOL:="10.0.0.0/8"} -: ${OSH_PRIVATE_SUBNET_POOL_NAME:="shared-default-subnetpool"} -: ${OSH_PRIVATE_SUBNET_POOL_DEF_PREFIX:="24"} -: ${OSH_VM_NAME:="osh-smoketest"} -: ${OSH_VM_KEY:="osh-smoketest-key"} - -# Source some functions that will help us -source ${WORK_DIR}/tools/gate/funcs/network.sh -source ${WORK_DIR}/tools/gate/funcs/openstack.sh - -# Turn on ip forwarding if its not already -if [ $(cat /proc/sys/net/ipv4/ip_forward) -eq 0 ]; then - sudo bash -c "echo 1 > /proc/sys/net/ipv4/ip_forward" -fi - -# Assign IP address to br-ex -sudo ip addr add ${OSH_BR_EX_ADDR} dev br-ex -sudo ip link set br-ex up -# Setup masquerading on default route dev to public subnet -sudo iptables -t nat -A POSTROUTING -o $(net_default_iface) -s ${OSH_EXT_SUBNET} -j MASQUERADE - -# Disable In-Band rules on br-ex bridge to ease debugging -OVS_VSWITCHD_POD=$(kubectl get -n openstack pods -l application=neutron,component=ovs-vswitchd --no-headers -o name | head -1 | awk -F '/' '{ print $NF }') -kubectl exec -n openstack ${OVS_VSWITCHD_POD} -- ovs-vsctl set Bridge br-ex other_config:disable-in-band=true - -# Create default networks -$NEUTRON net-create ${OSH_PRIVATE_NET_NAME} -$NEUTRON subnet-create \ - --name ${OSH_PRIVATE_SUBNET_NAME} \ - --ip-version 4 \ - --dns-nameserver ${OSH_EXT_DNS} \ - $($NEUTRON net-show private -f value -c id) \ - ${OSH_PRIVATE_SUBNET} -$NEUTRON router-create ${OSH_ROUTER} -$NEUTRON subnetpool-create \ - ${OSH_PRIVATE_SUBNET_POOL_NAME} \ - --default-prefixlen ${OSH_PRIVATE_SUBNET_POOL_DEF_PREFIX} \ - --pool-prefix ${OSH_PRIVATE_SUBNET_POOL} \ - --shared \ - --is-default=True -$NEUTRON net-create ${OSH_EXT_NET_NAME} -- --is-default \ - --router:external \ - --provider:network_type=flat \ - --provider:physical_network=public -$NEUTRON router-interface-add $($NEUTRON router-show ${OSH_ROUTER} -f value -c id) $($NEUTRON subnet-show private-subnet -f value -c id) -$NEUTRON subnet-create \ - --name ${OSH_EXT_SUBNET_NAME} \ - --ip-version 4 \ - $($NEUTRON net-show ${OSH_EXT_NET_NAME} -f value -c id) ${OSH_EXT_SUBNET} -- --enable_dhcp=False -$NEUTRON router-gateway-set $($NEUTRON router-show ${OSH_ROUTER} -f value -c id) $($NEUTRON net-show ${OSH_EXT_NET_NAME} -f value -c id) - -ROUTER_PUBLIC_IP=$($NEUTRON router-show ${OSH_ROUTER} -f value -c external_gateway_info | jq -r '.external_fixed_ips[].ip_address') -wait_for_ping ${ROUTER_PUBLIC_IP} - -# Loosen up security group to allow access to the VM -PROJECT=$($OPENSTACK project show admin -f value -c id) -SECURITY_GROUP=$($OPENSTACK security group list -f csv | grep ${PROJECT} | grep "default" | awk -F "," '{ print $1 }' | tr -d '"') -$OPENSTACK security group rule create ${SECURITY_GROUP} --protocol icmp --src-ip 0.0.0.0/0 -$OPENSTACK security group rule create ${SECURITY_GROUP} --protocol tcp --dst-port 22:22 --src-ip 0.0.0.0/0 - -# Setup SSH Keypair in Nova -KEYPAIR_LOC="$(mktemp).pem" -$OPENSTACK keypair create ${OSH_VM_KEY} > ${KEYPAIR_LOC} -chmod 600 ${KEYPAIR_LOC} - -# Boot a vm and wait for it to become active -FLAVOR=$($OPENSTACK flavor show "m1.tiny" -f value -c id) -IMAGE=$($OPENSTACK image list -f csv | awk -F ',' '{ print $2 "," $1 }' | grep "^\"Cirros" | head -1 | awk -F ',' '{ print $2 }' | tr -d '"') -NETWORK=$($NEUTRON net-show private -f value -c id) -$NOVA boot \ - --nic net-id=${NETWORK} \ - --flavor=${FLAVOR} \ - --image=${IMAGE} \ - --key-name=${OSH_VM_KEY} \ - --security-groups="default" \ - ${OSH_VM_NAME} -openstack_wait_for_vm ${OSH_VM_NAME} - -# Assign a floating IP to the VM -FLOATING_IP=$($OPENSTACK floating ip create ${OSH_EXT_NET_NAME} -f value -c floating_ip_address) -$OPENSTACK server add floating ip ${OSH_VM_NAME} ${FLOATING_IP} - -# Ping our VM -wait_for_ping ${FLOATING_IP} - -# Wait for SSH to come up -wait_for_ssh_port ${FLOATING_IP} - -# SSH into the VM and check it can reach the outside world -ssh-keyscan "$FLOATING_IP" >> ~/.ssh/known_hosts -ssh -i ${KEYPAIR_LOC} cirros@${FLOATING_IP} ping -q -c 1 -W 2 ${OSH_BR_EX_ADDR%/*} - -# SSH into the VM and check it can reach the metadata server -ssh -i ${KEYPAIR_LOC} cirros@${FLOATING_IP} curl -sSL 169.254.169.254 - -# Bonus round - display a Unicorn -ssh -i ${KEYPAIR_LOC} cirros@${FLOATING_IP} curl http://artscene.textfiles.com/asciiart/unicorn || true - -# Remove the test vm -$NOVA delete ${OSH_VM_NAME} diff --git a/tools/gate/provision_gate_worker_node.sh b/tools/gate/provision_gate_worker_node.sh index 0e834bfda0..d59e67719d 100755 --- a/tools/gate/provision_gate_worker_node.sh +++ b/tools/gate/provision_gate_worker_node.sh @@ -12,10 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. set -ex - +: ${WORK_DIR:="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"} +source ${WORK_DIR}/tools/gate/vars.sh cd ${WORK_DIR} -source /etc/os-release -export HOST_OS=${ID} source ${WORK_DIR}/tools/gate/funcs/common.sh source ${WORK_DIR}/tools/gate/funcs/network.sh source ${WORK_DIR}/tools/gate/funcs/kube.sh diff --git a/tools/gate/setup_gate.sh b/tools/gate/setup_gate.sh index 1acaa82c13..00fbabf4ed 100755 --- a/tools/gate/setup_gate.sh +++ b/tools/gate/setup_gate.sh @@ -12,35 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. set -ex - -export HELM_VERSION=${HELM_VERSION:-"v2.5.1"} -export KUBE_VERSION=${KUBE_VERSION:-"v1.6.8"} -export PVC_BACKEND=${PVC_BACKEND:-"ceph"} -export UPSTREAM_DNS=${UPSTREAM_DNS:-"8.8.8.8"} - -export SERVICE_LAUNCH_TIMEOUT=${SERVICE_LAUNCH_TIMEOUT:="600"} -export SERVICE_TEST_TIMEOUT=${SERVICE_TEST_TIMEOUT:="600"} - -export KUBECONFIG=${HOME}/.kubeadm-aio/admin.conf - -export LOOPBACK_CREATE=${LOOPBACK_CREATE:="false"} -export LOOPBACK_DEVS=${LOOPBACK_DEVS:="3"} -export LOOPBACK_SIZE=${LOOPBACK_SIZE:="500M"} -export LOOPBACK_DIR=${LOOPBACK_DIR:="/var/lib/iscsi-loopback"} - -export KUBEADM_IMAGE=openstackhelm/kubeadm-aio:${KUBE_VERSION}-dev -export CNI_POD_CIDR=${CNI_POD_CIDR:="192.168.0.0/16"} -export KUBE_CNI=${KUBE_CNI:="calico"} - export WORK_DIR=$(pwd) -source /etc/os-release -export HOST_OS=${ID} +source ${WORK_DIR}/tools/gate/vars.sh source ${WORK_DIR}/tools/gate/funcs/common.sh source ${WORK_DIR}/tools/gate/funcs/network.sh source ${WORK_DIR}/tools/gate/funcs/helm.sh # Setup the logging location: by default use the working dir as the root. -export LOGS_DIR=${LOGS_DIR:-"${WORK_DIR}/logs"} rm -rf ${LOGS_DIR} || true mkdir -p ${LOGS_DIR} @@ -73,29 +51,34 @@ if [ "x$INTEGRATION_TYPE" == "xlinter" ]; then helm_build > ${LOGS_DIR}/helm_build helm_plugin_template_install helm_template_run -fi - -# Setup the K8s Cluster -if [ "x$INTEGRATION" == "xaio" ]; then - bash ${WORK_DIR}/tools/gate/kubeadm_aio.sh -elif [ "x$INTEGRATION" == "xmulti" ]; then - bash ${WORK_DIR}/tools/gate/kubeadm_aio.sh - bash ${WORK_DIR}/tools/gate/setup_gate_worker_nodes.sh -fi - -# Deploy OpenStack-Helm -if [ "x$INTEGRATION_TYPE" == "xbasic" ]; then - bash ${WORK_DIR}/tools/gate/helm_dry_run.sh - bash ${WORK_DIR}/tools/gate/basic_launch.sh -elif [ "x$INTEGRATION_TYPE" == "xarmada" ]; then - bash ${WORK_DIR}/tools/gate/armada_launch.sh -fi - -if ! [ "x$INTEGRATION_TYPE" == "x" ]; then - # Run Basic Full Stack Tests +else + # Setup the K8s Cluster if [ "x$INTEGRATION" == "xaio" ]; then - bash ${WORK_DIR}/tools/gate/openstack_aio_launch.sh + bash ${WORK_DIR}/tools/gate/kubeadm_aio.sh + elif [ "x$INTEGRATION" == "xmulti" ]; then + bash ${WORK_DIR}/tools/gate/kubeadm_aio.sh + bash ${WORK_DIR}/tools/gate/setup_gate_worker_nodes.sh + fi + + # Deploy OpenStack-Helm + if ! [ "x$INTEGRATION_TYPE" == "x" ]; then + bash ${WORK_DIR}/tools/gate/helm_dry_run.sh + bash ${WORK_DIR}/tools/gate/launch-osh/common.sh + if [ "x$INTEGRATION_TYPE" == "xbasic" ]; then + bash ${WORK_DIR}/tools/gate/launch-osh/basic.sh + elif [ "x$INTEGRATION_TYPE" == "xarmada" ]; then + bash ${WORK_DIR}/tools/gate/launch-osh/armada.sh + fi + fi + + if ! [ "x$INTEGRATION_TYPE" == "x" ]; then + # Run Basic Full Stack Tests + if [ "x$INTEGRATION" == "xaio" ]; then + bash ${WORK_DIR}/tools/gate/openstack/network_launch.sh + bash ${WORK_DIR}/tools/gate/openstack/vm_cli_launch.sh + bash ${WORK_DIR}/tools/gate/openstack/vm_heat_launch.sh + fi + # Collect all logs from the environment + bash ${WORK_DIR}/tools/gate/dump_logs.sh 0 fi - # Collect all logs from the environment - bash ${WORK_DIR}/tools/gate/dump_logs.sh 0 fi diff --git a/tools/gate/setup_gate_worker_nodes.sh b/tools/gate/setup_gate_worker_nodes.sh index 51cdad57de..9bdde5195f 100755 --- a/tools/gate/setup_gate_worker_nodes.sh +++ b/tools/gate/setup_gate_worker_nodes.sh @@ -12,10 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. set -ex - -: ${SSH_PRIVATE_KEY:="/etc/nodepool/id_rsa"} -: ${PRIMARY_NODE_IP:="$(cat /etc/nodepool/primary_node | tail -1)"} -: ${SUB_NODE_IPS:="$(cat /etc/nodepool/sub_nodes)"} +: ${WORK_DIR:="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"} +source ${WORK_DIR}/tools/gate/vars.sh export SUB_NODE_COUNT="$(($(echo ${SUB_NODE_IPS} | wc -w) + 1))" sudo chown $(whoami) ${SSH_PRIVATE_KEY} diff --git a/tools/gate/vars.sh b/tools/gate/vars.sh new file mode 100755 index 0000000000..c6178b2368 --- /dev/null +++ b/tools/gate/vars.sh @@ -0,0 +1,77 @@ +#!/bin/bash +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Set work dir if not already done +: ${WORK_DIR:="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"} + +# Set logs directory +export LOGS_DIR=${LOGS_DIR:-"${WORK_DIR}/logs"} + +# Get Host OS +source /etc/os-release +export HOST_OS=${HOST_OS:="${ID}"} + +# Set versions of K8s and Helm to use +export HELM_VERSION=${HELM_VERSION:-"v2.5.1"} +export KUBE_VERSION=${KUBE_VERSION:-"v1.6.8"} + +# Set K8s-AIO options +export KUBECONFIG=${KUBECONFIG:="${HOME}/.kubeadm-aio/admin.conf"} +export KUBEADM_IMAGE=${KUBEADM_IMAGE:="openstackhelm/kubeadm-aio:${KUBE_VERSION}"} + +# Set K8s network options +export CNI_POD_CIDR=${CNI_POD_CIDR:="192.168.0.0/16"} +export KUBE_CNI=${KUBE_CNI:="calico"} + +# Set PVC Backend +export PVC_BACKEND=${PVC_BACKEND:-"ceph"} + +# Set Upstream DNS +export UPSTREAM_DNS=${UPSTREAM_DNS:-"8.8.8.8"} + +# Set gate script timeouts +export SERVICE_LAUNCH_TIMEOUT=${SERVICE_LAUNCH_TIMEOUT:="600"} +export SERVICE_TEST_TIMEOUT=${SERVICE_TEST_TIMEOUT:="600"} + +# Setup Loopback device options +export LOOPBACK_CREATE=${LOOPBACK_CREATE:="false"} +export LOOPBACK_DEVS=${LOOPBACK_DEVS:="3"} +export LOOPBACK_SIZE=${LOOPBACK_SIZE:="500M"} +export LOOPBACK_DIR=${LOOPBACK_DIR:="/var/lib/iscsi-loopback"} + +# Setup Multinode params +export SSH_PRIVATE_KEY=${SSH_PRIVATE_KEY:="/etc/nodepool/id_rsa"} +export PRIMARY_NODE_IP=${PRIMARY_NODE_IP:="$(cat /etc/nodepool/primary_node | tail -1)"} +export SUB_NODE_IPS=${SUB_NODE_IPS:="$(cat /etc/nodepool/sub_nodes)"} + +# Define OpenStack Test Params +export OSH_BR_EX_ADDR=${OSH_BR_EX_ADDR:="172.24.4.1/24"} +export OSH_EXT_SUBNET=${OSH_EXT_SUBNET:="172.24.4.0/24"} +export OSH_EXT_DNS=${OSH_EXT_DNS:="8.8.8.8"} +export OSH_EXT_NET_NAME=${OSH_EXT_NET_NAME:="public"} +export OSH_EXT_SUBNET_NAME=${OSH_EXT_SUBNET_NAME:="public-subnet"} +export OSH_ROUTER=${OSH_ROUTER:="router1"} +export OSH_PRIVATE_NET_NAME=${OSH_PRIVATE_NET_NAME:="private"} +export OSH_PRIVATE_SUBNET=${OSH_PRIVATE_SUBNET:="10.0.0.0/24"} +export OSH_PRIVATE_SUBNET_NAME=${OSH_PRIVATE_SUBNET_NAME:="private-subnet"} +export OSH_PRIVATE_SUBNET_POOL=${OSH_PRIVATE_SUBNET_POOL:="10.0.0.0/8"} +export OSH_PRIVATE_SUBNET_POOL_NAME=${OSH_PRIVATE_SUBNET_POOL_NAME:="shared-default-subnetpool"} +export OSH_PRIVATE_SUBNET_POOL_DEF_PREFIX=${OSH_PRIVATE_SUBNET_POOL_DEF_PREFIX:="24"} +export OSH_VM_FLAVOR=${OSH_VM_FLAVOR:="m1.tiny"} +export OSH_VM_NAME_CLI=${OSH_VM_NAME_CLI:="osh-smoketest"} +export OSH_VM_KEY_CLI=${OSH_VM_KEY_CLI:="osh-smoketest-key"} +export OSH_PUB_NET_STACK=${OSH_PUB_NET_STACK:="heat-public-net-deployment"} +export OSH_SUBNET_POOL_STACK=${OSH_SUBNET_POOL_STACK:="heat-subnet-pool-deployment"} +export OSH_BASIC_VM_STACK=${OSH_BASIC_VM_STACK:="heat-basic-vm-deployment"} +export OSH_VM_KEY_STACK=${OSH_VM_KEY_STACK:="heat-vm-key"}