Gate: Heat functional test

This PS adds a functional test of heat in the OSH gates, it also moves
most params to a seperate file - making config of the gate in non-zuul
environments simpler.

Change-Id: I37a1bc0dcc8000c5da8067a8d376c78f7cd6f7ab

tools/gate/dump_logs.sh

@@ -86,6 +86,7 @@ echo "${NODE_NAME}" > ${LOGS_DIR}/nodes/master.txt
 sudo docker logs kubelet 2> ${LOGS_DIR}/nodes/${NODE_NAME}/kubelet.txt
 sudo docker logs kubeadm-aio 2>&1 > ${LOGS_DIR}/nodes/${NODE_NAME}/kubeadm-aio.txt
 sudo docker images --digests --no-trunc --all > ${LOGS_DIR}/nodes/${NODE_NAME}/images.txt
+sudo du -h --max-depth=1 /var/lib/docker | sort -hr > ${LOGS_DIR}/nodes/${NODE_NAME}/docker-size.txt
 sudo iptables-save > ${LOGS_DIR}/nodes/${NODE_NAME}/iptables.txt
 sudo ip a > ${LOGS_DIR}/nodes/${NODE_NAME}/ip.txt
 sudo route -n > ${LOGS_DIR}/nodes/${NODE_NAME}/routes.txt
@@ -102,6 +103,7 @@ if [ "x$INTEGRATION" == "xmulti" ]; then
     ssh -i ${SSH_PRIVATE_KEY} $(whoami)@${NODE_IP} sudo docker logs kubelet 2> ${LOGS_DIR}/nodes/${NODE_NAME}/kubelet.txt
     ssh -i ${SSH_PRIVATE_KEY} $(whoami)@${NODE_IP} sudo docker logs kubeadm-aio 2>&1 > ${LOGS_DIR}/nodes/${NODE_NAME}/kubeadm-aio.txt
     ssh -i ${SSH_PRIVATE_KEY} $(whoami)@${NODE_IP} sudo docker images --digests --no-trunc --all > ${LOGS_DIR}/nodes/${NODE_NAME}/images.txt
+    ssh -i ${SSH_PRIVATE_KEY} $(whoami)@${NODE_IP} sudo du -h --max-depth=1 /var/lib/docker | sort -hr > ${LOGS_DIR}/nodes/${NODE_NAME}/docker-size.txt
     ssh -i ${SSH_PRIVATE_KEY} $(whoami)@${NODE_IP} sudo iptables-save > ${LOGS_DIR}/nodes/${NODE_NAME}/iptables.txt
     ssh -i ${SSH_PRIVATE_KEY} $(whoami)@${NODE_IP} sudo ip a > ${LOGS_DIR}/nodes/${NODE_NAME}/ip.txt
     ssh -i ${SSH_PRIVATE_KEY} $(whoami)@${NODE_IP} sudo route -n > ${LOGS_DIR}/nodes/${NODE_NAME}/routes.txt

tools/gate/files/heat-basic-vm-deployment.yaml

@@ -0,0 +1,82 @@
+heat_template_version: 2016-10-14
+
+parameters:
+  public_net:
+    type: string
+    default: public
+  image:
+    type: string
+    default: Cirros 0.3.5 64-bit
+  flavor:
+    type: string
+    default: m1.tiny
+  ssh_key:
+    type: string
+    default: heat-vm-key
+  cidr:
+    type: string
+    default: 10.11.11.0/24
+
+resources:
+  server:
+    type: OS::Nova::Server
+    properties:
+      image: {get_param: image}
+      flavor: {get_param: flavor}
+      key_name: {get_param: ssh_key}
+      networks:
+        - port: { get_resource: server_port }
+      user_data_format: RAW
+
+  router:
+    type: OS::Neutron::Router
+    properties:
+      external_gateway_info:
+        network: {get_param: public_net}
+
+  router_interface:
+    type: OS::Neutron::RouterInterface
+    properties:
+      router_id: { get_resource: router }
+      subnet_id: { get_resource: private_subnet }
+
+  private_net:
+    type: OS::Neutron::Net
+
+  private_subnet:
+    type: OS::Neutron::Subnet
+    properties:
+      network: { get_resource: private_net }
+      cidr: {get_param: cidr}
+      dns_nameservers:
+        - 8.8.8.8
+        - 8.8.4.4
+
+  port_security_group:
+    type: OS::Neutron::SecurityGroup
+    properties:
+      name: default_port_security_group
+      description: >
+        Default security group assigned to port.
+      rules: [
+        {remote_ip_prefix: 0.0.0.0/0,
+        protocol: tcp,
+        port_range_min: 22,
+        port_range_max: 22},
+        {remote_ip_prefix: 0.0.0.0/0,
+        protocol: icmp}]
+
+  server_port:
+    type: OS::Neutron::Port
+    properties:
+      network: {get_resource: private_net}
+      fixed_ips:
+        - subnet: { get_resource: private_subnet }
+      security_groups:
+        - { get_resource: port_security_group }
+
+  server_floating_ip:
+    type: OS::Neutron::FloatingIP
+    properties:
+      floating_network: {get_param: public_net}
+      port_id: { get_resource: server_port }

tools/gate/files/heat-public-net-deployment.yaml

@@ -0,0 +1,43 @@
+heat_template_version: 2016-10-14
+
+parameters:
+  network_name:
+    type: string
+    default: public
+
+  physical_network_name:
+    type: string
+    default: public
+
+  subnet_name:
+    type: string
+    default: public
+
+  subnet_cidr:
+    type: string
+    default: 172.24.4.0/24
+
+  subnet_gateway:
+    type: string
+    default: 172.24.4.1
+
+resources:
+  public_net:
+    type: OS::Neutron::ProviderNet
+    properties:
+      name: {get_param: network_name}
+      router_external: true
+      physical_network: {get_param: physical_network_name}
+      network_type: flat
+
+  private_subnet:
+    type: OS::Neutron::Subnet
+    properties:
+      name: {get_param: subnet_name}
+      network: { get_resource: public_net }
+      cidr: {get_param: subnet_cidr}
+      gateway_ip: {get_param: subnet_gateway}
+      enable_dhcp: false
+      dns_nameservers:
+        - 8.8.8.8
+        - 8.8.4.4

tools/gate/files/heat-subnet-pool-deployment.yaml

@@ -0,0 +1,24 @@
+heat_template_version: 2016-10-14
+
+parameters:
+  subnet_pool_name:
+    type: string
+    default: shared-default-subnetpool
+
+  subnet_pool_prefixes:
+    type: comma_delimited_list
+    default: ["10.0.0.0/8"]
+
+  subnet_pool_default_prefix_length:
+    type: number
+    default: 24
+
+resources:
+  public_net:
+    type: OS::Neutron::SubnetPool
+    properties:
+      name: {get_param: subnet_pool_name}
+      shared: true
+      is_default: true
+      default_prefixlen: {get_param: subnet_pool_default_prefix_length}
+      prefixes: {get_param: subnet_pool_prefixes}

tools/gate/funcs/openstack.sh

@@ -26,6 +26,9 @@ KEYSTONE_CREDS="--os-username ${KS_USER} \
   --os-project-domain-name ${KS_PROJECT_DOMAIN} \
   --os-user-domain-name ${KS_USER_DOMAIN} \
   --os-password ${KS_PASSWORD}"
+
+HEAT_POD=$(kubectl get -n openstack pods -l application=heat,component=engine --no-headers -o name | awk -F '/' '{ print $NF; exit }')
+HEAT="kubectl exec -n openstack ${HEAT_POD} -- heat ${KEYSTONE_CREDS}"
 NEUTRON_POD=$(kubectl get -n openstack pods -l application=heat,component=engine --no-headers -o name | awk -F '/' '{ print $NF; exit }')
 NEUTRON="kubectl exec -n openstack ${NEUTRON_POD} -- neutron ${KEYSTONE_CREDS}"
 NOVA_POD=$(kubectl get -n openstack pods -l application=heat,component=engine --no-headers -o name | awk -F '/' '{ print $NF; exit }')
@@ -94,3 +97,24 @@ function wait_for_ssh_port {
   done
   set -x
 }
+
+function openstack_wait_for_stack {
+  # Default wait timeout is 180 seconds
+  set +x
+  end=$(date +%s)
+  if ! [ -z $2 ]; then
+   end=$((end + $2))
+  else
+   end=$((end + 180))
+  fi
+  while true; do
+      STATUS=$($OPENSTACK stack show $1 -f value -c stack_status)
+      [ $STATUS == "CREATE_COMPLETE" ] && \
+          break || true
+      sleep 1
+      now=$(date +%s)
+      [ $now -gt $end ] && echo Stack failed to start. && \
+          $OPENSTACK stack show $1 && exit -1
+  done
+  set -x
+}

tools/gate/helm_dry_run.sh

@@ -12,7 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 set -ex
-
+: ${WORK_DIR:="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"}
+source ${WORK_DIR}/tools/gate/vars.sh
 source ${WORK_DIR}/tools/gate/funcs/helm.sh
 
 helm_build

tools/gate/kubeadm_aio.sh

@@ -12,7 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 set -ex
-
+: ${WORK_DIR:="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"}
+source ${WORK_DIR}/tools/gate/vars.sh
 source ${WORK_DIR}/tools/gate/funcs/network.sh
 source ${WORK_DIR}/tools/gate/funcs/kube.sh
 

tools/gate/launch-osh/armada.sh

@@ -12,38 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 set -ex
-: ${WORK_DIR:="$(pwd)"}
+: ${WORK_DIR:="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/.."}
+source ${WORK_DIR}/tools/gate/vars.sh
 source ${WORK_DIR}/tools/gate/funcs/helm.sh
 source ${WORK_DIR}/tools/gate/funcs/kube.sh
 source ${WORK_DIR}/tools/gate/funcs/network.sh
 
-# NOTE(portdirect): Temp workaround until module loading is supported by
-# OpenStack-Helm in Fedora
-if [ "x$HOST_OS" == "xfedora" ]; then
-  sudo modprobe openvswitch
-  sudo modprobe gre
-  sudo modprobe vxlan
-  sudo modprobe ip6_tables
-fi
-
-helm install --namespace=openstack ${WORK_DIR}/dns-helper --name=dns-helper
-kube_wait_for_pods openstack 180
-
-if ! [ "x$PVC_BACKEND" == "xceph" ]; then
-  echo "ARMADA LAUNCH only supports ceph currently"
-  exit 1
-fi
-
-kubectl label nodes ceph-mon=enabled --all
-kubectl label nodes ceph-osd=enabled --all
-kubectl label nodes ceph-mds=enabled --all
-CONTROLLER_MANAGER_POD=$(kubectl get -n kube-system pods -l component=kube-controller-manager --no-headers -o name | awk -F '/' '{ print $NF; exit }')
-kubectl exec -n kube-system ${CONTROLLER_MANAGER_POD} -- sh -c "cat > /etc/resolv.conf <<EOF
-nameserver 10.96.0.10
-nameserver ${UPSTREAM_DNS}
-search cluster.local svc.cluster.local
-EOF"
-
 ARMADA_MANIFEST=$(mktemp --suffix=.yaml)
 if [ "x$INTEGRATION" == "xaio" ]; then
   SUBNET_RANGE=$(find_subnet_range)

tools/gate/launch-osh/basic.sh

@@ -12,9 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 set -ex
-: ${WORK_DIR:="$(pwd)"}
+: ${WORK_DIR:="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/.."}
-: ${SERVICE_LAUNCH_TIMEOUT:="600"}
+source ${WORK_DIR}/tools/gate/vars.sh
-: ${SERVICE_TEST_TIMEOUT:="600"}
 source ${WORK_DIR}/tools/gate/funcs/helm.sh
 source ${WORK_DIR}/tools/gate/funcs/kube.sh
 source ${WORK_DIR}/tools/gate/funcs/network.sh
@@ -23,23 +22,7 @@ helm_build
 
 helm search
 
-# NOTE(portdirect): Temp workaround until module loading is supported by
-# OpenStack-Helm in Fedora
-if [ "x$HOST_OS" == "xfedora" ]; then
-  sudo modprobe openvswitch
-  sudo modprobe gre
-  sudo modprobe vxlan
-  sudo modprobe ip6_tables
-fi
-
-helm install --namespace=openstack ${WORK_DIR}/dns-helper --name=dns-helper
-kube_wait_for_pods openstack ${SERVICE_LAUNCH_TIMEOUT}
-
 if [ "x$PVC_BACKEND" == "xceph" ]; then
-  kubectl label nodes ceph-mon=enabled --all
-  kubectl label nodes ceph-osd=enabled --all
-  kubectl label nodes ceph-mds=enabled --all
-
   if [ "x$INTEGRATION" == "xmulti" ]; then
     SUBNET_RANGE="$(find_multi_subnet_range)"
   else

tools/gate/launch-osh/common.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+set -ex
+: ${WORK_DIR:="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/.."}
+source ${WORK_DIR}/tools/gate/vars.sh
+source ${WORK_DIR}/tools/gate/funcs/helm.sh
+source ${WORK_DIR}/tools/gate/funcs/kube.sh
+source ${WORK_DIR}/tools/gate/funcs/network.sh
+
+# NOTE(portdirect): Temp workaround until module loading is supported by
+# OpenStack-Helm in Fedora
+if [ "x$HOST_OS" == "xfedora" ]; then
+  sudo modprobe openvswitch
+  sudo modprobe gre
+  sudo modprobe vxlan
+  sudo modprobe ip6_tables
+fi
+
+if [ "x$PVC_BACKEND" == "xceph" ]; then
+  kubectl label nodes ceph-mon=enabled --all
+  kubectl label nodes ceph-osd=enabled --all
+  kubectl label nodes ceph-mds=enabled --all
+fi
+
+helm install --namespace=openstack ${WORK_DIR}/dns-helper --name=dns-helper
+kube_wait_for_pods openstack 180

tools/gate/openstack/network_launch.sh

@@ -0,0 +1,78 @@
+#!/bin/bash
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+set -xe
+: ${WORK_DIR:="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/.."}
+source ${WORK_DIR}/tools/gate/vars.sh
+source ${WORK_DIR}/tools/gate/funcs/network.sh
+source ${WORK_DIR}/tools/gate/funcs/openstack.sh
+
+# Turn on ip forwarding if its not already
+if [ $(cat /proc/sys/net/ipv4/ip_forward) -eq 0 ]; then
+  sudo bash -c "echo 1 > /proc/sys/net/ipv4/ip_forward"
+fi
+
+# Assign IP address to br-ex
+sudo ip addr add ${OSH_BR_EX_ADDR} dev br-ex
+sudo ip link set br-ex up
+# Setup masquerading on default route dev to public subnet
+sudo iptables -t nat -A POSTROUTING -o $(net_default_iface) -s ${OSH_EXT_SUBNET} -j MASQUERADE
+
+# Disable In-Band rules on br-ex bridge to ease debugging
+OVS_VSWITCHD_POD=$(kubectl get -n openstack pods -l application=neutron,component=ovs-vswitchd --no-headers -o name | head -1 | awk -F '/' '{ print $NF }')
+kubectl exec -n openstack ${OVS_VSWITCHD_POD} -- ovs-vsctl set Bridge br-ex other_config:disable-in-band=true
+
+
+if ! $OPENSTACK service list -f value -c Type | grep -q orchestration; then
+  echo "No orchestration service active: creating public network via CLI"
+  $NEUTRON net-create ${OSH_EXT_NET_NAME} -- --is-default \
+    --router:external \
+    --provider:network_type=flat \
+    --provider:physical_network=public
+  $NEUTRON subnet-create \
+    --name ${OSH_EXT_SUBNET_NAME} \
+    --ip-version 4 \
+    $($NEUTRON net-show ${OSH_EXT_NET_NAME} -f value -c id) ${OSH_EXT_SUBNET} -- \
+        --enable_dhcp=False
+
+  # Create default subnet pool
+  $NEUTRON subnetpool-create \
+    ${OSH_PRIVATE_SUBNET_POOL_NAME} \
+    --default-prefixlen ${OSH_PRIVATE_SUBNET_POOL_DEF_PREFIX} \
+    --pool-prefix ${OSH_PRIVATE_SUBNET_POOL} \
+    --shared \
+    --is-default=True
+else
+  echo "Orchestration service active: creating public network via Heat"
+  HEAT_TEMPLATE=$(cat ${WORK_DIR}/tools/gate/files/${OSH_PUB_NET_STACK}.yaml | base64 -w 0)
+  kubectl exec -n openstack ${OPENSTACK_POD} -- bash -c "echo $HEAT_TEMPLATE | base64 -d > /tmp/${OSH_PUB_NET_STACK}.yaml"
+  $OPENSTACK stack create \
+    --parameter network_name=${OSH_EXT_NET_NAME} \
+    --parameter physical_network_name=public \
+    --parameter subnet_name=${OSH_EXT_SUBNET_NAME} \
+    --parameter subnet_cidr=${OSH_EXT_SUBNET} \
+    --parameter subnet_gateway=${OSH_BR_EX_ADDR%/*} \
+    -t /tmp/${OSH_PUB_NET_STACK}.yaml \
+    ${OSH_PUB_NET_STACK}
+  openstack_wait_for_stack ${OSH_PUB_NET_STACK}
+
+  HEAT_TEMPLATE=$(cat ${WORK_DIR}/tools/gate/files/${OSH_SUBNET_POOL_STACK}.yaml | base64 -w 0)
+  kubectl exec -n openstack ${OPENSTACK_POD} -- bash -c "echo $HEAT_TEMPLATE | base64 -d > /tmp/${OSH_SUBNET_POOL_STACK}.yaml"
+  $OPENSTACK stack create \
+    --parameter subnet_pool_name=${OSH_PRIVATE_SUBNET_POOL_NAME} \
+    --parameter subnet_pool_prefixes=${OSH_PRIVATE_SUBNET_POOL} \
+    --parameter subnet_pool_default_prefix_length=${OSH_PRIVATE_SUBNET_POOL_DEF_PREFIX} \
+    -t /tmp/${OSH_SUBNET_POOL_STACK}.yaml \
+    ${OSH_SUBNET_POOL_STACK}
+  openstack_wait_for_stack ${OSH_SUBNET_POOL_STACK}
+fi

tools/gate/openstack/vm_cli_launch.sh

@@ -0,0 +1,91 @@
+#!/bin/bash
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+set -xe
+: ${WORK_DIR:="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/.."}
+source ${WORK_DIR}/tools/gate/vars.sh
+source ${WORK_DIR}/tools/gate/funcs/network.sh
+source ${WORK_DIR}/tools/gate/funcs/openstack.sh
+
+# Create default private network
+$NEUTRON net-create ${OSH_PRIVATE_NET_NAME}
+$NEUTRON subnet-create \
+  --name ${OSH_PRIVATE_SUBNET_NAME} \
+  --ip-version 4 \
+  --dns-nameserver ${OSH_EXT_DNS} \
+  $($NEUTRON net-show private -f value -c id) \
+  ${OSH_PRIVATE_SUBNET}
+
+# Create default router and link networks
+$NEUTRON router-create ${OSH_ROUTER}
+$NEUTRON router-interface-add \
+  $($NEUTRON router-show ${OSH_ROUTER} -f value -c id) \
+  $($NEUTRON subnet-show private-subnet -f value -c id)
+$NEUTRON router-gateway-set \
+  $($NEUTRON router-show ${OSH_ROUTER} -f value -c id) \
+  $($NEUTRON net-show ${OSH_EXT_NET_NAME} -f value -c id)
+
+ROUTER_PUBLIC_IP=$($NEUTRON router-show ${OSH_ROUTER} -f value -c external_gateway_info | jq -r '.external_fixed_ips[].ip_address')
+wait_for_ping ${ROUTER_PUBLIC_IP}
+
+# Loosen up security group to allow access to the VM
+PROJECT=$($OPENSTACK project show admin -f value -c id)
+SECURITY_GROUP=$($OPENSTACK security group list -f csv | grep ${PROJECT} | grep "default" | awk -F "," '{ print $1 }'  | tr -d '"')
+$OPENSTACK security group rule create ${SECURITY_GROUP} \
+  --protocol icmp \
+  --src-ip 0.0.0.0/0
+$OPENSTACK security group rule create ${SECURITY_GROUP} \
+  --protocol tcp \
+  --dst-port 22:22 \
+  --src-ip 0.0.0.0/0
+
+# Setup SSH Keypair in Nova
+KEYPAIR_LOC="$(mktemp).pem"
+$OPENSTACK keypair create ${OSH_VM_KEY_CLI} > ${KEYPAIR_LOC}
+chmod 600 ${KEYPAIR_LOC}
+
+# Boot a vm and wait for it to become active
+FLAVOR=$($OPENSTACK flavor show "${OSH_VM_FLAVOR}" -f value -c id)
+IMAGE=$($OPENSTACK image list -f csv | awk -F ',' '{ print $2 "," $1 }' | grep "^\"Cirros" | head -1 | awk -F ',' '{ print $2 }' | tr -d '"')
+NETWORK=$($NEUTRON net-show ${OSH_PRIVATE_NET_NAME} -f value -c id)
+$NOVA boot \
+    --nic net-id=${NETWORK} \
+    --flavor=${FLAVOR} \
+    --image=${IMAGE} \
+    --key-name=${OSH_VM_KEY_CLI} \
+    --security-groups="default" \
+    ${OSH_VM_NAME_CLI}
+openstack_wait_for_vm ${OSH_VM_NAME_CLI}
+
+# Assign a floating IP to the VM
+FLOATING_IP=$($OPENSTACK floating ip create ${OSH_EXT_NET_NAME} -f value -c floating_ip_address)
+$OPENSTACK server add floating ip ${OSH_VM_NAME_CLI} ${FLOATING_IP}
+
+# Ping our VM
+wait_for_ping ${FLOATING_IP} ${SERVICE_TEST_TIMEOUT}
+
+# Wait for SSH to come up
+wait_for_ssh_port ${FLOATING_IP} ${SERVICE_TEST_TIMEOUT}
+
+# SSH into the VM and check it can reach the outside world
+ssh-keyscan "$FLOATING_IP" >> ~/.ssh/known_hosts
+ssh -i ${KEYPAIR_LOC} cirros@${FLOATING_IP} ping -q -c 1 -W 2 ${OSH_BR_EX_ADDR%/*}
+
+# SSH into the VM and check it can reach the metadata server
+ssh -i ${KEYPAIR_LOC} cirros@${FLOATING_IP} curl -sSL 169.254.169.254
+
+# Bonus round - display a Unicorn
+ssh -i ${KEYPAIR_LOC} cirros@${FLOATING_IP} curl http://artscene.textfiles.com/asciiart/unicorn || true
+
+# Remove the test vm
+$NOVA delete ${OSH_VM_NAME_CLI}

tools/gate/openstack/vm_heat_launch.sh

@@ -0,0 +1,67 @@
+#!/bin/bash
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+set -xe
+: ${WORK_DIR:="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/.."}
+source ${WORK_DIR}/tools/gate/vars.sh
+source ${WORK_DIR}/tools/gate/funcs/network.sh
+source ${WORK_DIR}/tools/gate/funcs/openstack.sh
+
+# Setup SSH Keypair in Nova
+KEYPAIR_LOC="$(mktemp).pem"
+$OPENSTACK keypair create ${OSH_VM_KEY_STACK} > ${KEYPAIR_LOC}
+chmod 600 ${KEYPAIR_LOC}
+
+# NOTE(portdirect): We do this fancy, and seemingly pointless, footwork to get
+# the full image name for the cirros Image without having to be explicit.
+IMAGE_NAME=$($OPENSTACK image show -f value -c name \
+  $($OPENSTACK image list -f csv | awk -F ',' '{ print $2 "," $1 }' | \
+    grep "^\"Cirros" | head -1 | awk -F ',' '{ print $2 }' | tr -d '"'))
+
+HEAT_TEMPLATE=$(cat ${WORK_DIR}/tools/gate/files/${OSH_BASIC_VM_STACK}.yaml | base64 -w 0)
+kubectl exec -n openstack ${OPENSTACK_POD} -- bash -c "echo $HEAT_TEMPLATE | base64 -d > /tmp/${OSH_BASIC_VM_STACK}.yaml"
+$OPENSTACK stack create \
+  --parameter public_net=${OSH_EXT_NET_NAME} \
+  --parameter image="${IMAGE_NAME}" \
+  --parameter flavor=${OSH_VM_FLAVOR} \
+  --parameter ssh_key=${OSH_VM_KEY_STACK} \
+  --parameter cidr=${OSH_PRIVATE_SUBNET} \
+  -t /tmp/${OSH_BASIC_VM_STACK}.yaml \
+  ${OSH_BASIC_VM_STACK}
+openstack_wait_for_stack ${OSH_BASIC_VM_STACK} ${SERVICE_TEST_TIMEOUT}
+
+FLOATING_IP=$($OPENSTACK floating ip show \
+  $($OPENSTACK stack resource show \
+      ${OSH_BASIC_VM_STACK} \
+      server_floating_ip \
+      -f value -c physical_resource_id) \
+      -f value -c floating_ip_address)
+
+# Ping our VM
+wait_for_ping ${FLOATING_IP} ${SERVICE_TEST_TIMEOUT}
+
+# Wait for SSH to come up
+wait_for_ssh_port ${FLOATING_IP} ${SERVICE_TEST_TIMEOUT}
+
+# SSH into the VM and check it can reach the outside world
+ssh-keyscan "$FLOATING_IP" >> ~/.ssh/known_hosts
+ssh -i ${KEYPAIR_LOC} cirros@${FLOATING_IP} ping -q -c 1 -W 2 ${OSH_BR_EX_ADDR%/*}
+
+# SSH into the VM and check it can reach the metadata server
+ssh -i ${KEYPAIR_LOC} cirros@${FLOATING_IP} curl -sSL 169.254.169.254
+
+# Bonus round - display a Unicorn
+ssh -i ${KEYPAIR_LOC} cirros@${FLOATING_IP} curl http://artscene.textfiles.com/asciiart/unicorn || true
+
+# Remove the test stack
+$OPENSTACK stack delete ${OSH_BASIC_VM_STACK}

tools/gate/openstack_aio_launch.sh

@@ -1,124 +0,0 @@
-#!/bin/bash
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-set -xe
-
-: ${OSH_BR_EX_ADDR:="172.24.4.1/24"}
-: ${OSH_EXT_SUBNET:="172.24.4.0/24"}
-: ${OSH_EXT_DNS:="8.8.8.8"}
-: ${OSH_EXT_NET_NAME:="public"}
-: ${OSH_EXT_SUBNET_NAME:="public-subnet"}
-: ${OSH_ROUTER:="router1"}
-: ${OSH_PRIVATE_NET_NAME:="private"}
-: ${OSH_PRIVATE_SUBNET:="10.0.0.0/24"}
-: ${OSH_PRIVATE_SUBNET_NAME:="private-subnet"}
-: ${OSH_PRIVATE_SUBNET_POOL:="10.0.0.0/8"}
-: ${OSH_PRIVATE_SUBNET_POOL_NAME:="shared-default-subnetpool"}
-: ${OSH_PRIVATE_SUBNET_POOL_DEF_PREFIX:="24"}
-: ${OSH_VM_NAME:="osh-smoketest"}
-: ${OSH_VM_KEY:="osh-smoketest-key"}
-
-# Source some functions that will help us
-source ${WORK_DIR}/tools/gate/funcs/network.sh
-source ${WORK_DIR}/tools/gate/funcs/openstack.sh
-
-# Turn on ip forwarding if its not already
-if [ $(cat /proc/sys/net/ipv4/ip_forward) -eq 0 ]; then
-  sudo bash -c "echo 1 > /proc/sys/net/ipv4/ip_forward"
-fi
-
-# Assign IP address to br-ex
-sudo ip addr add ${OSH_BR_EX_ADDR} dev br-ex
-sudo ip link set br-ex up
-# Setup masquerading on default route dev to public subnet
-sudo iptables -t nat -A POSTROUTING -o $(net_default_iface) -s ${OSH_EXT_SUBNET} -j MASQUERADE
-
-# Disable In-Band rules on br-ex bridge to ease debugging
-OVS_VSWITCHD_POD=$(kubectl get -n openstack pods -l application=neutron,component=ovs-vswitchd --no-headers -o name | head -1 | awk -F '/' '{ print $NF }')
-kubectl exec -n openstack ${OVS_VSWITCHD_POD} -- ovs-vsctl set Bridge br-ex other_config:disable-in-band=true
-
-# Create default networks
-$NEUTRON net-create ${OSH_PRIVATE_NET_NAME}
-$NEUTRON subnet-create \
-    --name ${OSH_PRIVATE_SUBNET_NAME} \
-    --ip-version 4 \
-    --dns-nameserver ${OSH_EXT_DNS} \
-    $($NEUTRON net-show private -f value -c id) \
-    ${OSH_PRIVATE_SUBNET}
-$NEUTRON router-create ${OSH_ROUTER}
-$NEUTRON subnetpool-create \
-    ${OSH_PRIVATE_SUBNET_POOL_NAME} \
-    --default-prefixlen ${OSH_PRIVATE_SUBNET_POOL_DEF_PREFIX} \
-    --pool-prefix ${OSH_PRIVATE_SUBNET_POOL} \
-    --shared \
-    --is-default=True
-$NEUTRON net-create ${OSH_EXT_NET_NAME} -- --is-default \
-    --router:external \
-    --provider:network_type=flat \
-    --provider:physical_network=public
-$NEUTRON router-interface-add $($NEUTRON router-show ${OSH_ROUTER} -f value -c id) $($NEUTRON subnet-show private-subnet -f value -c id)
-$NEUTRON subnet-create \
-  --name ${OSH_EXT_SUBNET_NAME} \
-  --ip-version 4 \
-  $($NEUTRON net-show ${OSH_EXT_NET_NAME} -f value -c id) ${OSH_EXT_SUBNET} -- --enable_dhcp=False
-$NEUTRON router-gateway-set $($NEUTRON router-show ${OSH_ROUTER} -f value -c id) $($NEUTRON net-show ${OSH_EXT_NET_NAME} -f value -c id)
-
-ROUTER_PUBLIC_IP=$($NEUTRON router-show ${OSH_ROUTER} -f value -c external_gateway_info | jq -r '.external_fixed_ips[].ip_address')
-wait_for_ping ${ROUTER_PUBLIC_IP}
-
-# Loosen up security group to allow access to the VM
-PROJECT=$($OPENSTACK project show admin -f value -c id)
-SECURITY_GROUP=$($OPENSTACK security group list -f csv | grep ${PROJECT} | grep "default" | awk -F "," '{ print $1 }'  | tr -d '"')
-$OPENSTACK security group rule create ${SECURITY_GROUP} --protocol icmp --src-ip  0.0.0.0/0
-$OPENSTACK security group rule create ${SECURITY_GROUP} --protocol tcp --dst-port 22:22 --src-ip 0.0.0.0/0
-
-# Setup SSH Keypair in Nova
-KEYPAIR_LOC="$(mktemp).pem"
-$OPENSTACK keypair create ${OSH_VM_KEY} > ${KEYPAIR_LOC}
-chmod 600 ${KEYPAIR_LOC}
-
-# Boot a vm and wait for it to become active
-FLAVOR=$($OPENSTACK flavor show "m1.tiny" -f value -c id)
-IMAGE=$($OPENSTACK image list -f csv | awk -F ',' '{ print $2 "," $1 }' | grep "^\"Cirros" | head -1 | awk -F ',' '{ print $2 }' | tr -d '"')
-NETWORK=$($NEUTRON net-show private -f value -c id)
-$NOVA boot \
-    --nic net-id=${NETWORK} \
-    --flavor=${FLAVOR} \
-    --image=${IMAGE} \
-    --key-name=${OSH_VM_KEY} \
-    --security-groups="default" \
-    ${OSH_VM_NAME}
-openstack_wait_for_vm ${OSH_VM_NAME}
-
-# Assign a floating IP to the VM
-FLOATING_IP=$($OPENSTACK floating ip create ${OSH_EXT_NET_NAME} -f value -c floating_ip_address)
-$OPENSTACK server add floating ip ${OSH_VM_NAME} ${FLOATING_IP}
-
-# Ping our VM
-wait_for_ping ${FLOATING_IP}
-
-# Wait for SSH to come up
-wait_for_ssh_port ${FLOATING_IP}
-
-# SSH into the VM and check it can reach the outside world
-ssh-keyscan "$FLOATING_IP" >> ~/.ssh/known_hosts
-ssh -i ${KEYPAIR_LOC} cirros@${FLOATING_IP} ping -q -c 1 -W 2 ${OSH_BR_EX_ADDR%/*}
-
-# SSH into the VM and check it can reach the metadata server
-ssh -i ${KEYPAIR_LOC} cirros@${FLOATING_IP} curl -sSL 169.254.169.254
-
-# Bonus round - display a Unicorn
-ssh -i ${KEYPAIR_LOC} cirros@${FLOATING_IP} curl http://artscene.textfiles.com/asciiart/unicorn || true
-
-# Remove the test vm
-$NOVA delete ${OSH_VM_NAME}

tools/gate/provision_gate_worker_node.sh

@@ -12,10 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 set -ex
-
+: ${WORK_DIR:="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"}
+source ${WORK_DIR}/tools/gate/vars.sh
 cd ${WORK_DIR}
-source /etc/os-release
-export HOST_OS=${ID}
 source ${WORK_DIR}/tools/gate/funcs/common.sh
 source ${WORK_DIR}/tools/gate/funcs/network.sh
 source ${WORK_DIR}/tools/gate/funcs/kube.sh

tools/gate/setup_gate.sh

@@ -12,35 +12,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 set -ex
-
-export HELM_VERSION=${HELM_VERSION:-"v2.5.1"}
-export KUBE_VERSION=${KUBE_VERSION:-"v1.6.8"}
-export PVC_BACKEND=${PVC_BACKEND:-"ceph"}
-export UPSTREAM_DNS=${UPSTREAM_DNS:-"8.8.8.8"}
-
-export SERVICE_LAUNCH_TIMEOUT=${SERVICE_LAUNCH_TIMEOUT:="600"}
-export SERVICE_TEST_TIMEOUT=${SERVICE_TEST_TIMEOUT:="600"}
-
-export KUBECONFIG=${HOME}/.kubeadm-aio/admin.conf
-
-export LOOPBACK_CREATE=${LOOPBACK_CREATE:="false"}
-export LOOPBACK_DEVS=${LOOPBACK_DEVS:="3"}
-export LOOPBACK_SIZE=${LOOPBACK_SIZE:="500M"}
-export LOOPBACK_DIR=${LOOPBACK_DIR:="/var/lib/iscsi-loopback"}
-
-export KUBEADM_IMAGE=openstackhelm/kubeadm-aio:${KUBE_VERSION}-dev
-export CNI_POD_CIDR=${CNI_POD_CIDR:="192.168.0.0/16"}
-export KUBE_CNI=${KUBE_CNI:="calico"}
-
 export WORK_DIR=$(pwd)
-source /etc/os-release
+source ${WORK_DIR}/tools/gate/vars.sh
-export HOST_OS=${ID}
 source ${WORK_DIR}/tools/gate/funcs/common.sh
 source ${WORK_DIR}/tools/gate/funcs/network.sh
 source ${WORK_DIR}/tools/gate/funcs/helm.sh
 
 # Setup the logging location: by default use the working dir as the root.
-export LOGS_DIR=${LOGS_DIR:-"${WORK_DIR}/logs"}
 rm -rf ${LOGS_DIR} || true
 mkdir -p ${LOGS_DIR}
 
@@ -73,29 +51,34 @@ if [ "x$INTEGRATION_TYPE" == "xlinter" ]; then
   helm_build > ${LOGS_DIR}/helm_build
   helm_plugin_template_install
   helm_template_run
-fi
+else
-
+  # Setup the K8s Cluster
-# Setup the K8s Cluster
-if [ "x$INTEGRATION" == "xaio" ]; then
- bash ${WORK_DIR}/tools/gate/kubeadm_aio.sh
-elif [ "x$INTEGRATION" == "xmulti" ]; then
- bash ${WORK_DIR}/tools/gate/kubeadm_aio.sh
- bash ${WORK_DIR}/tools/gate/setup_gate_worker_nodes.sh
-fi
-
-# Deploy OpenStack-Helm
-if [ "x$INTEGRATION_TYPE" == "xbasic" ]; then
-  bash ${WORK_DIR}/tools/gate/helm_dry_run.sh
-  bash ${WORK_DIR}/tools/gate/basic_launch.sh
-elif [ "x$INTEGRATION_TYPE" == "xarmada" ]; then
-  bash ${WORK_DIR}/tools/gate/armada_launch.sh
-fi
-
-if ! [ "x$INTEGRATION_TYPE" == "x" ]; then
-  # Run Basic Full Stack Tests
   if [ "x$INTEGRATION" == "xaio" ]; then
-   bash ${WORK_DIR}/tools/gate/openstack_aio_launch.sh
+   bash ${WORK_DIR}/tools/gate/kubeadm_aio.sh
+  elif [ "x$INTEGRATION" == "xmulti" ]; then
+   bash ${WORK_DIR}/tools/gate/kubeadm_aio.sh
+   bash ${WORK_DIR}/tools/gate/setup_gate_worker_nodes.sh
+  fi
+
+  # Deploy OpenStack-Helm
+  if ! [ "x$INTEGRATION_TYPE" == "x" ]; then
+    bash ${WORK_DIR}/tools/gate/helm_dry_run.sh
+    bash ${WORK_DIR}/tools/gate/launch-osh/common.sh
+    if [ "x$INTEGRATION_TYPE" == "xbasic" ]; then
+      bash ${WORK_DIR}/tools/gate/launch-osh/basic.sh
+    elif [ "x$INTEGRATION_TYPE" == "xarmada" ]; then
+      bash ${WORK_DIR}/tools/gate/launch-osh/armada.sh
+    fi
+  fi
+
+  if ! [ "x$INTEGRATION_TYPE" == "x" ]; then
+    # Run Basic Full Stack Tests
+    if [ "x$INTEGRATION" == "xaio" ]; then
+     bash ${WORK_DIR}/tools/gate/openstack/network_launch.sh
+     bash ${WORK_DIR}/tools/gate/openstack/vm_cli_launch.sh
+     bash ${WORK_DIR}/tools/gate/openstack/vm_heat_launch.sh
+    fi
+    # Collect all logs from the environment
+    bash ${WORK_DIR}/tools/gate/dump_logs.sh 0
   fi
-  # Collect all logs from the environment
-  bash ${WORK_DIR}/tools/gate/dump_logs.sh 0
 fi

tools/gate/setup_gate_worker_nodes.sh

@@ -12,10 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 set -ex
-
+: ${WORK_DIR:="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"}
-: ${SSH_PRIVATE_KEY:="/etc/nodepool/id_rsa"}
+source ${WORK_DIR}/tools/gate/vars.sh
-: ${PRIMARY_NODE_IP:="$(cat /etc/nodepool/primary_node | tail -1)"}
-: ${SUB_NODE_IPS:="$(cat /etc/nodepool/sub_nodes)"}
 export SUB_NODE_COUNT="$(($(echo ${SUB_NODE_IPS} | wc -w) + 1))"
 
 sudo chown $(whoami) ${SSH_PRIVATE_KEY}

tools/gate/vars.sh

@@ -0,0 +1,77 @@
+#!/bin/bash
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Set work dir if not already done
+: ${WORK_DIR:="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"}
+
+# Set logs directory
+export LOGS_DIR=${LOGS_DIR:-"${WORK_DIR}/logs"}
+
+# Get Host OS
+source /etc/os-release
+export HOST_OS=${HOST_OS:="${ID}"}
+
+# Set versions of K8s and Helm to use
+export HELM_VERSION=${HELM_VERSION:-"v2.5.1"}
+export KUBE_VERSION=${KUBE_VERSION:-"v1.6.8"}
+
+# Set K8s-AIO options
+export KUBECONFIG=${KUBECONFIG:="${HOME}/.kubeadm-aio/admin.conf"}
+export KUBEADM_IMAGE=${KUBEADM_IMAGE:="openstackhelm/kubeadm-aio:${KUBE_VERSION}"}
+
+# Set K8s network options
+export CNI_POD_CIDR=${CNI_POD_CIDR:="192.168.0.0/16"}
+export KUBE_CNI=${KUBE_CNI:="calico"}
+
+# Set PVC Backend
+export PVC_BACKEND=${PVC_BACKEND:-"ceph"}
+
+# Set Upstream DNS
+export UPSTREAM_DNS=${UPSTREAM_DNS:-"8.8.8.8"}
+
+# Set gate script timeouts
+export SERVICE_LAUNCH_TIMEOUT=${SERVICE_LAUNCH_TIMEOUT:="600"}
+export SERVICE_TEST_TIMEOUT=${SERVICE_TEST_TIMEOUT:="600"}
+
+# Setup Loopback device options
+export LOOPBACK_CREATE=${LOOPBACK_CREATE:="false"}
+export LOOPBACK_DEVS=${LOOPBACK_DEVS:="3"}
+export LOOPBACK_SIZE=${LOOPBACK_SIZE:="500M"}
+export LOOPBACK_DIR=${LOOPBACK_DIR:="/var/lib/iscsi-loopback"}
+
+# Setup Multinode params
+export SSH_PRIVATE_KEY=${SSH_PRIVATE_KEY:="/etc/nodepool/id_rsa"}
+export PRIMARY_NODE_IP=${PRIMARY_NODE_IP:="$(cat /etc/nodepool/primary_node | tail -1)"}
+export SUB_NODE_IPS=${SUB_NODE_IPS:="$(cat /etc/nodepool/sub_nodes)"}
+
+# Define OpenStack Test Params
+export OSH_BR_EX_ADDR=${OSH_BR_EX_ADDR:="172.24.4.1/24"}
+export OSH_EXT_SUBNET=${OSH_EXT_SUBNET:="172.24.4.0/24"}
+export OSH_EXT_DNS=${OSH_EXT_DNS:="8.8.8.8"}
+export OSH_EXT_NET_NAME=${OSH_EXT_NET_NAME:="public"}
+export OSH_EXT_SUBNET_NAME=${OSH_EXT_SUBNET_NAME:="public-subnet"}
+export OSH_ROUTER=${OSH_ROUTER:="router1"}
+export OSH_PRIVATE_NET_NAME=${OSH_PRIVATE_NET_NAME:="private"}
+export OSH_PRIVATE_SUBNET=${OSH_PRIVATE_SUBNET:="10.0.0.0/24"}
+export OSH_PRIVATE_SUBNET_NAME=${OSH_PRIVATE_SUBNET_NAME:="private-subnet"}
+export OSH_PRIVATE_SUBNET_POOL=${OSH_PRIVATE_SUBNET_POOL:="10.0.0.0/8"}
+export OSH_PRIVATE_SUBNET_POOL_NAME=${OSH_PRIVATE_SUBNET_POOL_NAME:="shared-default-subnetpool"}
+export OSH_PRIVATE_SUBNET_POOL_DEF_PREFIX=${OSH_PRIVATE_SUBNET_POOL_DEF_PREFIX:="24"}
+export OSH_VM_FLAVOR=${OSH_VM_FLAVOR:="m1.tiny"}
+export OSH_VM_NAME_CLI=${OSH_VM_NAME_CLI:="osh-smoketest"}
+export OSH_VM_KEY_CLI=${OSH_VM_KEY_CLI:="osh-smoketest-key"}
+export OSH_PUB_NET_STACK=${OSH_PUB_NET_STACK:="heat-public-net-deployment"}
+export OSH_SUBNET_POOL_STACK=${OSH_SUBNET_POOL_STACK:="heat-subnet-pool-deployment"}
+export OSH_BASIC_VM_STACK=${OSH_BASIC_VM_STACK:="heat-basic-vm-deployment"}
+export OSH_VM_KEY_STACK=${OSH_VM_KEY_STACK:="heat-vm-key"}