diff --git a/provisioning_scripts/fibre_channel/install_fc_drivers.sh b/provisioning_scripts/fibre_channel/install_fc_drivers.sh new file mode 100644 index 0000000..72fe6df --- /dev/null +++ b/provisioning_scripts/fibre_channel/install_fc_drivers.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +# Copyright (C) 2015 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2015 Pure Storage, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# +# See the License for the specific language governing permissions and +# limitations under the License. + +sudo apt-get -y -qq install linux-image-extra-$(uname -r) > /dev/null +echo $? +sudo apt-get -y -qq install sysfsutils > /dev/null +echo $? diff --git a/provisioning_scripts/fibre_channel/invoke_fc_passthrough.sh b/provisioning_scripts/fibre_channel/invoke_fc_passthrough.sh new file mode 100644 index 0000000..ec9c6fe --- /dev/null +++ b/provisioning_scripts/fibre_channel/invoke_fc_passthrough.sh @@ -0,0 +1,176 @@ +#!/usr/bin/env bash + +# Copyright (C) 2015 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2015 Pure Storage, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# +# See the License for the specific language governing permissions and +# limitations under the License. + + +# Shell commands to get virsh the information it +# needs to successfully pass through a Fibre Channel PCI Card to the virtual +# machine this script is running on. The instance only knows its IP address, +# while its Virsh name is required for pass through. This script uses Nova on +# the provider blade as an intermediary to find the name. Meanwhile, this +# script finds the Fibre Channel PCI card on the provider and generates the +# information Virsh needs to attach it. +# +# Expect four env variables, the provider hostname (optionally user if needed) +# the private key file we should use to connect to the provider, and the file +# that should be sourced for OpenStack credentials. +# +# export FC_PROVIDER=my.provider.hostname +# export FC_PROVIDER_USER=root +# export FC_PROVIDER_KEY=/opt/nodepool-scripts/passthrough +# export FC_PROVIDER_RC=/root/keystonerc_jenkins +# +# /opt/nodepool-scripts/invoke-fc-passthrough.sh + +eth0_ip=$(hostname -I | cut -f1 -d' ') + +PROVIDER=${FC_PROVIDER} +if [[ -z $PROVIDER ]]; then + eth0_ip_base=$(echo $eth0_ip | cut -f1,2,3 -d.) + PROVIDER="${eth0_ip_base}.1" +fi + +PROVIDER_KEY=${FC_PROVIDER_KEY:-"/opt-nodepool-scripts/passthrough"} +PROVIDER_RC=${FC_PROVIDER_RC:-"keystonerc_jenkins"} + +CURRENT_USER=$(whoami) +PROVIDER_USER=${FC_PROVIDER_USER:-$CURRENT_USER} + +# Passthrough is a private key that needs to be setup for the provider +# and any compute nodes that might end up hosting the VM we want passthrough on. +# We will assume ownership of the key (probably as the jenkins user..), also +# assuming the group is the same name as the user... +sudo chown $CURRENT_USER:$CURRENT_USER $PROVIDER_KEY +chmod 0400 $PROVIDER_KEY + +# Get our NOVA_ID +NOVA_LIST=$(ssh -i $PROVIDER_KEY $PROVIDER_USER@$PROVIDER "source $PROVIDER_RC && nova list") +nova_result=$? +NOVA_ID=$(echo "$NOVA_LIST" | grep ACTIVE | grep -v deleting | grep $eth0_ip | cut -d \| -f 2 | tr -d '[:space:]') +echo "NOVA_ID result: $nova_result" +if [[ $nova_result -ne 0 || -z "$NOVA_ID" ]]; then + echo "Unable to get Nova ID. Aborting. Debug info:" + echo $NOVA_LIST + echo "NOVA_ID: $NOVA_ID" + exit 2 +fi + +# Get instance details +NOVA_DETAILS=$(ssh -i $PROVIDER_KEY $PROVIDER_USER@$PROVIDER "source $PROVIDER_RC && nova show $NOVA_ID") +nova_results=$? + +# Get our Virsh name +VIRSH_NAME=$(echo "$NOVA_DETAILS" | grep instance_name | cut -d \| -f 3 | tr -d '[:space:]') +virsh_result=$? +echo "VIRSH_NAME result: $virsh_result" +if [[ $nova_result -ne 0 || $virsh_result -ne 0 || -z "$VIRSH_NAME" ]]; then + echo "Unable to get Virsh Name. Aborting. Debug info:" + echo "NOVA_LIST:" + echo $NOVA_LIST + echo "NOVA_DETAILS:" + echo $NOVA_DETAILS + echo "VIRSH_NAME: $VIRSH_NAME" + exit 2 +fi + +# Get the hypervisor_hostname +HYPERVISOR=$(echo "$NOVA_DETAILS" | grep hypervisor_hostname | cut -d \| -f 3 | tr -d '[:space:]') +hypervisor_result=$? +echo "HYPERVISOR result: $hypervisor_result" +if [[ $hypervisor_result -ne 0 || -z "$HYPERVISOR" ]]; then + echo "Unable to get Hypervisor Host Name. Aborting. Debug info:" + echo "NOVA_LIST:" + echo $NOVA_LIST + echo "NOVA_DETAILS:" + echo $NOVA_DETAILS + echo "HYPERVISOR: $HYPERVISOR" + exit 2 +fi +echo "Found Hypervisor hostname: $HYPERVISOR" + +fc_pci_device=$(ssh -i $PROVIDER_KEY $PROVIDER_USER@$HYPERVISOR 'echo $fc_pci_device') + +if [[ -z $fc_pci_device ]]; then + echo "No FC device known. Set fc_pci_device in your /etc/profile.d or /etc/environment (depending on distro and ssh configuration) to the desired 'Class Device path', e.g. '0000:21:00.2'" + exit 2 +fi + +echo "Found pci devices: $fc_pci_device" + +exit_code=1 +errexit=$(set +o | grep errexit) +#Ignore errors +set +e +for pci in $fc_pci_device; do + echo $pci + BUS=$(echo $pci | cut -d : -f2) + SLOT=$(echo $pci | cut -d : -f3 | cut -d . -f1) + FUNCTION=$(echo $pci | cut -d : -f3 | cut -d . -f2) + XML="
" + echo $XML + fcoe=`mktemp --suffix=_fcoe.xml` + echo $XML > $fcoe + + scp -i $PROVIDER_KEY $fcoe $PROVIDER_USER@$HYPERVISOR:/tmp/ + + # Run passthrough and clean up. + # TODO: At the point where we can do more than one node on a provider we + # will need to do this cleanup at the end of the job and not *before* attaching + # since we won't know which ones are still in use + echo $(sudo lspci | grep -i fib) + ssh -i $PROVIDER_KEY $PROVIDER_USER@$HYPERVISOR "virsh nodedev-dettach pci_0000_${BUS}_${SLOT}_${FUNCTION}" + + detach_result=$? + echo "Detach result: $detach_result" + if [[ $detach_result -ne 0 ]]; then + echo "Detach failed. Trying next device..." + continue + fi + + echo $(sudo lspci | grep -i fib) + ssh -i $PROVIDER_KEY $PROVIDER_USER@$HYPERVISOR "virsh attach-device $VIRSH_NAME /tmp/fcoe.xml" + attach_result=$? + echo "Attach result: $attach_result" + if [[ $attach_result -eq 0 ]]; then + echo "Attached succeed. Trying next device..." + exit_code=0 + fi + echo $(sudo lspci | grep -i fib) + +done +$errexit + +if [[ $exit_code -ne 0 ]]; then + echo "FC Passthrough failed. Aborting." + exit $exit_code +fi + +# Make sure that really it worked... +sudo modprobe lpfc +echo $? + +sudo systool -c fc_host -v +echo $? + +echo $(sudo lspci | grep -i fib) + +device_path=$(sudo systool -c fc_host -v | grep "Device path") +if [[ ${device_path} -eq 0 ]]; then + echo "Failed to install FC Drivers. Aborting." + exit 1 +fi \ No newline at end of file diff --git a/provisioning_scripts/fibre_channel/setup_fc_for_provider.sh b/provisioning_scripts/fibre_channel/setup_fc_for_provider.sh new file mode 100644 index 0000000..65dc51c --- /dev/null +++ b/provisioning_scripts/fibre_channel/setup_fc_for_provider.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +# Copyright (C) 2015 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2015 Pure Storage, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# +# See the License for the specific language governing permissions and +# limitations under the License. + +# Run this as root on the OpenStack provider to setup the fc_pci_device +# environment variable needed for FC CI testing. +# +# If needed it will add an entry to /etc/profile.d/ with the variable. + +if [[ -z $fc_pci_device ]]; then + # Get all 'online' fc_host + # Don't override any pre-set values because the device may not be "Online" + # on subsequent runs + HOST=$(sudo systool -c fc_host -A port_state | grep -B1 -m 1 "Online") + if [[ -z $HOST ]]; then + echo "Error, unable to find a FC Host that is 'Online'. You can add the 'fc_pci_device' variable manually to vars.sh" + else + fc_pci_device=$(systool -c fc_host -v | grep -B12 "Online" | grep "Class Device path" | cut -d / -f 6 | tr '\n' ' ') + echo "Auto-detected FC PCI DEVICE: $fc_pci_device" + fi + echo export $fc_pci_device >> /etc/profile.d/fc_devices.sh +fi