openstack/ironic
Code Issues Proposed changes

[Devstack] Rework VMs connection logic

Browse Source 
Devstack currently plugs the simulated baremetal VMs into OVS using the
libvirt bridge driver, this caused a problem because libvirt unplugs the
VM from the network when it is turned off. To fix this an extra bridge
was added between the VM and OVS to allow the OVS port to persist even
when the VM was turned off. This patch replaces how the devstack
simulated baremetal VMs are plugged into OVS with a manually created tap
interface, this removes the need for an extra bridge, because manually
created tap interfaces aren't unplugged when the VM is turned off.

Allow to connect several interfaces to a node by setting
IRONIC_VM_INTERFACE_COUNT devstack variable.

Co-Authored-By: Vasyl Saienko <vsaienko@mirantis.com>

Change-Id: Iafd470445d59f0e2009e65ddaf65a6c603a1e1c1
This commit is contained in:
Sam Betts 2017-03-02 13:32:26 +00:00 committed by Vasyl Saienko
parent 8aec5fb6a5
commit 4ec88c9b8a
4 changed files with 75 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

66
devstack/lib/ironic
View File

@ -131,6 +131,8 @@ IRONIC_VM_EMULATOR=${IRONIC_VM_EMULATOR:-'/usr/bin/qemu-system-x86_64'}
IRONIC_VM_ENGINE=${IRONIC_VM_ENGINE:-qemu}
IRONIC_VM_NETWORK_BRIDGE=${IRONIC_VM_NETWORK_BRIDGE:-brbm}
IRONIC_VM_NETWORK_RANGE=${IRONIC_VM_NETWORK_RANGE:-192.0.2.0/24}
# Number of NICs to create Ironic node with. Take affect only for neutron network interface.
IRONIC_VM_INTERFACE_COUNT=${IRONIC_VM_INTERFACE_COUNT:-1}
IRONIC_VM_MACS_CSV_FILE=${IRONIC_VM_MACS_CSV_FILE:-$IRONIC_DATA_DIR/ironic_macs.csv}
IRONIC_AUTHORIZED_KEYS_FILE=${IRONIC_AUTHORIZED_KEYS_FILE:-$HOME/.ssh/authorized_keys}
IRONIC_CLEAN_NET_NAME=${IRONIC_CLEAN_NET_NAME:-${IRONIC_PROVISION_NETWORK_NAME:-${PRIVATE_NETWORK_NAME}}}
@ -959,6 +961,21 @@ function configure_ironic {
sudo groupadd $LIBVIRT_GROUP
fi
add_user_to_group $STACK_USER $LIBVIRT_GROUP
# Add /dev/net/tun to cgroup_device_acls, needed for type=ethernet interfaces
if ! sudo grep -q '^cgroup_device_acl' /etc/libvirt/qemu.conf; then
cat <<EOF | sudo tee -a /etc/libvirt/qemu.conf
cgroup_device_acl = [
"/dev/null", "/dev/full", "/dev/zero",
"/dev/random", "/dev/urandom",
"/dev/ptmx", "/dev/kvm", "/dev/kqemu",
"/dev/rtc", "/dev/hpet","/dev/net/tun",
"/dev/vfio/vfio",
]
EOF
restart_libvirt
fi
fi
}
@ -1362,11 +1379,16 @@ function create_bridge_and_vms {
vm_opts+=" -L $UEFI_LOADER_PATH -N $UEFI_NVRAM_PATH"
fi
local bridge_mac
bridge_mac=$(ip link show dev $IRONIC_VM_NETWORK_BRIDGE | egrep -o "ether [A-Za-z0-9:]+"|sed "s/ether\ //")
for vm_name in $(_ironic_bm_vm_names); do
sudo -E su $STACK_USER -c "$IRONIC_SCRIPTS_DIR/create-node.sh -n $vm_name \
-c $IRONIC_VM_SPECS_CPU -m $IRONIC_VM_SPECS_RAM -d $IRONIC_VM_SPECS_DISK \
-a $IRONIC_VM_SPECS_CPU_ARCH -b $IRONIC_VM_NETWORK_BRIDGE $vm_opts \
-p $vbmc_port -o $pdu_outlet -f $IRONIC_VM_SPECS_DISK_FORMAT $log_arg" >> $IRONIC_VM_MACS_CSV_FILE
-a $IRONIC_VM_SPECS_CPU_ARCH -b $IRONIC_VM_NETWORK_BRIDGE $vm_opts -p $vbmc_port -o $pdu_outlet \
-i $IRONIC_VM_INTERFACE_COUNT -f $IRONIC_VM_SPECS_DISK_FORMAT -M $PUBLIC_BRIDGE_MTU $log_arg" >> $IRONIC_VM_MACS_CSV_FILE
echo " ${bridge_mac} $IRONIC_VM_NETWORK_BRIDGE" >> $IRONIC_VM_MACS_CSV_FILE
vbmc_port=$((vbmc_port+1))
pdu_outlet=$((pdu_outlet+1))
done
@ -1430,6 +1452,8 @@ function enroll_nodes {
node_prefix=$(get_ironic_node_prefix)
if [[ "$IRONIC_IS_HARDWARE" == "False" ]]; then
local interface_info
interface_info=$(echo $hardware_info | awk '{print $1}')
local ironic_node_cpu=$IRONIC_VM_SPECS_CPU
local ironic_node_ram=$IRONIC_VM_SPECS_RAM
local ironic_node_disk=$IRONIC_VM_SPECS_DISK
@ -1481,8 +1505,8 @@ function enroll_nodes {
fi
if [[ "$IRONIC_IS_HARDWARE" == "False" ]]; then
local mac_address
mac_address=$(echo $hardware_info | awk '{print $1}')
local interface_info
interface_info=$(echo $hardware_info | awk '{print $1}')
if is_deployed_by_ipmitool; then
local vbmc_port
@ -1494,17 +1518,16 @@ function enroll_nodes {
node_options+=" -i snmp_outlet=$pdu_outlet"
fi
# Local-link-connection options
if [[ "${IRONIC_USE_LINK_LOCAL}" == "True" ]]; then
local llc_opts=""
if [[ "${IRONIC_USE_LINK_LOCAL}" == "True" ]]; then
local switch_info
local switch_id
local port_id
switch_info=$(echo $hardware_info |awk '{print $4}')
switch_id=$(echo $hardware_info |awk '{print $5}')
port_id=$(echo $hardware_info |awk '{print $6}')
switch_id=$(echo $hardware_info |awk '{print $4}')
switch_info=$(echo $hardware_info |awk '{print $5}')
llc_opts="-l switch_id=${switch_id} -l switch_info=${switch_info} -l port_id=${port_id}"
# NOTE(vsaienko) we will add port_id later in the code.
llc_opts="-l switch_id=${switch_id} -l switch_info=${switch_info} "
local ironic_api_version='--ironic-api-version latest'
fi
@ -1608,7 +1631,19 @@ function enroll_nodes {
# In case we using portgroups, we should API version that support them.
# Othervise API will return 406 ERROR
ironic $ironic_api_version port-create --address $mac_address --node $node_id $llc_opts
# NOTE(vsaienko) interface_info is in the following format here:
# mac1,tap-node0i1;mac2,tap-node0i2;...;macN,tap-node0iN
for info in ${interface_info//;/ }; do
local mac_address=""
local port_id=""
local llc_port_opt=""
mac_address=$(echo $info| awk -F ',' '{print $1}')
port_id=$(echo $info| awk -F ',' '{print $2}')
if [[ "${IRONIC_USE_LINK_LOCAL}" == "True" ]]; then
llc_port_opt+=" -l port_id=${port_id} "
fi
ironic $ironic_api_version port-create --address $mac_address --node $node_id $llc_opts $llc_port_opt
done
# NOTE(vsaienko) use node-update instead of specifying network_interface
# during node creation. If node is added with latest version of API it
@ -1955,12 +1990,11 @@ function cleanup_baremetal_basic_ops {
local vm_name
for vm_name in $(_ironic_bm_vm_names); do
sudo su $STACK_USER -c "$IRONIC_SCRIPTS_DIR/cleanup-node.sh $vm_name"
# Cleanup node bridge/interfaces
sudo ip link set ovs-$vm_name down
sudo ip link set br-$vm_name down
sudo ovs-vsctl del-port ovs-$vm_name
sudo ip link del dev ovs-$vm_name
sudo ip link del dev br-$vm_name
for i in $(seq 1 $IRONIC_VM_INTERFACE_COUNT); do
sudo ip tuntap del dev tap-${vm_name}i${i} mode tap
done
done
sudo ovs-vsctl --if-exists del-br $IRONIC_VM_NETWORK_BRIDGE

7
devstack/tools/ironic/scripts/configure-vm.py
View File

@ -71,9 +71,8 @@ def main():
help="What boot device to use (hd/network).")
parser.add_argument('--libvirt-nic-driver', default='virtio',
help='The libvirt network driver to use')
parser.add_argument('--bridge', default="br-seed",
help='The linux bridge name to use for seeding \
the baremetal pseudo-node\'s OS image')
parser.add_argument('--interface-count', default=1, type=int,
help='The number of interfaces to add to VM.'),
parser.add_argument('--console-log',
help='File to log console')
parser.add_argument('--emulator', default=None,
@ -99,7 +98,7 @@ def main():
'memory': args.memory,
'cpus': args.cpus,
'bootdev': args.bootdev,
'bridge': args.bridge,
'interface_count': args.interface_count,
'nicdriver': args.libvirt_nic_driver,
'emulator': args.emulator,
'disk_format': args.disk_format,

27
devstack/tools/ironic/scripts/create-node.sh
View File

@ -12,10 +12,12 @@ export PS4='+ ${BASH_SOURCE:-}:${FUNCNAME[0]:-}:L${LINENO:-}: '
# Keep track of the DevStack directory
TOP_DIR=$(cd $(dirname "$0")/.. && pwd)
while getopts "n:c:m:d:a:b:e:E:p:o:f:l:L:N:" arg; do
while getopts "n:c:i:m:M:d:a:b:e:E:p:o:f:l:L:N:" arg; do
case $arg in
n) NAME=$OPTARG;;
c) CPU=$OPTARG;;
i) INTERFACE_COUNT=$OPTARG;;
M) INTERFACE_MTU=$OPTARG;;
m) MEM=$(( 1024 * OPTARG ));;
# Extra G to allow fuzz for partition table : flavor size and registered
# size need to be different to actual size.
@ -88,12 +90,15 @@ fi
# it will be plugged to OVS.
# This is needed in order to have interface in OVS even
# when VM is in shutdown state
INTERFACE_COUNT=${INTERFACE_COUNT:-1}
sudo brctl addbr br-$NAME
sudo ip link set br-$NAME up
sudo ovs-vsctl add-port $BRIDGE ovs-$NAME -- set Interface ovs-$NAME type=internal
sudo ip link set ovs-$NAME up
sudo brctl addif br-$NAME ovs-$NAME
for int in $(seq 1 $INTERFACE_COUNT); do
tapif=tap-${NAME}i${int}
sudo ip tuntap add dev $tapif mode tap
sudo ip link set $tapif mtu $INTERFACE_MTU
sudo ip link set $tapif up
sudo ovs-vsctl add-port $BRIDGE $tapif
done
if ! virsh list --all | grep -q $NAME; then
virsh vol-list --pool $LIBVIRT_STORAGE_POOL | grep -q $VOL_NAME &&
@ -110,7 +115,8 @@ if ! virsh list --all | grep -q $NAME; then
$TOP_DIR/scripts/configure-vm.py \
--bootdev network --name $NAME --image "$volume_path" \
--arch $ARCH --cpus $CPU --memory $MEM --libvirt-nic-driver $LIBVIRT_NIC_DRIVER \
--bridge br-$NAME --disk-format $DISK_FORMAT $VM_LOGGING --engine $ENGINE $UEFI_OPTS $vm_opts >&2
--disk-format $DISK_FORMAT $VM_LOGGING --engine $ENGINE $UEFI_OPTS $vm_opts \
--interface-count $INTERFACE_COUNT >&2
# Createa Virtual BMC for the node if IPMI is used
if [[ $(type -P vbmc) != "" ]]; then
@ -119,7 +125,6 @@ if ! virsh list --all | grep -q $NAME; then
fi
fi
# echo mac
VM_MAC=$(virsh dumpxml $NAME | grep "mac address" | head -1 | cut -d\' -f2)
switch_id=$(ip link show dev $BRIDGE | egrep -o "ether [A-Za-z0-9:]+"|sed "s/ether\ //")
echo $VM_MAC $VBMC_PORT $PDU_OUTLET $BRIDGE $switch_id ovs-$NAME
# echo mac in format mac1,ovs-node-0i1;mac2,ovs-node-0i2;...;macN,ovs-node0iN
VM_MAC=$(echo -n $(virsh domiflist $NAME |awk '/tap-/{print $5","$1}')|tr ' ' ';')
echo -n "$VM_MAC $VBMC_PORT $PDU_OUTLET"

9
devstack/tools/ironic/templates/vm.xml
View File

@ -44,11 +44,14 @@
<controller type='ide' index='0'>
<address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x1'/>
</controller>
<interface type='bridge'>
<source bridge='{{ bridge }}'/>
{% for n in range(1, interface_count+1) %}
<interface type='ethernet'>
<target dev='{{ "tap-" + name + "i" + n|string }}'/>
<model type='{{ nicdriver }}'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x04' function='0x0'/>
<script path='no'/>
<address type='pci' domain='0x0000' bus='0x01' slot='{{ "0x0" + n|string }}' function='0x0'/>
</interface>
{% endfor %}
<input type='mouse' bus='ps2'/>
<graphics type='vnc' port='-1' autoport='yes' listen='0.0.0.0'/>
<video>