From 7860f2ba3189b0361693c8ee9c65d8d03fb115d6 Mon Sep 17 00:00:00 2001 From: Sean Dague Date: Tue, 17 Nov 2015 11:59:07 -0500 Subject: [PATCH] install ebtables locking workaround ebtables is racing with itself when nova and libvirt attempt to create rules at the same time in the nat table. ebtables now has an explicit --concurrent flag, that all tools must opt into to prevent ebtables from inherently being unsafe to run. libvirt gained this support in 1.2.11, which is too new for our ubuntu primary testing environment. Nova still hasn't added this support, though even if it did, we'd run into the issue with libvirt. We can do the most ghetto thing possible and create a wrapper for ebtables that does explicit locking on it's own. It's pretty terrible, but it should work. And it is the kind of work around that people unable to upgrade libvirt will probably need to do. This is an opt in value which we should set in the gate to True. Related-Bug: #1501558 Change-Id: Ic6fa847eba34c21593b9df86a1c2c179534d0ba5 --- files/ebtables.workaround | 23 +++++++++++++++++++++ lib/nova_plugins/functions-libvirt | 5 +++++ stackrc | 10 +++++++++ tools/install_ebtables_workaround.sh | 31 ++++++++++++++++++++++++++++ 4 files changed, 69 insertions(+) create mode 100644 files/ebtables.workaround create mode 100755 tools/install_ebtables_workaround.sh diff --git a/files/ebtables.workaround b/files/ebtables.workaround new file mode 100644 index 0000000000..c8af51fad5 --- /dev/null +++ b/files/ebtables.workaround @@ -0,0 +1,23 @@ +#!/bin/bash +# +# Copyright 2015 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# +# This is a terrible, terrible, truly terrible work around for +# environments that have libvirt < 1.2.11. ebtables requires that you +# specifically tell it you would like to not race and get punched in +# the face when 2 run at the same time with a --concurrent flag. + +flock -w 300 /var/lock/ebtables.nova /sbin/ebtables.real $@ diff --git a/lib/nova_plugins/functions-libvirt b/lib/nova_plugins/functions-libvirt index 78c59786d8..045fc8b919 100644 --- a/lib/nova_plugins/functions-libvirt +++ b/lib/nova_plugins/functions-libvirt @@ -31,6 +31,11 @@ function install_libvirt { fi install_package libvirt-bin libvirt-dev pip_install_gr libvirt-python + if [[ "$EBTABLES_RACE_FIX" == "True" ]]; then + # Work around for bug #1501558. We can remove this once we + # get to a version of Ubuntu that has new enough libvirt. + TOP_DIR=$TOP_DIR $TOP_DIR/tools/install_ebtables_workaround.sh + fi #pip_install_gr elif is_fedora || is_suse; then install_package kvm diff --git a/stackrc b/stackrc index 76a5756dde..53ed25d83e 100644 --- a/stackrc +++ b/stackrc @@ -769,6 +769,16 @@ GIT_DEPTH=${GIT_DEPTH:-0} # Use native SSL for servers in ``SSL_ENABLED_SERVICES`` USE_SSL=$(trueorfalse False USE_SSL) +# ebtables is inherently racey. If you run it by two or more processes +# simultaneously it will collide, badly, in the kernel and produce +# failures or corruption of ebtables. The only way around it is for +# all tools running ebtables to only ever do so with the --concurrent +# flag. This requires libvirt >= 1.2.11. +# +# If you don't have this then the following work around will replace +# ebtables with a wrapper script so that it is safe to run without +# that flag. +EBTABLES_RACE_FIX=$(trueorfalse False EBTABLES_RACE_FIX) # Following entries need to be last items in file diff --git a/tools/install_ebtables_workaround.sh b/tools/install_ebtables_workaround.sh new file mode 100755 index 0000000000..45ced87f13 --- /dev/null +++ b/tools/install_ebtables_workaround.sh @@ -0,0 +1,31 @@ +#!/bin/bash -eu +# +# Copyright 2015 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# +# This replaces the ebtables on your system with a wrapper script that +# does implicit locking. This is needed if libvirt < 1.2.11 on your platform. + +EBTABLES=/sbin/ebtables +EBTABLESREAL=/sbin/ebtables.real +FILES=$TOP_DIR/files + +if [[ -f "$EBTABLES" ]]; then + if file $EBTABLES | grep ELF; then + sudo mv $EBTABLES $EBTABLESREAL + sudo install -m 0755 $FILES/ebtables.workaround $EBTABLES + echo "Replaced ebtables with locking workaround" + fi +fi