openstack/charm-nova-compute
Code Issues Proposed changes

Prevent repeated nova-compute service restart

Browse Source 
Once a ceph broker request has completed and the
nova-compute service has been restarted as a result,
ensure that this does not get retriggered when the
ceph relation fires as a result of a peer unit's data
coming onto the wire and no new information is provided
to the local unit.

Change-Id: Ie359a0ec9af7edfb9d453dcf4dbd9880af324d37
Closes-Bug: 1694963
This commit is contained in:
Edward Hope-Morley 2017-06-07 18:39:10 +01:00
parent 3515641f44
commit cff1cfa5a4
9 changed files with 243 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
hooks/charmhelpers/contrib/network/ip.py
View File

@ -244,9 +244,11 @@ def is_ipv6_disabled():
result = subprocess.check_output(
['sysctl', 'net.ipv6.conf.all.disable_ipv6'],
stderr=subprocess.STDOUT)
return "net.ipv6.conf.all.disable_ipv6 = 1" in result
except subprocess.CalledProcessError:
return True
if six.PY3:
result = result.decode('UTF-8')
return "net.ipv6.conf.all.disable_ipv6 = 1" in result
def get_iface_addr(iface='eth0', inet_type='AF_INET', inc_aliases=False,

101
hooks/charmhelpers/contrib/openstack/amulet/utils.py
View File

@ -25,9 +25,12 @@ import urlparse
import cinderclient.v1.client as cinder_client
import glanceclient.v1.client as glance_client
import heatclient.v1.client as heat_client
import keystoneclient.v2_0 as keystone_client
from keystoneclient.auth.identity import v3 as keystone_id_v3
from keystoneclient import session as keystone_session
from keystoneclient.v2_0 import client as keystone_client
from keystoneauth1.identity import (
v3,
v2,
)
from keystoneauth1 import session as keystone_session
from keystoneclient.v3 import client as keystone_client_v3
from novaclient import exceptions
@ -368,12 +371,20 @@ class OpenStackAmuletUtils(AmuletUtils):
port)
if not api_version or api_version == 2:
ep = base_ep + "/v2.0"
return keystone_client.Client(username=username, password=password,
tenant_name=project_name,
auth_url=ep)
auth = v2.Password(
username=username,
password=password,
tenant_name=project_name,
auth_url=ep
)
sess = keystone_session.Session(auth=auth)
client = keystone_client.Client(session=sess)
# This populates the client.service_catalog
client.auth_ref = auth.get_access(sess)
return client
else:
ep = base_ep + "/v3"
auth = keystone_id_v3.Password(
auth = v3.Password(
user_domain_name=user_domain_name,
username=username,
password=password,
@ -382,36 +393,45 @@ class OpenStackAmuletUtils(AmuletUtils):
project_name=project_name,
auth_url=ep
)
return keystone_client_v3.Client(
session=keystone_session.Session(auth=auth)
)
sess = keystone_session.Session(auth=auth)
client = keystone_client_v3.Client(session=sess)
# This populates the client.service_catalog
client.auth_ref = auth.get_access(sess)
return client
def authenticate_keystone_admin(self, keystone_sentry, user, password,
tenant=None, api_version=None,
keystone_ip=None):
keystone_ip=None, user_domain_name=None,
project_domain_name=None,
project_name=None):
"""Authenticates admin user with the keystone admin endpoint."""
self.log.debug('Authenticating keystone admin...')
if not keystone_ip:
keystone_ip = keystone_sentry.info['public-address']
user_domain_name = None
domain_name = None
if api_version == 3:
# To support backward compatibility usage of this function
if not project_name:
project_name = tenant
if api_version == 3 and not user_domain_name:
user_domain_name = 'admin_domain'
domain_name = user_domain_name
if api_version == 3 and not project_domain_name:
project_domain_name = 'admin_domain'
if api_version == 3 and not project_name:
project_name = 'admin'
return self.authenticate_keystone(keystone_ip, user, password,
project_name=tenant,
api_version=api_version,
user_domain_name=user_domain_name,
domain_name=domain_name,
admin_port=True)
return self.authenticate_keystone(
keystone_ip, user, password,
api_version=api_version,
user_domain_name=user_domain_name,
project_domain_name=project_domain_name,
project_name=project_name,
admin_port=True)
def authenticate_keystone_user(self, keystone, user, password, tenant):
"""Authenticates a regular user with the keystone public endpoint."""
self.log.debug('Authenticating keystone user ({})...'.format(user))
ep = keystone.service_catalog.url_for(service_type='identity',
endpoint_type='publicURL')
interface='publicURL')
keystone_ip = urlparse.urlparse(ep).hostname
return self.authenticate_keystone(keystone_ip, user, password,
@ -421,22 +441,32 @@ class OpenStackAmuletUtils(AmuletUtils):
"""Authenticates admin user with glance."""
self.log.debug('Authenticating glance admin...')
ep = keystone.service_catalog.url_for(service_type='image',
endpoint_type='adminURL')
return glance_client.Client(ep, token=keystone.auth_token)
interface='adminURL')
if keystone.session:
return glance_client.Client(ep, session=keystone.session)
else:
return glance_client.Client(ep, token=keystone.auth_token)
def authenticate_heat_admin(self, keystone):
"""Authenticates the admin user with heat."""
self.log.debug('Authenticating heat admin...')
ep = keystone.service_catalog.url_for(service_type='orchestration',
endpoint_type='publicURL')
return heat_client.Client(endpoint=ep, token=keystone.auth_token)
interface='publicURL')
if keystone.session:
return heat_client.Client(endpoint=ep, session=keystone.session)
else:
return heat_client.Client(endpoint=ep, token=keystone.auth_token)
def authenticate_nova_user(self, keystone, user, password, tenant):
"""Authenticates a regular user with nova-api."""
self.log.debug('Authenticating nova user ({})...'.format(user))
ep = keystone.service_catalog.url_for(service_type='identity',
endpoint_type='publicURL')
if novaclient.__version__[0] >= "7":
interface='publicURL')
if keystone.session:
return nova_client.Client(NOVA_CLIENT_VERSION,
session=keystone.session,
auth_url=ep)
elif novaclient.__version__[0] >= "7":
return nova_client.Client(NOVA_CLIENT_VERSION,
username=user, password=password,
project_name=tenant, auth_url=ep)
@ -449,12 +479,15 @@ class OpenStackAmuletUtils(AmuletUtils):
"""Authenticates a regular user with swift api."""
self.log.debug('Authenticating swift user ({})...'.format(user))
ep = keystone.service_catalog.url_for(service_type='identity',
endpoint_type='publicURL')
return swiftclient.Connection(authurl=ep,
user=user,
key=password,
tenant_name=tenant,
auth_version='2.0')
interface='publicURL')
if keystone.session:
return swiftclient.Connection(session=keystone.session)
else:
return swiftclient.Connection(authurl=ep,
user=user,
key=password,
tenant_name=tenant,
auth_version='2.0')
def create_flavor(self, nova, name, ram, vcpus, disk, flavorid="auto",
ephemeral=0, swap=0, rxtx_factor=1.0, is_public=True):

2
hooks/charmhelpers/contrib/openstack/keystone.py
View File

@ -29,7 +29,7 @@ def get_api_suffix(api_version):
@returns the api suffix formatted according to the given api
version
"""
return 'v2.0' if api_version in (2, "2.0") else 'v3'
return 'v2.0' if api_version in (2, "2", "2.0") else 'v3'
def format_endpoint(schema, addr, port, api_version):

42
hooks/charmhelpers/contrib/storage/linux/ceph.py
View File

@ -63,6 +63,7 @@ from charmhelpers.core.host import (
from charmhelpers.fetch import (
apt_install,
)
from charmhelpers.core.unitdata import kv
from charmhelpers.core.kernel import modprobe
from charmhelpers.contrib.openstack.utils import config_flags_parser
@ -1314,6 +1315,47 @@ def send_request_if_needed(request, relation='ceph'):
relation_set(relation_id=rid, broker_req=request.request)
def is_broker_action_done(action, rid=None, unit=None):
"""Check whether broker action has completed yet.
@param action: name of action to be performed
@returns True if action complete otherwise False
"""
rdata = relation_get(rid, unit) or {}
broker_rsp = rdata.get(get_broker_rsp_key())
if not broker_rsp:
return False
rsp = CephBrokerRsp(broker_rsp)
unit_name = local_unit().partition('/')[2]
key = "unit_{}_ceph_broker_action.{}".format(unit_name, action)
kvstore = kv()
val = kvstore.get(key=key)
if val and val == rsp.request_id:
return True
return False
def mark_broker_action_done(action, rid=None, unit=None):
"""Mark action as having been completed.
@param action: name of action to be performed
@returns None
"""
rdata = relation_get(rid, unit) or {}
broker_rsp = rdata.get(get_broker_rsp_key())
if not broker_rsp:
return
rsp = CephBrokerRsp(broker_rsp)
unit_name = local_unit().partition('/')[2]
key = "unit_{}_ceph_broker_action.{}".format(unit_name, action)
kvstore = kv()
kvstore.set(key=key, value=rsp.request_id)
kvstore.flush()
class CephConfContext(object):
"""Ceph config (ceph.conf) context.

7
hooks/nova_compute_hooks.py
View File

@ -67,6 +67,8 @@ from charmhelpers.contrib.storage.linux.ceph import (
delete_keyring,
send_request_if_needed,
is_request_complete,
is_broker_action_done,
mark_broker_action_done,
)
from charmhelpers.payload.execd import execd_preinstall
from nova_compute_utils import (
@ -398,8 +400,11 @@ def ceph_changed(rid=None, unit=None):
log('Request complete')
# Ensure that nova-compute is restarted since only now can we
# guarantee that ceph resources are ready, but only if not paused.
if not is_unit_paused_set():
if (not is_unit_paused_set() and
not is_broker_action_done('nova_compute_restart', rid,
unit)):
service_restart('nova-compute')
mark_broker_action_done('nova_compute_restart', rid, unit)
else:
send_request_if_needed(get_ceph_request())

4
tests/charmhelpers/contrib/network/ip.py
View File

@ -244,9 +244,11 @@ def is_ipv6_disabled():
result = subprocess.check_output(
['sysctl', 'net.ipv6.conf.all.disable_ipv6'],
stderr=subprocess.STDOUT)
return "net.ipv6.conf.all.disable_ipv6 = 1" in result
except subprocess.CalledProcessError:
return True
if six.PY3:
result = result.decode('UTF-8')
return "net.ipv6.conf.all.disable_ipv6 = 1" in result
def get_iface_addr(iface='eth0', inet_type='AF_INET', inc_aliases=False,

101
tests/charmhelpers/contrib/openstack/amulet/utils.py
View File

@ -25,9 +25,12 @@ import urlparse
import cinderclient.v1.client as cinder_client
import glanceclient.v1.client as glance_client
import heatclient.v1.client as heat_client
import keystoneclient.v2_0 as keystone_client
from keystoneclient.auth.identity import v3 as keystone_id_v3
from keystoneclient import session as keystone_session
from keystoneclient.v2_0 import client as keystone_client
from keystoneauth1.identity import (
v3,
v2,
)
from keystoneauth1 import session as keystone_session
from keystoneclient.v3 import client as keystone_client_v3
from novaclient import exceptions
@ -368,12 +371,20 @@ class OpenStackAmuletUtils(AmuletUtils):
port)
if not api_version or api_version == 2:
ep = base_ep + "/v2.0"
return keystone_client.Client(username=username, password=password,
tenant_name=project_name,
auth_url=ep)
auth = v2.Password(
username=username,
password=password,
tenant_name=project_name,
auth_url=ep
)
sess = keystone_session.Session(auth=auth)
client = keystone_client.Client(session=sess)
# This populates the client.service_catalog
client.auth_ref = auth.get_access(sess)
return client
else:
ep = base_ep + "/v3"
auth = keystone_id_v3.Password(
auth = v3.Password(
user_domain_name=user_domain_name,
username=username,
password=password,
@ -382,36 +393,45 @@ class OpenStackAmuletUtils(AmuletUtils):
project_name=project_name,
auth_url=ep
)
return keystone_client_v3.Client(
session=keystone_session.Session(auth=auth)
)
sess = keystone_session.Session(auth=auth)
client = keystone_client_v3.Client(session=sess)
# This populates the client.service_catalog
client.auth_ref = auth.get_access(sess)
return client
def authenticate_keystone_admin(self, keystone_sentry, user, password,
tenant=None, api_version=None,
keystone_ip=None):
keystone_ip=None, user_domain_name=None,
project_domain_name=None,
project_name=None):
"""Authenticates admin user with the keystone admin endpoint."""
self.log.debug('Authenticating keystone admin...')
if not keystone_ip:
keystone_ip = keystone_sentry.info['public-address']
user_domain_name = None
domain_name = None
if api_version == 3:
# To support backward compatibility usage of this function
if not project_name:
project_name = tenant
if api_version == 3 and not user_domain_name:
user_domain_name = 'admin_domain'
domain_name = user_domain_name
if api_version == 3 and not project_domain_name:
project_domain_name = 'admin_domain'
if api_version == 3 and not project_name:
project_name = 'admin'
return self.authenticate_keystone(keystone_ip, user, password,
project_name=tenant,
api_version=api_version,
user_domain_name=user_domain_name,
domain_name=domain_name,
admin_port=True)
return self.authenticate_keystone(
keystone_ip, user, password,
api_version=api_version,
user_domain_name=user_domain_name,
project_domain_name=project_domain_name,
project_name=project_name,
admin_port=True)
def authenticate_keystone_user(self, keystone, user, password, tenant):
"""Authenticates a regular user with the keystone public endpoint."""
self.log.debug('Authenticating keystone user ({})...'.format(user))
ep = keystone.service_catalog.url_for(service_type='identity',
endpoint_type='publicURL')
interface='publicURL')
keystone_ip = urlparse.urlparse(ep).hostname
return self.authenticate_keystone(keystone_ip, user, password,
@ -421,22 +441,32 @@ class OpenStackAmuletUtils(AmuletUtils):
"""Authenticates admin user with glance."""
self.log.debug('Authenticating glance admin...')
ep = keystone.service_catalog.url_for(service_type='image',
endpoint_type='adminURL')
return glance_client.Client(ep, token=keystone.auth_token)
interface='adminURL')
if keystone.session:
return glance_client.Client(ep, session=keystone.session)
else:
return glance_client.Client(ep, token=keystone.auth_token)
def authenticate_heat_admin(self, keystone):
"""Authenticates the admin user with heat."""
self.log.debug('Authenticating heat admin...')
ep = keystone.service_catalog.url_for(service_type='orchestration',
endpoint_type='publicURL')
return heat_client.Client(endpoint=ep, token=keystone.auth_token)
interface='publicURL')
if keystone.session:
return heat_client.Client(endpoint=ep, session=keystone.session)
else:
return heat_client.Client(endpoint=ep, token=keystone.auth_token)
def authenticate_nova_user(self, keystone, user, password, tenant):
"""Authenticates a regular user with nova-api."""
self.log.debug('Authenticating nova user ({})...'.format(user))
ep = keystone.service_catalog.url_for(service_type='identity',
endpoint_type='publicURL')
if novaclient.__version__[0] >= "7":
interface='publicURL')
if keystone.session:
return nova_client.Client(NOVA_CLIENT_VERSION,
session=keystone.session,
auth_url=ep)
elif novaclient.__version__[0] >= "7":
return nova_client.Client(NOVA_CLIENT_VERSION,
username=user, password=password,
project_name=tenant, auth_url=ep)
@ -449,12 +479,15 @@ class OpenStackAmuletUtils(AmuletUtils):
"""Authenticates a regular user with swift api."""
self.log.debug('Authenticating swift user ({})...'.format(user))
ep = keystone.service_catalog.url_for(service_type='identity',
endpoint_type='publicURL')
return swiftclient.Connection(authurl=ep,
user=user,
key=password,
tenant_name=tenant,
auth_version='2.0')
interface='publicURL')
if keystone.session:
return swiftclient.Connection(session=keystone.session)
else:
return swiftclient.Connection(authurl=ep,
user=user,
key=password,
tenant_name=tenant,
auth_version='2.0')
def create_flavor(self, nova, name, ram, vcpus, disk, flavorid="auto",
ephemeral=0, swap=0, rxtx_factor=1.0, is_public=True):

42
tests/charmhelpers/contrib/storage/linux/ceph.py
View File

@ -63,6 +63,7 @@ from charmhelpers.core.host import (
from charmhelpers.fetch import (
apt_install,
)
from charmhelpers.core.unitdata import kv
from charmhelpers.core.kernel import modprobe
from charmhelpers.contrib.openstack.utils import config_flags_parser
@ -1314,6 +1315,47 @@ def send_request_if_needed(request, relation='ceph'):
relation_set(relation_id=rid, broker_req=request.request)
def is_broker_action_done(action, rid=None, unit=None):
"""Check whether broker action has completed yet.
@param action: name of action to be performed
@returns True if action complete otherwise False
"""
rdata = relation_get(rid, unit) or {}
broker_rsp = rdata.get(get_broker_rsp_key())
if not broker_rsp:
return False
rsp = CephBrokerRsp(broker_rsp)
unit_name = local_unit().partition('/')[2]
key = "unit_{}_ceph_broker_action.{}".format(unit_name, action)
kvstore = kv()
val = kvstore.get(key=key)
if val and val == rsp.request_id:
return True
return False
def mark_broker_action_done(action, rid=None, unit=None):
"""Mark action as having been completed.
@param action: name of action to be performed
@returns None
"""
rdata = relation_get(rid, unit) or {}
broker_rsp = rdata.get(get_broker_rsp_key())
if not broker_rsp:
return
rsp = CephBrokerRsp(broker_rsp)
unit_name = local_unit().partition('/')[2]
key = "unit_{}_ceph_broker_action.{}".format(unit_name, action)
kvstore = kv()
kvstore.set(key=key, value=rsp.request_id)
kvstore.flush()
class CephConfContext(object):
"""Ceph config (ceph.conf) context.

13
unit_tests/test_nova_compute_hooks.py
View File

@ -528,10 +528,14 @@ class NovaComputeRelationsTests(CharmTestCase):
'Could not create ceph keyring: peer not ready?'
)
@patch.object(hooks, 'mark_broker_action_done')
@patch.object(hooks, 'is_broker_action_done')
@patch('nova_compute_context.service_name')
@patch.object(hooks, 'CONFIGS')
def test_ceph_changed_with_key_and_relation_data(self, configs,
service_name):
service_name,
is_broker_action_done,
mark_broker_action_done):
self.test_config.set('libvirt-image-backend', 'rbd')
self.is_request_complete.return_value = True
self.assert_libvirt_rbd_imagebackend_allowed.return_value = True
@ -540,7 +544,9 @@ class NovaComputeRelationsTests(CharmTestCase):
configs.write = MagicMock()
service_name.return_value = 'nova-compute'
self.ensure_ceph_keyring.return_value = True
is_broker_action_done.return_value = False
hooks.ceph_changed()
self.assertTrue(mark_broker_action_done.called)
ex = [
call('/var/lib/charm/nova-compute/ceph.conf'),
call('/etc/ceph/secret.xml'),
@ -549,6 +555,11 @@ class NovaComputeRelationsTests(CharmTestCase):
self.assertEqual(ex, configs.write.call_args_list)
self.service_restart.assert_called_with('nova-compute')
is_broker_action_done.return_value = True
mark_broker_action_done.reset_mock()
hooks.ceph_changed()
self.assertFalse(mark_broker_action_done.called)
@patch('charmhelpers.contrib.storage.linux.ceph.CephBrokerRq'
'.add_op_request_access_to_group')
@patch('charmhelpers.contrib.storage.linux.ceph.CephBrokerRq'