From ffff1cb13281894dcbbdfa4bc6f63fec4673f10c Mon Sep 17 00:00:00 2001 From: ZhiQiang Fan Date: Wed, 5 Mar 2014 10:58:59 +0800 Subject: [PATCH] Skip central agent interval_task when keystone fails There is a chance when ceilometer-agent-central try to trigger interval_task but keystone is not available, then an exception is raised but not caught, the outside oslo.loopingcall will stop calling the task function, finally leading to the agent-central service is no longer available forever, even the keystone is available again. We can skip the particular interval_task when keystone is not available. Change-Id: I0849cafadac8fb8a7670aaa4cc76dc708bdb25a1 Closes-Bug: #1287613 --- ceilometer/central/manager.py | 23 ++++++++++++++--------- ceilometer/tests/central/test_manager.py | 9 +++++++++ 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/ceilometer/central/manager.py b/ceilometer/central/manager.py index e2b1542ca..8b4817b2f 100644 --- a/ceilometer/central/manager.py +++ b/ceilometer/central/manager.py @@ -20,6 +20,7 @@ from keystoneclient.v2_0 import client as ksclient from oslo.config import cfg from ceilometer import agent +from ceilometer.openstack.common.gettextutils import _ # noqa from ceilometer.openstack.common import log from ceilometer.openstack.common import service as os_service from ceilometer import service @@ -35,15 +36,19 @@ class AgentManager(agent.AgentManager): super(AgentManager, self).__init__('central') def interval_task(self, task): - self.keystone = ksclient.Client( - username=cfg.CONF.service_credentials.os_username, - password=cfg.CONF.service_credentials.os_password, - tenant_id=cfg.CONF.service_credentials.os_tenant_id, - tenant_name=cfg.CONF.service_credentials.os_tenant_name, - cacert=cfg.CONF.service_credentials.os_cacert, - auth_url=cfg.CONF.service_credentials.os_auth_url, - region_name=cfg.CONF.service_credentials.os_region_name, - insecure=cfg.CONF.service_credentials.insecure) + try: + self.keystone = ksclient.Client( + username=cfg.CONF.service_credentials.os_username, + password=cfg.CONF.service_credentials.os_password, + tenant_id=cfg.CONF.service_credentials.os_tenant_id, + tenant_name=cfg.CONF.service_credentials.os_tenant_name, + cacert=cfg.CONF.service_credentials.os_cacert, + auth_url=cfg.CONF.service_credentials.os_auth_url, + region_name=cfg.CONF.service_credentials.os_region_name, + insecure=cfg.CONF.service_credentials.insecure) + except Exception as e: + LOG.error(_('Skip interval_task because Keystone error: %s'), e) + return super(AgentManager, self).interval_task(task) diff --git a/ceilometer/tests/central/test_manager.py b/ceilometer/tests/central/test_manager.py index 0a85dc9e8..2258ce483 100644 --- a/ceilometer/tests/central/test_manager.py +++ b/ceilometer/tests/central/test_manager.py @@ -50,3 +50,12 @@ class TestRunTasks(agentbase.BaseAgentManagerTestCase): polling_tasks = self.mgr.setup_polling_tasks() self.mgr.interval_task(polling_tasks.values()[0]) self.assertTrue(self.Pollster.resources) + + def test_skip_task_when_keystone_fail(self): + """Test for https://bugs.launchpad.net/ceilometer/+bug/1287613.""" + self.useFixture(mockpatch.Patch( + 'keystoneclient.v2_0.client.Client', + side_effect=Exception)) + polling_tasks = self.mgr.setup_polling_tasks() + self.mgr.interval_task(polling_tasks.values()[0]) + self.assertFalse(self.Pollster.samples)