From ed68b3119a83581b57f8a308ba5baf3f2c71d177 Mon Sep 17 00:00:00 2001 From: Kun Huang Date: Tue, 13 Oct 2015 22:02:15 +0800 Subject: [PATCH] Fix random miss in comsumer/publisher broker The error in bug report is comfirmed but reproduced randomly. Runing [0] 100 times and you could get about 3 failures on average. The reason is that publisher finished between L39 and L40 here [1]. Before this patch, we are starting comsumers first and then publisher. Before publisher doing publish(), comsumer threads are in while-true loop because the thread event is_published is not setted. At the moment publish work has been done, which means the is_published is setted, each comsumer thread could be in any position of the while-true loop. If it happens between L39 and L40, the broker will miss the job. If there is only one comsumer here (it is in our test case), the miss happens. To clarify it: 39 while True: 40 if not queue: <<< this moment, publisher finished >>> <<< which means queue is not empty >>> <<< and is_published is setted. >>> 41 if is_published.isSet(): 42 break 43 time.sleep(0.1) 44 continue 45 else: 46 try: 47 args = queue.popleft() 48 except IndexError: 49 # consumed by other thread 50 continue To fix this is very simple: do publish job BEFORE the comsumers start and remove the unrequired is_published :) [0] tox -e py27 tests.unit.plugins.openstack.context.keystone.test_users.UserGeneratorTestCase.test_users_and_tenants_in_context [1] https://github.com/openstack/rally/blob/master/rally/common/broker.py#L39-L40 Change-Id: Ieab25e531382e5c7832eeaecb962d6139a654b96 Closes-bug: #1417281 --- tests/unit/plugins/openstack/context/keystone/test_users.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/unit/plugins/openstack/context/keystone/test_users.py b/tests/unit/plugins/openstack/context/keystone/test_users.py index 523819c9..3e797c46 100644 --- a/tests/unit/plugins/openstack/context/keystone/test_users.py +++ b/tests/unit/plugins/openstack/context/keystone/test_users.py @@ -221,9 +221,8 @@ class UserGeneratorTestCase(test.ScenarioTestCase): "nova-network") nova_admin.networks.disassociate.assert_called_once_with(networks[0]) - @mock.patch("%s.broker.time.sleep" % CTX) @mock.patch("%s.keystone" % CTX) - def test__create_tenants(self, mock_keystone, mock_sleep): + def test__create_tenants(self, mock_keystone): user_generator = users.UserGenerator(self.context) user_generator.config["tenants"] = 1 tenants = user_generator._create_tenants() @@ -231,9 +230,8 @@ class UserGeneratorTestCase(test.ScenarioTestCase): id, tenant = tenants.popitem() self.assertIn("name", tenant) - @mock.patch("%s.broker.time.sleep" % CTX) @mock.patch("%s.keystone" % CTX) - def test__create_users(self, mock_keystone, mock_sleep): + def test__create_users(self, mock_keystone): user_generator = users.UserGenerator(self.context) user_generator.context["tenants"] = {"t1": {"id": "t1", "name": "t1"}, "t2": {"id": "t2", "name": "t2"}}