Add unit test to show herd protection in action

The discussion around the previous patch had a simple test case which
was used to show invalid and valid behavior. Turns out, that can be a
test.

Change-Id: I39550d44f4e83f803624a833a081101bad9dc545

shade/tests/unit/test_caching.py

@@ -11,7 +11,9 @@
 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 # License for the specific language governing permissions and limitations
 # under the License.
+import concurrent
 import tempfile
+import time
 
 from glanceclient.v2 import shell
 import mock
@@ -139,6 +141,32 @@ class TestMemoryCache(base.TestCase):
             meta.obj_list_to_dict([project, project_b]),
             self.cloud.list_projects())
 
+    @mock.patch('shade.OpenStackCloud.nova_client')
+    def test_list_servers_herd(self, nova_mock):
+        self.cloud._SERVER_AGE = 0
+        fake_server = fakes.FakeServer('1234', '', 'ACTIVE')
+        nova_mock.servers.list.return_value = [fake_server]
+        with concurrent.futures.ThreadPoolExecutor(16) as pool:
+            for i in range(16):
+                pool.submit(lambda: self.cloud.list_servers())
+                # It's possible to race-condition 16 threads all in the
+                # single initial lock without a tiny sleep
+                time.sleep(0.001)
+        self.assertGreater(nova_mock.servers.list.call_count, 1)
+
+    @mock.patch('shade.OpenStackCloud.nova_client')
+    def test_list_servers_no_herd(self, nova_mock):
+        self.cloud._SERVER_AGE = 2
+        fake_server = fakes.FakeServer('1234', '', 'ACTIVE')
+        nova_mock.servers.list.return_value = [fake_server]
+        with concurrent.futures.ThreadPoolExecutor(16) as pool:
+            for i in range(16):
+                pool.submit(lambda: self.cloud.list_servers())
+                # It's possible to race-condition 16 threads all in the
+                # single initial lock without a tiny sleep
+                time.sleep(0.001)
+        self.assertEqual(1, nova_mock.servers.list.call_count)
+
     @mock.patch('shade.OpenStackCloud.cinder_client')
     def test_list_volumes(self, cinder_mock):
         fake_volume = fakes.FakeVolume('volume1', 'available',

test-requirements.txt

@@ -13,3 +13,4 @@ testscenarios>=0.4,<0.5
 testtools>=0.9.32
 warlock>=1.0.1,<2
 reno
+futures;python_version<'3.2'