Merge "Declarative HTTP testing for the Ceilometer API"

.testr.conf

@@ -2,6 +2,8 @@
 test_command=OS_STDOUT_CAPTURE=${OS_STDOUT_CAPTURE:-1} \
              OS_STDERR_CAPTURE=${OS_STDERR_CAPTURE:-1} \
              OS_TEST_TIMEOUT=${OS_TEST_TIMEOUT:-600} \
-             ${PYTHON:-python} -m subunit.run discover ceilometer/tests -t . $LISTOPT $IDOPTION
+             ${PYTHON:-python} -m subunit.run discover ${OS_TEST_PATH:-./ceilometer/tests} -t . $LISTOPT $IDOPTION
 test_id_option=--load-list $IDFILE
 test_list_option=--list
+# NOTE(chdent): Only used/matches on gabbi-related tests.
+group_regex=(gabbi\.driver\.test_gabbi_[^_]+)_

ceilometer/tests/gabbi/fixtures.py

@@ -0,0 +1,116 @@
+#
+# Copyright 2015 Red Hat. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""Fixtures used during Gabbi-based test runs."""
+
+from __future__ import print_function
+
+import datetime
+import os
+import random
+from unittest import case
+import uuid
+
+from gabbi import fixture
+from oslo.config import fixture as fixture_config
+
+from ceilometer.publisher import utils
+from ceilometer import sample
+from ceilometer import service
+from ceilometer import storage
+
+
+# TODO(chdent): For now only MongoDB is supported, because of easy
+# database name handling and intentional focus on the API, not the
+# data store.
+ENGINES = ['MONGODB']
+
+
+class ConfigFixture(fixture.GabbiFixture):
+    """Establish the relevant configuration for a test run."""
+
+    def start_fixture(self):
+        """Set up config."""
+
+        service.prepare_service([])
+        conf = fixture_config.Config().conf
+        self.conf = conf
+        conf.import_opt('policy_file', 'ceilometer.openstack.common.policy')
+        conf.set_override('policy_file',
+                          os.path.abspath('etc/ceilometer/policy.json'))
+
+        # A special pipeline is required to use the direct publisher.
+        conf.set_override('pipeline_cfg_file',
+                          'etc/ceilometer/gabbi_pipeline.yaml')
+
+        # Determine the database connection.
+        db_url = None
+        for engine in ENGINES:
+            try:
+                db_url = os.environ['CEILOMETER_TEST_%s_URL' % engine]
+            except KeyError:
+                pass
+        if db_url is None:
+            raise case.SkipTest('No database connection configured')
+
+        database_name = '%s-%s' % (db_url, str(uuid.uuid4()))
+        conf.set_override('connection', database_name, group='database')
+        conf.set_override('metering_connection', '', group='database')
+        conf.set_override('event_connection', '', group='database')
+        conf.set_override('alarm_connection', '', group='database')
+
+        conf.set_override('pecan_debug', True, group='api')
+
+    def stop_fixture(self):
+        """Clean up the config."""
+        self.conf.reset()
+
+
+class SampleDataFixture(fixture.GabbiFixture):
+    """Instantiate some sample data for use in testing."""
+
+    def start_fixture(self):
+        """Create some samples."""
+        conf = fixture_config.Config().conf
+        self.conn = storage.get_connection_from_config(conf)
+        timestamp = datetime.datetime.utcnow()
+        project_id = str(uuid.uuid4())
+        self.source = str(uuid.uuid4())
+        resource_metadata = {'farmed_by': 'nancy'}
+
+        for name in ['cow', 'pig', 'sheep']:
+            resource_metadata.update({'breed': name}),
+            c = sample.Sample(name='livestock',
+                              type='gauge',
+                              unit='head',
+                              volume=int(10 * random.random()),
+                              user_id='farmerjon',
+                              project_id=project_id,
+                              resource_id=project_id,
+                              timestamp=timestamp,
+                              resource_metadata=resource_metadata,
+                              source=self.source,
+                              )
+            data = utils.meter_message_from_counter(
+                c, conf.publisher.telemetry_secret)
+            self.conn.record_metering_data(data)
+
+    def stop_fixture(self):
+        """Destroy the samples."""
+        # NOTE(chdent): print here for sake of info during testing.
+        # This will go away eventually.
+        print('resource',
+              self.conn.db.resource.remove({'source': self.source}))
+        print('meter', self.conn.db.meter.remove({'source': self.source}))

ceilometer/tests/gabbi/gabbits/basic.yaml

@@ -0,0 +1,24 @@
+#
+# Some simple tests just to confirm that the system works.
+#
+fixtures:
+    - ConfigFixture
+
+tests:
+
+# Root gives us some information on where to go from here.
+-    name: quick root check
+     url: /
+     response_headers:
+         content-type: application/json; charset=UTF-8
+     response_strings:
+         - '"base": "application/json"'
+     response_json_paths:
+         versions.values.[0].status: stable
+         versions.values.[0].media-types.[0].base: application/json
+
+# NOTE(chdent): Ideally since / has a links ref to /v2, /v2 ought not 404!
+-    name: v2 visit
+     desc: this demonstrates a bug in the info in /
+     url: $RESPONSE['versions.values.[0].links.[0].href']
+     status: 404

ceilometer/tests/gabbi/gabbits/capabilities.yaml

@@ -0,0 +1,16 @@
+#
+# Explore the capabilities API
+#
+fixtures:
+    - ConfigFixture
+
+tests:
+
+- name: get capabilities
+  desc: retrieve capabilities for the mongo store
+  url: /v2/capabilities
+  response_json_paths:
+      $.alarm_storage.['storage:production_ready']: true
+      $.event_storage.['storage:production_ready']: true
+      $.storage.['storage:production_ready']: true
+      $.api.['meters:pagination']: false

ceilometer/tests/gabbi/gabbits/clean-samples.yaml

@@ -0,0 +1,75 @@
+# Post a simple sample, sir, and the retrieve it in various ways.
+fixtures:
+    - ConfigFixture
+
+tests:
+- name: post sample for meter
+  desc: post a single sample
+  url: /v2/meters/apples
+  method: POST
+  request_headers:
+      content-type: application/json
+  data: |
+    [
+        {
+            "counter_name": "apples",
+            "project_id": "35b17138-b364-4e6a-a131-8f3099c5be68",
+            "user_id": "efd87807-12d2-4b38-9c70-5f5c2ac427ff",
+            "counter_unit": "instance",
+            "counter_volume": 1,
+            "resource_id": "bd9431c1-8d69-4ad3-803a-8d4a6b89fd36",
+            "resource_metadata": {
+                "name2": "value2",
+                "name1": "value1"
+            },
+            "counter_type": "gauge"
+        }
+    ]
+
+  response_json_paths:
+      $.[0].counter_name: apples
+  status: 200
+  response_headers:
+      content-type: application/json; charset=UTF-8
+
+# TODO(chdent): No location header in the response!!!!
+# - name: get sample
+#   desc: get the sample we just posted
+#   url: $LOCATION
+
+- name: get samples for meter
+  desc: get all the samples at that meter
+  url: /v2/meters/apples
+  response_json_paths:
+      $.[0].counter_name: apples
+      $.[0].counter_volume: 1
+      $.[0].resource_metadata.name2: value2
+
+- name: get resources
+  desc: get the resources that exist because of the sample
+  url: /v2/resources
+  response_json_paths:
+      $.[0].metadata.name2: value2
+
+# NOTE(chdent): We assume that the first item in links is self.
+# Need to determine how to express the more correct JSONPath here
+# (if possible).
+- name: get resource
+  desc: get just one of those resources via self
+  url: $RESPONSE['$[0].links[0].href']
+  response_json_paths:
+      $.metadata.name2: value2
+
+- name: get samples
+  desc: get all the created samples
+  url: /v2/samples
+  response_json_paths:
+      $.[0].metadata.name2: value2
+      $.[0].meter: apples
+
+- name: get one sample
+  desc: get the one sample that exists
+  url: /v2/samples/$RESPONSE['$[0].id']
+  response_json_paths:
+      $.metadata.name2: value2
+      $.meter: apples

ceilometer/tests/gabbi/gabbits/fixture-samples.yaml

@@ -0,0 +1,18 @@
+#
+# Demonstrate a simple sample fixture.
+#
+fixtures:
+    - ConfigFixture
+    - SampleDataFixture
+
+tests:
+- name: get fixture samples
+  desc: get all the samples at livestock
+  url: /v2/meters/livestock
+  response_json_paths:
+      $.[0].counter_name: livestock
+      $.[1].counter_name: livestock
+      $.[2].counter_name: livestock
+      $.[2].user_id: farmerjon
+      $.[0].resource_metadata.breed: cow
+      $.[1].resource_metadata.farmed_by: nancy

ceilometer/tests/gabbi/test_gabbi.py

@@ -0,0 +1,37 @@
+#
+# Copyright 2015 Red Hat. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""A test module to exercise the Ceilometer API with gabbi
+
+For the sake of exploratory development.
+"""
+
+import os
+
+from gabbi import driver
+
+from ceilometer.api import app
+from ceilometer.tests.gabbi import fixtures as fixture_module
+
+
+TESTS_DIR = 'gabbits'
+
+
+def load_tests(loader, tests, pattern):
+    """Provide a TestSuite to the discovery process."""
+    test_dir = os.path.join(os.path.dirname(__file__), TESTS_DIR)
+    return driver.build_tests(test_dir, loader, host=None,
+                              intercept=app.VersionSelectorApplication,
+                              fixture_module=fixture_module)

doc/source/contributing/source.rst

@@ -88,6 +88,19 @@ run through tox_.
    For reference, the ``debug`` tox environment implements the instructions
    here: https://wiki.openstack.org/wiki/Testr#Debugging_.28pdb.29_Tests
 
+5. There is a growing suite of tests which use a tool called `gabbi`_ to
+   test and validate the behavior of the Ceilometer API. These tests are run
+   when using the usual ``py27`` tox target but if desired they can be run by
+   themselves::
+
+   $ tox -e gabbi
+
+   The YAML files used to drive the gabbi tests can be found in
+   ``ceilometer/tests/gabbi/gabbits``. If you are adding to or adjusting the
+   API you should consider adding tests here.
+
+.. _gabbi: https://gabbi.readthedocs.org/
+
 .. seealso::
 
    * tox_

etc/ceilometer/gabbi_pipeline.yaml

@@ -0,0 +1,19 @@
+# A limited pipeline for use with the Gabbi spike.
+# direct writes to the the metering database without using an
+# intermediary dispatcher.
+#
+# This is one of several things that will need some extensive
+# tidying to be more right.
+---
+sources:
+    - name: meter_source
+      interval: 1
+      meters:
+          - "*"
+      sinks:
+          - meter_sink
+sinks:
+    - name: meter_sink
+      transformers:
+      publishers:
+          - direct://

test-requirements.txt

@@ -25,3 +25,4 @@ sphinxcontrib-pecanwsme>=0.8
 testrepository>=0.0.18
 testscenarios>=0.4
 testtools>=0.9.36,!=1.2.0
+gabbi>=0.5.0

tox.ini

@@ -36,6 +36,17 @@ deps = -r{toxinidir}/requirements-py3.txt
 commands = python -m testtools.run \
          ceilometer.tests.test_utils
 
+# NOTE(chdent): The gabbi tests are also run under the primary tox
+# targets. This target simply provides a target to directly run just
+# gabbi tests without needing to discovery across the entire body of
+# tests.
+[testenv:gabbi]
+setenv = OS_TEST_PATH=ceilometer/tests/gabbi
+commands =
+    bash -x {toxinidir}/setup-test-env-mongodb.sh \
+      python setup.py testr --testr-args="{posargs}"
+
+
 [testenv:cover]
 commands = bash -x {toxinidir}/setup-test-env-mongodb.sh python setup.py testr --slowest --coverage --testr-args="{posargs}"