diff --git a/.testr.conf b/.testr.conf index dbdbc6817..4b4a9d655 100644 --- a/.testr.conf +++ b/.testr.conf @@ -2,6 +2,8 @@ test_command=OS_STDOUT_CAPTURE=${OS_STDOUT_CAPTURE:-1} \ OS_STDERR_CAPTURE=${OS_STDERR_CAPTURE:-1} \ OS_TEST_TIMEOUT=${OS_TEST_TIMEOUT:-600} \ - ${PYTHON:-python} -m subunit.run discover ceilometer/tests -t . $LISTOPT $IDOPTION + ${PYTHON:-python} -m subunit.run discover ${OS_TEST_PATH:-./ceilometer/tests} -t . $LISTOPT $IDOPTION test_id_option=--load-list $IDFILE test_list_option=--list +# NOTE(chdent): Only used/matches on gabbi-related tests. +group_regex=(gabbi\.driver\.test_gabbi_[^_]+)_ diff --git a/ceilometer/tests/gabbi/__init__.py b/ceilometer/tests/gabbi/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/ceilometer/tests/gabbi/fixtures.py b/ceilometer/tests/gabbi/fixtures.py new file mode 100644 index 000000000..a9c9d38a6 --- /dev/null +++ b/ceilometer/tests/gabbi/fixtures.py @@ -0,0 +1,116 @@ +# +# Copyright 2015 Red Hat. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Fixtures used during Gabbi-based test runs.""" + +from __future__ import print_function + +import datetime +import os +import random +from unittest import case +import uuid + +from gabbi import fixture +from oslo.config import fixture as fixture_config + +from ceilometer.publisher import utils +from ceilometer import sample +from ceilometer import service +from ceilometer import storage + + +# TODO(chdent): For now only MongoDB is supported, because of easy +# database name handling and intentional focus on the API, not the +# data store. +ENGINES = ['MONGODB'] + + +class ConfigFixture(fixture.GabbiFixture): + """Establish the relevant configuration for a test run.""" + + def start_fixture(self): + """Set up config.""" + + service.prepare_service([]) + conf = fixture_config.Config().conf + self.conf = conf + conf.import_opt('policy_file', 'ceilometer.openstack.common.policy') + conf.set_override('policy_file', + os.path.abspath('etc/ceilometer/policy.json')) + + # A special pipeline is required to use the direct publisher. + conf.set_override('pipeline_cfg_file', + 'etc/ceilometer/gabbi_pipeline.yaml') + + # Determine the database connection. + db_url = None + for engine in ENGINES: + try: + db_url = os.environ['CEILOMETER_TEST_%s_URL' % engine] + except KeyError: + pass + if db_url is None: + raise case.SkipTest('No database connection configured') + + database_name = '%s-%s' % (db_url, str(uuid.uuid4())) + conf.set_override('connection', database_name, group='database') + conf.set_override('metering_connection', '', group='database') + conf.set_override('event_connection', '', group='database') + conf.set_override('alarm_connection', '', group='database') + + conf.set_override('pecan_debug', True, group='api') + + def stop_fixture(self): + """Clean up the config.""" + self.conf.reset() + + +class SampleDataFixture(fixture.GabbiFixture): + """Instantiate some sample data for use in testing.""" + + def start_fixture(self): + """Create some samples.""" + conf = fixture_config.Config().conf + self.conn = storage.get_connection_from_config(conf) + timestamp = datetime.datetime.utcnow() + project_id = str(uuid.uuid4()) + self.source = str(uuid.uuid4()) + resource_metadata = {'farmed_by': 'nancy'} + + for name in ['cow', 'pig', 'sheep']: + resource_metadata.update({'breed': name}), + c = sample.Sample(name='livestock', + type='gauge', + unit='head', + volume=int(10 * random.random()), + user_id='farmerjon', + project_id=project_id, + resource_id=project_id, + timestamp=timestamp, + resource_metadata=resource_metadata, + source=self.source, + ) + data = utils.meter_message_from_counter( + c, conf.publisher.telemetry_secret) + self.conn.record_metering_data(data) + + def stop_fixture(self): + """Destroy the samples.""" + # NOTE(chdent): print here for sake of info during testing. + # This will go away eventually. + print('resource', + self.conn.db.resource.remove({'source': self.source})) + print('meter', self.conn.db.meter.remove({'source': self.source})) diff --git a/ceilometer/tests/gabbi/gabbits/basic.yaml b/ceilometer/tests/gabbi/gabbits/basic.yaml new file mode 100644 index 000000000..2a67e5101 --- /dev/null +++ b/ceilometer/tests/gabbi/gabbits/basic.yaml @@ -0,0 +1,24 @@ +# +# Some simple tests just to confirm that the system works. +# +fixtures: + - ConfigFixture + +tests: + +# Root gives us some information on where to go from here. +- name: quick root check + url: / + response_headers: + content-type: application/json; charset=UTF-8 + response_strings: + - '"base": "application/json"' + response_json_paths: + versions.values.[0].status: stable + versions.values.[0].media-types.[0].base: application/json + +# NOTE(chdent): Ideally since / has a links ref to /v2, /v2 ought not 404! +- name: v2 visit + desc: this demonstrates a bug in the info in / + url: $RESPONSE['versions.values.[0].links.[0].href'] + status: 404 diff --git a/ceilometer/tests/gabbi/gabbits/capabilities.yaml b/ceilometer/tests/gabbi/gabbits/capabilities.yaml new file mode 100644 index 000000000..320fde47a --- /dev/null +++ b/ceilometer/tests/gabbi/gabbits/capabilities.yaml @@ -0,0 +1,16 @@ +# +# Explore the capabilities API +# +fixtures: + - ConfigFixture + +tests: + +- name: get capabilities + desc: retrieve capabilities for the mongo store + url: /v2/capabilities + response_json_paths: + $.alarm_storage.['storage:production_ready']: true + $.event_storage.['storage:production_ready']: true + $.storage.['storage:production_ready']: true + $.api.['meters:pagination']: false diff --git a/ceilometer/tests/gabbi/gabbits/clean-samples.yaml b/ceilometer/tests/gabbi/gabbits/clean-samples.yaml new file mode 100644 index 000000000..eff56efb0 --- /dev/null +++ b/ceilometer/tests/gabbi/gabbits/clean-samples.yaml @@ -0,0 +1,75 @@ +# Post a simple sample, sir, and the retrieve it in various ways. +fixtures: + - ConfigFixture + +tests: +- name: post sample for meter + desc: post a single sample + url: /v2/meters/apples + method: POST + request_headers: + content-type: application/json + data: | + [ + { + "counter_name": "apples", + "project_id": "35b17138-b364-4e6a-a131-8f3099c5be68", + "user_id": "efd87807-12d2-4b38-9c70-5f5c2ac427ff", + "counter_unit": "instance", + "counter_volume": 1, + "resource_id": "bd9431c1-8d69-4ad3-803a-8d4a6b89fd36", + "resource_metadata": { + "name2": "value2", + "name1": "value1" + }, + "counter_type": "gauge" + } + ] + + response_json_paths: + $.[0].counter_name: apples + status: 200 + response_headers: + content-type: application/json; charset=UTF-8 + +# TODO(chdent): No location header in the response!!!! +# - name: get sample +# desc: get the sample we just posted +# url: $LOCATION + +- name: get samples for meter + desc: get all the samples at that meter + url: /v2/meters/apples + response_json_paths: + $.[0].counter_name: apples + $.[0].counter_volume: 1 + $.[0].resource_metadata.name2: value2 + +- name: get resources + desc: get the resources that exist because of the sample + url: /v2/resources + response_json_paths: + $.[0].metadata.name2: value2 + +# NOTE(chdent): We assume that the first item in links is self. +# Need to determine how to express the more correct JSONPath here +# (if possible). +- name: get resource + desc: get just one of those resources via self + url: $RESPONSE['$[0].links[0].href'] + response_json_paths: + $.metadata.name2: value2 + +- name: get samples + desc: get all the created samples + url: /v2/samples + response_json_paths: + $.[0].metadata.name2: value2 + $.[0].meter: apples + +- name: get one sample + desc: get the one sample that exists + url: /v2/samples/$RESPONSE['$[0].id'] + response_json_paths: + $.metadata.name2: value2 + $.meter: apples diff --git a/ceilometer/tests/gabbi/gabbits/fixture-samples.yaml b/ceilometer/tests/gabbi/gabbits/fixture-samples.yaml new file mode 100644 index 000000000..94369703b --- /dev/null +++ b/ceilometer/tests/gabbi/gabbits/fixture-samples.yaml @@ -0,0 +1,18 @@ +# +# Demonstrate a simple sample fixture. +# +fixtures: + - ConfigFixture + - SampleDataFixture + +tests: +- name: get fixture samples + desc: get all the samples at livestock + url: /v2/meters/livestock + response_json_paths: + $.[0].counter_name: livestock + $.[1].counter_name: livestock + $.[2].counter_name: livestock + $.[2].user_id: farmerjon + $.[0].resource_metadata.breed: cow + $.[1].resource_metadata.farmed_by: nancy diff --git a/ceilometer/tests/gabbi/test_gabbi.py b/ceilometer/tests/gabbi/test_gabbi.py new file mode 100644 index 000000000..e4c256a8f --- /dev/null +++ b/ceilometer/tests/gabbi/test_gabbi.py @@ -0,0 +1,37 @@ +# +# Copyright 2015 Red Hat. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""A test module to exercise the Ceilometer API with gabbi + +For the sake of exploratory development. +""" + +import os + +from gabbi import driver + +from ceilometer.api import app +from ceilometer.tests.gabbi import fixtures as fixture_module + + +TESTS_DIR = 'gabbits' + + +def load_tests(loader, tests, pattern): + """Provide a TestSuite to the discovery process.""" + test_dir = os.path.join(os.path.dirname(__file__), TESTS_DIR) + return driver.build_tests(test_dir, loader, host=None, + intercept=app.VersionSelectorApplication, + fixture_module=fixture_module) diff --git a/doc/source/contributing/source.rst b/doc/source/contributing/source.rst index 823650c27..35e0650d9 100644 --- a/doc/source/contributing/source.rst +++ b/doc/source/contributing/source.rst @@ -88,6 +88,19 @@ run through tox_. For reference, the ``debug`` tox environment implements the instructions here: https://wiki.openstack.org/wiki/Testr#Debugging_.28pdb.29_Tests +5. There is a growing suite of tests which use a tool called `gabbi`_ to + test and validate the behavior of the Ceilometer API. These tests are run + when using the usual ``py27`` tox target but if desired they can be run by + themselves:: + + $ tox -e gabbi + + The YAML files used to drive the gabbi tests can be found in + ``ceilometer/tests/gabbi/gabbits``. If you are adding to or adjusting the + API you should consider adding tests here. + +.. _gabbi: https://gabbi.readthedocs.org/ + .. seealso:: * tox_ diff --git a/etc/ceilometer/gabbi_pipeline.yaml b/etc/ceilometer/gabbi_pipeline.yaml new file mode 100644 index 000000000..e90516f0b --- /dev/null +++ b/etc/ceilometer/gabbi_pipeline.yaml @@ -0,0 +1,19 @@ +# A limited pipeline for use with the Gabbi spike. +# direct writes to the the metering database without using an +# intermediary dispatcher. +# +# This is one of several things that will need some extensive +# tidying to be more right. +--- +sources: + - name: meter_source + interval: 1 + meters: + - "*" + sinks: + - meter_sink +sinks: + - name: meter_sink + transformers: + publishers: + - direct:// diff --git a/test-requirements.txt b/test-requirements.txt index c5b6f3fa6..d3d35303e 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -25,3 +25,4 @@ sphinxcontrib-pecanwsme>=0.8 testrepository>=0.0.18 testscenarios>=0.4 testtools>=0.9.36,!=1.2.0 +gabbi>=0.5.0 diff --git a/tox.ini b/tox.ini index d1958d73f..cf21cea81 100644 --- a/tox.ini +++ b/tox.ini @@ -36,6 +36,17 @@ deps = -r{toxinidir}/requirements-py3.txt commands = python -m testtools.run \ ceilometer.tests.test_utils +# NOTE(chdent): The gabbi tests are also run under the primary tox +# targets. This target simply provides a target to directly run just +# gabbi tests without needing to discovery across the entire body of +# tests. +[testenv:gabbi] +setenv = OS_TEST_PATH=ceilometer/tests/gabbi +commands = + bash -x {toxinidir}/setup-test-env-mongodb.sh \ + python setup.py testr --testr-args="{posargs}" + + [testenv:cover] commands = bash -x {toxinidir}/setup-test-env-mongodb.sh python setup.py testr --slowest --coverage --testr-args="{posargs}"