Move functional tests to tempest_plugin

Remove pifpaf and dependencies to gnocchi for tox test runs.

Remove redundant gabbi tests.

Move wsme & query unit tests from fuctional to unit test folder where
they belong.

Remove integration tests which has been covered in the new tempest
cases in the depends on patch.

Remove run-tests.sh in favour of just calling stestr.

Depends-On: https://review.opendev.org/c/openstack/telemetry-tempest-plugin/+/901838
Change-Id: I5b33cba27dfcb2bbdf12c9bc6b28ecff9aa18d93

aodh/tests/functional/api/test_versions.py

@@ -1,65 +0,0 @@
-# Copyright 2014 OpenStack Foundation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from aodh.tests.functional import api
-
-V2_MEDIA_TYPES = [
-    {
-        'base': 'application/json',
-        'type': 'application/vnd.openstack.telemetry-v2+json'
-    }, {
-        'base': 'application/xml',
-        'type': 'application/vnd.openstack.telemetry-v2+xml'
-    }
-]
-
-V2_HTML_DESCRIPTION = {
-    'href': 'https://docs.openstack.org/',
-    'rel': 'describedby',
-    'type': 'text/html',
-}
-
-V2_EXPECTED_RESPONSE = {
-    'id': 'v2',
-    'links': [
-        {
-            'rel': 'self',
-            'href': 'http://localhost/v2',
-        },
-        V2_HTML_DESCRIPTION
-    ],
-    'media-types': V2_MEDIA_TYPES,
-    'status': 'stable',
-    'updated': '2013-02-13T00:00:00Z',
-}
-
-V2_VERSION_RESPONSE = {
-    "version": V2_EXPECTED_RESPONSE
-}
-
-VERSIONS_RESPONSE = {
-    "versions": {
-        "values": [
-            V2_EXPECTED_RESPONSE
-        ]
-    }
-}
-
-
-class TestVersions(api.FunctionalTest):
-
-    def test_versions(self):
-        data = self.get_json('/')
-        self.assertEqual(VERSIONS_RESPONSE, data)

aodh/tests/functional/api/v2/test_app.py

@@ -35,47 +35,6 @@ class TestApiMiddleware(v2.FunctionalTest):
         else:
             return self.en_US_translated_error
 
-    def test_json_parsable_error_middleware_404(self):
-        response = self.get_json('/invalid_path',
-                                 expect_errors=True,
-                                 headers={"Accept":
-                                          "application/json"}
-                                 )
-        self.assertEqual(404, response.status_int)
-        self.assertEqual("application/json", response.content_type)
-        self.assertTrue(response.json['error_message'])
-        response = self.get_json('/invalid_path',
-                                 expect_errors=True,
-                                 headers={"Accept":
-                                          "application/json,application/xml"}
-                                 )
-        self.assertEqual(404, response.status_int)
-        self.assertEqual("application/json", response.content_type)
-        self.assertTrue(response.json['error_message'])
-        response = self.get_json('/invalid_path',
-                                 expect_errors=True,
-                                 headers={"Accept":
-                                          "application/xml;q=0.8, \
-                                          application/json"}
-                                 )
-        self.assertEqual(404, response.status_int)
-        self.assertEqual("application/json", response.content_type)
-        self.assertTrue(response.json['error_message'])
-        response = self.get_json('/invalid_path',
-                                 expect_errors=True
-                                 )
-        self.assertEqual(404, response.status_int)
-        self.assertEqual("application/json", response.content_type)
-        self.assertTrue(response.json['error_message'])
-        response = self.get_json('/invalid_path',
-                                 expect_errors=True,
-                                 headers={"Accept":
-                                          "text/html,*/*"}
-                                 )
-        self.assertEqual(404, response.status_int)
-        self.assertEqual("application/json", response.content_type)
-        self.assertTrue(response.json['error_message'])
-
     def test_json_parsable_error_middleware_translation_400(self):
         # Ensure translated messages get placed properly into json faults
         with mock.patch.object(i18n, 'translate',
@@ -89,25 +48,6 @@ class TestApiMiddleware(v2.FunctionalTest):
         self.assertEqual(self.no_lang_translated_error,
                          response.json['error_message']['faultstring'])
 
-    def test_xml_parsable_error_middleware_404(self):
-        response = self.get_json('/invalid_path',
-                                 expect_errors=True,
-                                 headers={"Accept":
-                                          "application/xml,*/*"}
-                                 )
-        self.assertEqual(404, response.status_int)
-        self.assertEqual("application/xml", response.content_type)
-        self.assertEqual('error_message', response.xml.tag)
-        response = self.get_json('/invalid_path',
-                                 expect_errors=True,
-                                 headers={"Accept":
-                                          "application/json;q=0.8 \
-                                          ,application/xml"}
-                                 )
-        self.assertEqual(404, response.status_int)
-        self.assertEqual("application/xml", response.content_type)
-        self.assertEqual('error_message', response.xml.tag)
-
     def test_xml_parsable_error_middleware_translation_400(self):
         # Ensure translated messages get placed properly into xml faults
         with mock.patch.object(i18n, 'translate',

aodh/tests/functional/api/v2/test_capabilities.py

@@ -1,30 +0,0 @@
-#
-# Copyright Ericsson AB 2014. All rights reserved
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-from aodh.tests.functional.api import v2 as tests_api
-
-
-class TestCapabilitiesController(tests_api.FunctionalTest):
-
-    def setUp(self):
-        super(TestCapabilitiesController, self).setUp()
-        self.url = '/capabilities'
-
-    def test_capabilities(self):
-        data = self.get_json(self.url)
-        # check that capabilities data contains both 'api' and 'storage' fields
-        self.assertIsNotNone(data)
-        self.assertNotEqual({}, data)
-        self.assertIn('api', data)
-        self.assertIn('alarm_storage', data)

aodh/tests/functional/gabbi/fixtures.py

@@ -1,131 +0,0 @@
-#
-# Copyright 2015 Red Hat. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-"""Fixtures used during Gabbi-based test runs."""
-
-import os
-import unittest
-from unittest import mock
-
-from gabbi import fixture
-from oslo_config import cfg
-from oslo_config import fixture as fixture_config
-from oslo_policy import opts
-from oslo_utils import uuidutils
-import sqlalchemy_utils
-from urllib import parse as urlparse
-
-from aodh.api import app
-from aodh.api import rbac
-from aodh import service
-from aodh import storage
-
-
-# NOTE(chdent): Hack to restore semblance of global configuration to
-# pass to the WSGI app used per test suite. LOAD_APP_KWARGS are the olso
-# configuration, and the pecan application configuration of
-# which the critical part is a reference to the current indexer.
-LOAD_APP_KWARGS = None
-
-
-def setup_app():
-    global LOAD_APP_KWARGS
-    return app.load_app(**LOAD_APP_KWARGS)
-
-
-class ConfigFixture(fixture.GabbiFixture):
-    """Establish the relevant configuration for a test run."""
-
-    def start_fixture(self):
-        """Set up config."""
-
-        global LOAD_APP_KWARGS
-
-        self.conf = None
-        self.conn = None
-
-        # Determine the database connection.
-        db_url = os.environ.get(
-            'AODH_TEST_STORAGE_URL', "").replace(
-                "mysql://", "mysql+pymysql://")
-        if not db_url:
-            raise unittest.TestCase.failureException(
-                'No database connection configured')
-
-        conf = service.prepare_service([], config_files=[])
-        # NOTE(jd): prepare_service() is called twice: first by load_app() for
-        # Pecan, then Pecan calls pastedeploy, which starts the app, which has
-        # no way to pass the conf object so that Paste apps calls again
-        # prepare_service. In real life, that's not a problem, but here we want
-        # to be sure that the second time the same conf object is returned
-        # since we tweaked it. To that, once we called prepare_service() we
-        # mock it so it returns the same conf object.
-        self.prepare_service = service.prepare_service
-        service.prepare_service = mock.Mock()
-        service.prepare_service.return_value = conf
-        conf = fixture_config.Config(conf).conf
-        self.conf = conf
-        opts.set_defaults(self.conf)
-
-        rbac.enforce = mock.Mock()
-        conf.set_override('auth_mode', None, group='api')
-
-        parsed_url = urlparse.urlparse(db_url)
-        if parsed_url.scheme != 'sqlite':
-            parsed_url = list(parsed_url)
-            parsed_url[2] += '-%s' % uuidutils.generate_uuid(dashed=False)
-            db_url = urlparse.urlunparse(parsed_url)
-
-        conf.set_override('connection', db_url, group='database')
-
-        if (parsed_url[0].startswith("mysql")
-           or parsed_url[0].startswith("postgresql")):
-            sqlalchemy_utils.create_database(conf.database.connection)
-
-        self.conn = storage.get_connection_from_config(self.conf)
-        self.conn.upgrade()
-
-        LOAD_APP_KWARGS = {
-            'conf': conf,
-        }
-
-    def stop_fixture(self):
-        """Reset the config and remove data."""
-        if self.conn:
-            self.conn.clear()
-        if self.conf:
-            self.conf.reset()
-        service.prepare_service = self.prepare_service
-
-
-class CORSConfigFixture(fixture.GabbiFixture):
-    """Inject mock configuration for the CORS middleware."""
-
-    def start_fixture(self):
-        # Here we monkeypatch GroupAttr.__getattr__, necessary because the
-        # paste.ini method of initializing this middleware creates its own
-        # ConfigOpts instance, bypassing the regular config fixture.
-
-        def _mock_getattr(instance, key):
-            if key != 'allowed_origin':
-                return self._original_call_method(instance, key)
-            return "http://valid.example.com"
-
-        self._original_call_method = cfg.ConfigOpts.GroupAttr.__getattr__
-        cfg.ConfigOpts.GroupAttr.__getattr__ = _mock_getattr
-
-    def stop_fixture(self):
-        """Remove the monkeypatch."""
-        cfg.ConfigOpts.GroupAttr.__getattr__ = self._original_call_method

aodh/tests/functional/gabbi/gabbits/alarms.yaml

@@ -1,194 +0,0 @@
-# Requests to cover the basic endpoints for alarms.
-
-fixtures:
-    - ConfigFixture
-
-tests:
-- name: list alarms none
-  desc: Lists alarms, none yet exist
-  GET: /v2/alarms
-  response_strings:
-      - "[]"
-
-- name: try to PUT an alarm
-  desc: what does PUT do
-  PUT: /v2/alarms
-  request_headers:
-      content-type: application/json
-  data:
-      name: added_alarm_defaults2
-      type: gnocchi_resources_threshold
-      gnocchi_resources_threshold_rule:
-          metric: ameter
-          resource_id: random_id
-          resource_type: instance
-          aggregation_method: max
-          threshold: 300.0
-  status: 405
-  response_headers:
-      allow: GET, POST
-
-- name: try to POST an event type alarm
-  desc: what does POST response be
-  POST: /v2/alarms
-  request_headers:
-      content-type: application/json
-  data:
-      name: instance_off
-      type: event
-      event_rule:
-          query: [{'field': "{=:", 'op': "eq", 'type': "string", 'value': "sample_string"}]
-  status: 400
-  response_strings:
-      - "Query value or traits invalid:"
-
-- name: try to POST an event type alarm2
-  desc: what does POST response be
-  POST: /v2/alarms
-  request_headers:
-      content-type: application/json
-  data:
-      name: instance_off
-      type: event
-      event_rule:
-          query: [{'field': "traits.instance_id", 'op': "eq", 'type': "", 'value': "default_string_datatype_isconsidered"}]
-  status: 201
-
-- name: try to POST an event type alarm3
-  desc: what does POST response be
-  POST: /v2/alarms
-  request_headers:
-      content-type: application/json
-  data:
-      name: instance_off
-      type: event
-      event_rule:
-          query: [{'field': "traits.instance_id", 'op': "lt", 'type': "integer", 'value': "1234567"}]
-  status: 201
-
-- name: try to POST an event type alarm4
-  desc: what does POST response be
-  POST: /v2/alarms
-  request_headers:
-      content-type: application/json
-  data:
-      name: instance_off
-      type: event
-      event_rule:
-          query: [{'field': "traits.instance_id", 'op': "lt", 'type': "integer", 'value': "hello"}]
-  status: 400
-  response_strings:
-      - "Unable to convert the value hello to the expected data type integer"
-
-- name: try to POST an event type alarm5
-  desc: what does POST response be
-  POST: /v2/alarms
-  request_headers:
-      content-type: application/json
-  data:
-      name: instance_off
-      type: event
-      event_rule:
-          query: [{'field': "traits.instance_id", 'op': "ltt", 'type': "integer", 'value': "1234567"}]
-  status: 400
-  response_strings:
-      - "Query value or traits invalid:"
-
-- name: createAlarm
-  desc: Creates an alarm.
-  POST: /v2/alarms
-  request_headers:
-      content-type: application/json
-  data:
-      ok_actions: null
-      name: added_alarm_defaults
-      type: gnocchi_resources_threshold
-      gnocchi_resources_threshold_rule:
-          metric: ameter
-          resource_id: random_id
-          resource_type: instance
-          aggregation_method: max
-          threshold: 300.0
-  status: 201
-  response_headers:
-      location: /$SCHEME://$NETLOC/v2/alarms/
-      content-type: application/json
-  response_json_paths:
-      $.severity: low
-      $.gnocchi_resources_threshold_rule.threshold: 300.0
-      $.gnocchi_resources_threshold_rule.comparison_operator: eq
-
-- name: showAlarm
-  desc: Shows information for a specified alarm.
-  GET: /v2/alarms/$RESPONSE['$.alarm_id']
-  response_json_paths:
-      $.severity: low
-      $.alarm_id: $RESPONSE['$.alarm_id']
-      $.gnocchi_resources_threshold_rule.threshold: 300.0
-      $.gnocchi_resources_threshold_rule.comparison_operator: eq
-  response_headers:
-      content-type: application/json
-
-- name: updateAlarm
-  desc: Updates a specified alarm.
-  PUT: /v2/alarms/$RESPONSE['$.alarm_id']
-  request_headers:
-      content-type: application/json
-  data:
-      name: added_alarm_defaults
-      severity: moderate
-      type: gnocchi_resources_threshold
-      gnocchi_resources_threshold_rule:
-          metric: ameter
-          resource_id: random_id
-          resource_type: instance
-          aggregation_method: max
-          threshold: 200.0
-# TODO(chdent): why do we have a response, why not status: 204?
-# status: 204
-  response_json_paths:
-      $.gnocchi_resources_threshold_rule.threshold: 200.0
-      $.severity: moderate
-      $.state: insufficient data
-
-- name: showAlarmHistory
-  desc: Assembles the history for a specified alarm.
-  GET: /v2/alarms/$RESPONSE['$.alarm_id']/history?q.field=type&q.op=eq&q.value=rule%20change
-  response_json_paths:
-      $[0].type: rule change
-
-- name: updateAlarmState
-  desc: Sets the state of a specified alarm.
-  PUT: /v2/alarms/$RESPONSE['$[0].alarm_id']/state
-  request_headers:
-      content-type: application/json
-  data: '"alarm"'
-# TODO(chdent): really? Of what possible use is this?
-  response_json_paths:
-      $: alarm
-
-# Get a list of alarms so we can extract an id for the next test
-- name: list alarms
-  desc: Lists alarms, only one
-  GET: /v2/alarms
-  response_json_paths:
-      $[0].name: added_alarm_defaults
-
-- name: showAlarmState
-  desc: Gets the state of a specified alarm.
-  GET: /v2/alarms/$RESPONSE['$[0].alarm_id']/state
-  response_headers:
-      content-type: application/json
-  response_json_paths:
-      $: alarm
-
-- name: deleteAlarm
-  desc: Deletes a specified alarm.
-  DELETE: /v2/alarms/$HISTORY['list alarms'].$RESPONSE['$[0].alarm_id']
-  status: 204
-
-- name: list alarms none end
-  desc: Lists alarms, none now exist
-  GET: /v2/alarms
-  response_strings:
-      - "[]"

aodh/tests/functional/gabbi/gabbits/basic.yaml

@@ -1,24 +0,0 @@
-#
-# Some simple tests just to confirm that the system works.
-#
-fixtures:
-    - ConfigFixture
-
-tests:
-
-# Root gives us some information on where to go from here.
--    name: quick root check
-     GET: /
-     response_headers:
-         content-type: application/json
-     response_strings:
-         - '"base": "application/json"'
-     response_json_paths:
-         versions.values.[0].status: stable
-         versions.values.[0].media-types.[0].base: application/json
-
-# NOTE(chdent): Ideally since / has a links ref to /v2, /v2 ought not 404!
--    name: v2 visit
-     desc: this demonstrates a bug in the info in /
-     GET: $RESPONSE['versions.values.[0].links.[0].href']
-     status: 404

aodh/tests/functional/gabbi/gabbits/capabilities.yaml

@@ -1,12 +0,0 @@
-#
-# Explore the capabilities API
-#
-fixtures:
-    - ConfigFixture
-
-tests:
-
-- name: get capabilities
-  GET: /v2/capabilities
-  response_json_paths:
-      $.alarm_storage.['storage:production_ready']: true

aodh/tests/functional/gabbi/gabbits/healthcheck.yaml

@@ -1,7 +0,0 @@
-fixtures:
-    - ConfigFixture
-
-tests:
-    - name: healthcheck
-      GET: /healthcheck
-      status: 200

aodh/tests/functional/gabbi/gabbits/middleware.yaml

@@ -1,44 +0,0 @@
-#
-# Test the middlewares. Just CORS for now.
-#
-
-fixtures:
-    - ConfigFixture
-    - CORSConfigFixture
-
-tests:
-
-    - name: valid cors options
-      OPTIONS: /
-      status: 200
-      request_headers:
-          origin: http://valid.example.com
-          access-control-request-method: GET
-      response_headers:
-          access-control-allow-origin: http://valid.example.com
-
-    - name: invalid cors options
-      OPTIONS: /
-      status: 200
-      request_headers:
-          origin: http://invalid.example.com
-          access-control-request-method: GET
-      response_forbidden_headers:
-          - access-control-allow-origin
-
-    - name: valid cors get
-      GET: /
-      status: 200
-      request_headers:
-          origin: http://valid.example.com
-          access-control-request-method: GET
-      response_headers:
-          access-control-allow-origin: http://valid.example.com
-
-    - name: invalid cors get
-      GET: /
-      status: 200
-      request_headers:
-          origin: http://invalid.example.com
-      response_forbidden_headers:
-          - access-control-allow-origin

aodh/tests/functional/gabbi/test_gabbi.py

@@ -1,36 +0,0 @@
-#
-# Copyright 2015 Red Hat. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-"""A test module to exercise the aodh API with gabbi
-
-For the sake of exploratory development.
-"""
-
-import os
-
-from gabbi import driver
-
-from aodh.tests.functional.gabbi import fixtures as fixture_module
-
-
-TESTS_DIR = 'gabbits'
-
-
-def load_tests(loader, tests, pattern):
-    """Provide a TestSuite to the discovery process."""
-    test_dir = os.path.join(os.path.dirname(__file__), TESTS_DIR)
-    return driver.build_tests(test_dir, loader, host=None,
-                              intercept=fixture_module.setup_app,
-                              fixture_module=fixture_module)

run-tests.sh

@@ -1,39 +0,0 @@
-#!/bin/bash -x
-set -e
-
-AODH_TEST_DRIVERS=${AODH_TEST_DRIVERS:-postgresql}
-export GABBI_LIVE_FAIL_IF_NO_TEST=1
-export AODH_SERVICE_TOKEN=foobar # Needed for gabbi
-export AODH_SERVICE_ROLES=admin
-
-# mysqld may be installed to /usr/sbin, which isn't in
-# PATH on some distributions
-export PATH=$PATH:/usr/sbin
-
-# unit tests
-
-export OS_TEST_PATH=aodh/tests/unit
-stestr run $*
-
-# functional tests
-
-export OS_TEST_PATH=aodh/tests/functional
-for indexer in ${AODH_TEST_DRIVERS}
-do
-    pifpaf -g AODH_TEST_STORAGE_URL run $indexer -- stestr run $*
-done
-
-# live functional tests
-
-cleanup(){
-    type -t database_stop >/dev/null && database_stop || true
-}
-trap cleanup EXIT
-
-export OS_TEST_PATH=aodh/tests/functional_live
-for indexer in ${AODH_TEST_DRIVERS}
-do
-    eval $(pifpaf -e DATABASE run $indexer)
-    pifpaf -e AODH run aodh --database-url $DATABASE_URL -- stestr run $*
-    cleanup
-done

tox.ini

@@ -17,8 +17,6 @@ setenv =
     mysql: AODH_TEST_DEPS=mysql
     postgresql: AODH_TEST_DEPS=postgresql
 deps =
-    gnocchi[postgresql, file]
-    pifpaf[gnocchi]>=1.0.1
     -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master}
     .[{env:AODH_TEST_DEPS}]
     -r{toxinidir}/test-requirements.txt
@@ -30,11 +28,11 @@ passenv =
     OS_LOG_CAPTURE
     AODH_TEST_DRIVERS
 commands =
-    {toxinidir}/run-tests.sh {posargs}
+    stestr --test-path=./aodh/tests run
     aodh-config-generator
 allowlist_externals =
     bash
-    {toxinidir}/run-tests.sh
+    stestr
 
 [testenv:cover]
 setenv =
@@ -48,7 +46,7 @@ commands =
     coverage html -d cover
     coverage xml -o cover/coverage.xml
     coverage report
-    pifpaf -g AODH_TEST_STORAGE_URL run mysql -- stestr --test-path=./aodh/tests run
+    stestr --test-path=./aodh/tests run
     coverage report
 
 [testenv:pep8]