diff --git a/rally_openstack/scenarios/elasticsearch/__init__.py b/rally_openstack/scenarios/elasticsearch/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/rally_openstack/scenarios/elasticsearch/logging.py b/rally_openstack/scenarios/elasticsearch/logging.py new file mode 100644 index 00000000..175ae407 --- /dev/null +++ b/rally_openstack/scenarios/elasticsearch/logging.py @@ -0,0 +1,106 @@ +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import json +import requests + +from rally.common import cfg +from rally.common import logging +from rally.common import utils as commonutils +from rally.task import types +from rally.task import utils +from rally.task import validation + +from rally_openstack import consts +from rally_openstack import scenario + +CONF = cfg.CONF +LOG = logging.getLogger(__name__) + +"""Scenario for Elasticsearch logging system.""" + + +@types.convert(image={"type": "glance_image"}, + flavor={"type": "nova_flavor"}) +@validation.add("required_services", services=[consts.Service.NOVA]) +@validation.add("required_platform", platform="openstack", admin=True) +@scenario.configure(context={"cleanup@openstack": ["nova"]}, + name="ElasticsearchLogging.log_instance", + platform="openstack") +class ElasticsearchLogInstanceName(scenario.OpenStackScenario): + """Test logging instance in conjunction with Elasticsearch system. + + Let OpenStack platform already has logging agent (for example, Filebeat), + which sends nova logs to Elasticsearch through data processing pipeline + (e.g. Logstash). The test verifies Openstack nova logs stored in logging + system. It creates nova instance with random name and after instance + becomes available, checks it's name in Elasticsearch indices by querying. + """ + + def _create_server(self, image, flavor, seed): + server = self.clients("nova").servers.create(seed, image, flavor) + LOG.info("Server %s create started" % seed) + self.sleep_between(CONF.openstack.nova_server_boot_prepoll_delay) + utils.wait_for_status( + server, + ready_statuses=["ACTIVE"], + update_resource=utils.get_from_manager(), + timeout=CONF.openstack.nova_server_boot_timeout, + check_interval=CONF.openstack.nova_server_boot_poll_interval + ) + LOG.info("Server %s is active" % seed) + + def _check_server_name(self, name, logging_vip, elasticsearch_port, + sleep_time, retries_total): + request_data = {"query": { + "bool": {"must": [{"match_phrase": {"Payload": name}}], + "should": [{"range": {"Timestamp": {"gte": "now-2m", + "lte": "now"}}}], + "minimum_should_match": 1}}} + LOG.info("Check server name %s in elasticsearch" % name) + i = 0 + while i < retries_total: + LOG.debug("Attempt number %s" % (i + 1)) + resp = requests.get("http://%(ip)s:%(port)s/_search" % { + "ip": logging_vip, "port": elasticsearch_port}, + data=json.dumps(request_data)) + result = resp.json() + if result["hits"]["total"] < 1 and i + 1 >= retries_total: + LOG.debug("No instance data found in Elasticsearch") + self.assertGreater(result["hits"]["total"], 0) + elif result["hits"]["total"] < 1: + i += 1 + commonutils.interruptable_sleep(sleep_time) + else: + LOG.debug("Instance data found in Elasticsearch") + self.assertGreater(result["hits"]["total"], 0) + break + + def run(self, image, flavor, logging_vip, elasticsearch_port, + sleep_time=5, retries_total=30): + """Create nova instance and check it indexed in elasticsearch. + + :param image: image for server + :param flavor: flavor for server + :param logging_vip: logging system IP to check server name in + elasticsearch index + :param elasticsearch_port: elasticsearch port to use for check server + :param sleep_time: sleep time in seconds between elasticsearch request + :param retries_total: total number of retries to check server name in + elasticsearch + """ + seed = self.generate_random_name() + self._create_server(image, flavor, seed) + self._check_server_name(seed, logging_vip, elasticsearch_port, + sleep_time, retries_total) diff --git a/requirements.txt b/requirements.txt index 0a4149b6..27fb99c2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -31,4 +31,5 @@ python-swiftclient>=3.2.0 # Apache Software License python-troveclient>=2.2.0 # Apache Software License python-watcherclient>=1.1.0 # Apache Software License python-zaqarclient>=1.0.0 # Apache Software License +requests>=2.14.2 # Apache License, Version 2.0 kubernetes>1.0.0 # Apache License Version 2.0 diff --git a/samples/tasks/scenarios/elasticsearch/log-instance.json b/samples/tasks/scenarios/elasticsearch/log-instance.json new file mode 100644 index 00000000..aa49e09c --- /dev/null +++ b/samples/tasks/scenarios/elasticsearch/log-instance.json @@ -0,0 +1,51 @@ +{% set flavor_name = flavor_name or "esc_test.small" %} +{% set image_name = image_name or "testVM" %} +{ + "ElasticsearchLogging.log_instance": [ + { + "args": { + "flavor": { + "name": "{{ flavor_name }}" + }, + "image": { + "name": "{{ image_name }}" + }, + "logging_vip": "10.0.0.5", + "elasticsearch_port": 5601, + "sleep_time": 5, + "retries_total": 30 + }, + "runner": { + "type": "constant", + "times": 10, + "concurrency": 1 + }, + "context": { + "users": { + "tenants": 1, + "users_per_tenant": 1 + }, + "flavors": [ + { + "name": "{{ flavor_name }}", + "ram": 512, + "disk": 1, + "vcpus": 1 + } + ], + "images": { + "image_name": "{{ image_name }}", + "image_url": "http://download.cirros-cloud.net/0.3.5/cirros-0.3.5-x86_64-disk.img", + "disk_format": "qcow2", + "container_format": "bare", + "visibility": "public" + } + }, + "sla": { + "failure_rate": { + "max": 0 + } + } + } + ] +} \ No newline at end of file diff --git a/samples/tasks/scenarios/elasticsearch/log-instance.yaml b/samples/tasks/scenarios/elasticsearch/log-instance.yaml new file mode 100644 index 00000000..04e483ee --- /dev/null +++ b/samples/tasks/scenarios/elasticsearch/log-instance.yaml @@ -0,0 +1,37 @@ +{% set flavor_name = flavor_name or "esc_test.small" %} +{% set image_name = image_name or "testVM" %} +--- + ElasticsearchLogging.log_instance: + - + args: + flavor: + name: {{ flavor_name }} + image: + name: {{ image_name }} + logging_vip: 10.0.0.5 + elasticsearch_port: 5601 + sleep_time: 5 + retries_total: 30 + runner: + type: "constant" + times: 10 + concurrency: 1 + context: + users: + tenants: 1 + users_per_tenant: 1 + flavors: + - + name: {{ flavor_name }} + ram: 512 + disk: 1 + vcpus: 1 + images: + image_name: {{ image_name }} + image_url: http://download.cirros-cloud.net/0.3.5/cirros-0.3.5-x86_64-disk.img + disk_format: qcow2 + container_format: bare + visibility: public + sla: + failure_rate: + max: 0