From 8d11344b4b79772de3377ca7d78ca6871e0c0f77 Mon Sep 17 00:00:00 2001 From: Sai Sindhur Malleni Date: Tue, 13 Jun 2017 11:26:39 -0400 Subject: [PATCH] Remove hardcoded references to overcloud credentials 1. Make overcloud credentials a configurable parameter in browbeat configuration 2. Make venv format consistent in browbeat configuration Change-Id: I2fa05725c89e1bdb9487af70567efaf8ff19bd34 --- .../template-configs/templates/browbeat-api-ci.yaml.j2 | 3 ++- .../template-configs/templates/browbeat-basic.yaml.j2 | 3 ++- browbeat-complete.yaml | 5 +++-- browbeat-config.yaml | 5 +++-- conf/browbeat-glance-complete.yaml | 3 ++- conf/browbeat-keystone-complete.yaml | 3 ++- conf/browbeat-keystone-minimal.yaml | 3 ++- conf/ceilometer-minimal.yaml | 3 ++- conf/ceilometer-persist-response-timing.yaml | 3 ++- conf/gnocchi-minimal.yaml | 3 ++- conf/perfkit-benchmarks.yaml | 3 ++- conf/telemetry-nova-neutron-gnocchi-10k-1000-per-30m.yaml | 3 ++- conf/telemetry-nova-neutron-gnocchi-1k-100-per-30m.yaml | 3 ++- conf/telemetry-nova-neutron-gnocchi-1k-500-per-30m.yaml | 3 ++- conf/telemetry-nova-neutron-gnocchi-5k-500-per-30m.yaml | 3 ++- conf/telemetry-nova-persist-instances-with-network.yaml | 3 ++- conf/telemetry-nova-persist-instances.yaml | 3 ++- lib/PerfKit.py | 8 +++++--- lib/Rally.py | 8 ++++---- lib/Shaker.py | 3 ++- lib/validate.yaml | 4 ++++ 21 files changed, 51 insertions(+), 27 deletions(-) diff --git a/ansible/oooq/roles/template-configs/templates/browbeat-api-ci.yaml.j2 b/ansible/oooq/roles/template-configs/templates/browbeat-api-ci.yaml.j2 index 1e4fba1ce..ee633db51 100644 --- a/ansible/oooq/roles/template-configs/templates/browbeat-api-ci.yaml.j2 +++ b/ansible/oooq/roles/template-configs/templates/browbeat-api-ci.yaml.j2 @@ -2,6 +2,7 @@ browbeat: results : results/ rerun: 3 cloud_name: {{ browbeat_cloud_name }} + overcloud_credentials: /home/stack/overcloudrc elasticsearch: enabled: {{ elastic_enabled }} host: {{ elastic_host }} @@ -42,7 +43,7 @@ rally: enabled: true sleep_before: 5 sleep_after: 5 - venv: /home/stack/rally-venv/bin/activate + venv: /home/stack/rally-venv plugins: - netcreate-boot: rally/rally-plugins/netcreate-boot - netcreate-boot-ping: rally/rally-plugins/netcreate-boot-ping diff --git a/ansible/oooq/roles/template-configs/templates/browbeat-basic.yaml.j2 b/ansible/oooq/roles/template-configs/templates/browbeat-basic.yaml.j2 index 238c5e159..d8dfb6cb9 100644 --- a/ansible/oooq/roles/template-configs/templates/browbeat-basic.yaml.j2 +++ b/ansible/oooq/roles/template-configs/templates/browbeat-basic.yaml.j2 @@ -4,6 +4,7 @@ browbeat: results : results/ rerun: 1 cloud_name: {{ browbeat_cloud_name }} + overcloud_credentials: /home/stack/overcloudrc elasticsearch: enabled: {{ elastic_enabled }} host: {{ elastic_host }} @@ -44,7 +45,7 @@ rally: enabled: true sleep_before: 5 sleep_after: 5 - venv: /home/stack/rally-venv/bin/activate + venv: /home/stack/rally-venv plugins: - netcreate-boot: rally/rally-plugins/netcreate-boot - netcreate-boot-ping: rally/rally-plugins/netcreate-boot-ping diff --git a/browbeat-complete.yaml b/browbeat-complete.yaml index d89f3b5c7..4b471b44f 100644 --- a/browbeat-complete.yaml +++ b/browbeat-complete.yaml @@ -3,6 +3,7 @@ browbeat: results : results/ rerun: 3 cloud_name: openstack + overcloud_credentials: /home/stack/overcloudrc elasticsearch: enabled: false regather: false @@ -43,7 +44,7 @@ perfkit: enabled: true sleep_before: 0 sleep_after: 0 - venv: /home/stack/perfkit-venv/bin/activate + venv: /home/stack/perfkit-venv default: image: centos7 machine_type: m1.small @@ -62,7 +63,7 @@ rally: enabled: true sleep_before: 5 sleep_after: 5 - venv: /home/stack/rally-venv/bin/activate + venv: /home/stack/rally-venv plugins: - netcreate-boot: rally/rally-plugins/netcreate-boot - netcreate-boot-ping: rally/rally-plugins/netcreate-boot-ping diff --git a/browbeat-config.yaml b/browbeat-config.yaml index 0f975f960..fad56da9f 100644 --- a/browbeat-config.yaml +++ b/browbeat-config.yaml @@ -3,6 +3,7 @@ browbeat: results : results/ rerun: 1 cloud_name: openstack + overcloud_credentials: /home/stack/overcloudrc elasticsearch: enabled: true host: 1.1.1.1 @@ -43,7 +44,7 @@ perfkit: enabled: false sleep_before: 0 sleep_after: 0 - venv: /home/stack/perfkit-venv/bin/activate + venv: /home/stack/perfkit-venv default: image: centos7 machine_type: m1.small @@ -95,7 +96,7 @@ rally: enabled: true sleep_before: 5 sleep_after: 5 - venv: /home/stack/rally-venv/bin/activate + venv: /home/stack/rally-venv plugins: - netcreate-boot: rally/rally-plugins/netcreate-boot - netcreate-boot-ping: rally/rally-plugins/netcreate-boot-ping diff --git a/conf/browbeat-glance-complete.yaml b/conf/browbeat-glance-complete.yaml index cfebfa58a..cfcd2289c 100644 --- a/conf/browbeat-glance-complete.yaml +++ b/conf/browbeat-glance-complete.yaml @@ -3,6 +3,7 @@ browbeat: results : results/ rerun: 1 cloud_name: openstack + overcloud_credentials: /home/stack/overcloudrc elasticsearch: enabled: false regather: false @@ -43,7 +44,7 @@ rally: enabled: true sleep_before: 5 sleep_after: 5 - venv: /home/stack/rally-venv/bin/activate + venv: /home/stack/rally-venv plugins: - glance-create-boot-delete: rally/rally-plugins/glance-create-boot-delete benchmarks: diff --git a/conf/browbeat-keystone-complete.yaml b/conf/browbeat-keystone-complete.yaml index 34b229536..f04de5b17 100644 --- a/conf/browbeat-keystone-complete.yaml +++ b/conf/browbeat-keystone-complete.yaml @@ -4,6 +4,7 @@ browbeat: results : results/ rerun: 3 cloud_name: openstack + overcloud_credentials: /home/stack/overcloudrc elasticsearch: enabled: false regather: false @@ -45,7 +46,7 @@ rally: enabled: true sleep_before: 5 sleep_after: 5 - venv: /home/stack/rally-venv/bin/activate + venv: /home/stack/rally-venv benchmarks: - name: authenticate enabled: true diff --git a/conf/browbeat-keystone-minimal.yaml b/conf/browbeat-keystone-minimal.yaml index 9b0a926ac..1cc582a1d 100644 --- a/conf/browbeat-keystone-minimal.yaml +++ b/conf/browbeat-keystone-minimal.yaml @@ -4,6 +4,7 @@ browbeat: results : results/ rerun: 1 cloud_name: openstack + overcloud_credentials: /home/stack/overcloudrc elasticsearch: enabled: false regather: false @@ -45,7 +46,7 @@ rally: enabled: true sleep_before: 5 sleep_after: 5 - venv: /home/stack/rally-venv/bin/activate + venv: /home/stack/rally-venv benchmarks: - name: authenticate enabled: true diff --git a/conf/ceilometer-minimal.yaml b/conf/ceilometer-minimal.yaml index c2e793c78..2650c1c26 100644 --- a/conf/ceilometer-minimal.yaml +++ b/conf/ceilometer-minimal.yaml @@ -3,6 +3,7 @@ browbeat: results : results/ rerun: 1 cloud_name: openstack + overcloud_credentials: /home/stack/overcloudrc elasticsearch: enabled: false regather: false @@ -43,7 +44,7 @@ rally: enabled: true sleep_before: 5 sleep_after: 5 - venv: /home/stack/rally-venv/bin/activate + venv: /home/stack/rally-venv plugins: - validate-ceilometer: rally/rally-plugins/validate-ceilometer - nova-ceilometer: rally/rally-plugins/nova-ceilometer diff --git a/conf/ceilometer-persist-response-timing.yaml b/conf/ceilometer-persist-response-timing.yaml index b4ab758d0..26297d641 100644 --- a/conf/ceilometer-persist-response-timing.yaml +++ b/conf/ceilometer-persist-response-timing.yaml @@ -6,6 +6,7 @@ browbeat: results : results/ rerun: 1 cloud_name: openstack + overcloud_credentials: /home/stack/overcloudrc elasticsearch: enabled: false regather: false @@ -46,7 +47,7 @@ rally: enabled: true sleep_before: 5 sleep_after: 5 - venv: /home/stack/rally-venv/bin/activate + venv: /home/stack/rally-venv plugins: - nova: rally/rally-plugins/nova benchmarks: diff --git a/conf/gnocchi-minimal.yaml b/conf/gnocchi-minimal.yaml index ccd4d54de..b7a37d871 100644 --- a/conf/gnocchi-minimal.yaml +++ b/conf/gnocchi-minimal.yaml @@ -4,6 +4,7 @@ browbeat: results : results/ rerun: 1 cloud_name: openstack + overcloud_credentials: /home/stack/overcloudrc elasticsearch: enabled: false host: 1.1.1.1 @@ -44,7 +45,7 @@ rally: enabled: true sleep_before: 0 sleep_after: 0 - venv: /home/stack/rally-venv/bin/activate + venv: /home/stack/rally-venv plugins: - gnocchi: rally/rally-plugins/gnocchi benchmarks: diff --git a/conf/perfkit-benchmarks.yaml b/conf/perfkit-benchmarks.yaml index b88e69370..e00c2e904 100644 --- a/conf/perfkit-benchmarks.yaml +++ b/conf/perfkit-benchmarks.yaml @@ -8,6 +8,7 @@ browbeat: results : results/ rerun: 1 cloud_name: openstack + overcloud_credentials: /home/stack/overcloudrc elasticsearch: enabled: false regather: false @@ -48,7 +49,7 @@ perfkit: enabled: true sleep_before: 0 sleep_after: 0 - venv: /home/stack/perfkit-venv/bin/activate + venv: /home/stack/perfkit-venv default: image: centos7 machine_type: m1.small diff --git a/conf/telemetry-nova-neutron-gnocchi-10k-1000-per-30m.yaml b/conf/telemetry-nova-neutron-gnocchi-10k-1000-per-30m.yaml index b7b5c7e4f..4249bb524 100644 --- a/conf/telemetry-nova-neutron-gnocchi-10k-1000-per-30m.yaml +++ b/conf/telemetry-nova-neutron-gnocchi-10k-1000-per-30m.yaml @@ -10,6 +10,7 @@ browbeat: results : results/ rerun: 1 cloud_name: openstack + overcloud_credentials: /home/stack/overcloudrc elasticsearch: enabled: false host: 1.1.1.1 @@ -50,7 +51,7 @@ rally: enabled: true sleep_before: 5 sleep_after: 5 - venv: /home/stack/rally-venv/bin/activate + venv: /home/stack/rally-venv plugins: - nova: rally/rally-plugins/nova - neutron: rally/rally-plugins/neutron diff --git a/conf/telemetry-nova-neutron-gnocchi-1k-100-per-30m.yaml b/conf/telemetry-nova-neutron-gnocchi-1k-100-per-30m.yaml index 8f92ba7d5..92638dfca 100644 --- a/conf/telemetry-nova-neutron-gnocchi-1k-100-per-30m.yaml +++ b/conf/telemetry-nova-neutron-gnocchi-1k-100-per-30m.yaml @@ -10,6 +10,7 @@ browbeat: results : results/ rerun: 1 cloud_name: openstack + overcloud_credentials: /home/stack/overcloudrc elasticsearch: enabled: false host: 1.1.1.1 @@ -50,7 +51,7 @@ rally: enabled: true sleep_before: 5 sleep_after: 5 - venv: /home/stack/rally-venv/bin/activate + venv: /home/stack/rally-venv plugins: - nova: rally/rally-plugins/nova - neutron: rally/rally-plugins/neutron diff --git a/conf/telemetry-nova-neutron-gnocchi-1k-500-per-30m.yaml b/conf/telemetry-nova-neutron-gnocchi-1k-500-per-30m.yaml index 134b6a1c8..775a48eea 100644 --- a/conf/telemetry-nova-neutron-gnocchi-1k-500-per-30m.yaml +++ b/conf/telemetry-nova-neutron-gnocchi-1k-500-per-30m.yaml @@ -10,6 +10,7 @@ browbeat: results : results/ rerun: 1 cloud_name: openstack + overcloud_credentials: /home/stack/overcloudrc elasticsearch: enabled: false host: 1.1.1.1 @@ -50,7 +51,7 @@ rally: enabled: true sleep_before: 5 sleep_after: 5 - venv: /home/stack/rally-venv/bin/activate + venv: /home/stack/rally-venv plugins: - nova: rally/rally-plugins/nova - neutron: rally/rally-plugins/neutron diff --git a/conf/telemetry-nova-neutron-gnocchi-5k-500-per-30m.yaml b/conf/telemetry-nova-neutron-gnocchi-5k-500-per-30m.yaml index a167db397..78a93d7b7 100644 --- a/conf/telemetry-nova-neutron-gnocchi-5k-500-per-30m.yaml +++ b/conf/telemetry-nova-neutron-gnocchi-5k-500-per-30m.yaml @@ -10,6 +10,7 @@ browbeat: results : results/ rerun: 1 cloud_name: openstack + overcloud_credentials: /home/stack/overcloudrc elasticsearch: enabled: false host: 1.1.1.1 @@ -50,7 +51,7 @@ rally: enabled: true sleep_before: 5 sleep_after: 5 - venv: /home/stack/rally-venv/bin/activate + venv: /home/stack/rally-venv plugins: - nova: rally/rally-plugins/nova - neutron: rally/rally-plugins/neutron diff --git a/conf/telemetry-nova-persist-instances-with-network.yaml b/conf/telemetry-nova-persist-instances-with-network.yaml index 805d33fee..f3aed12f9 100644 --- a/conf/telemetry-nova-persist-instances-with-network.yaml +++ b/conf/telemetry-nova-persist-instances-with-network.yaml @@ -8,6 +8,7 @@ browbeat: results : results/ rerun: 1 cloud_name: openstack + overcloud_credentials: /home/stack/overcloudrc elasticsearch: enabled: false host: 1.1.1.1 @@ -50,7 +51,7 @@ rally: # Sleeps 30 minutes after each iteration to allow for system resources to degrade or # display stepped degrading (If the system is hitting saturation or high utilization) sleep_after: 1800 - venv: /home/stack/rally-venv/bin/activate + venv: /home/stack/rally-venv plugins: - nova: rally/rally-plugins/nova - neutron: rally/rally-plugins/neutron diff --git a/conf/telemetry-nova-persist-instances.yaml b/conf/telemetry-nova-persist-instances.yaml index 66766bea3..3d8a81725 100644 --- a/conf/telemetry-nova-persist-instances.yaml +++ b/conf/telemetry-nova-persist-instances.yaml @@ -7,6 +7,7 @@ browbeat: results : results/ rerun: 1 cloud_name: openstack + overcloud_credentials: /home/stack/overcloudrc elasticsearch: enabled: false regather: false @@ -50,7 +51,7 @@ rally: # Sleeps 30 minutes after each iteration to allow for system resources to degrade or # display stepped degrading (If the system is hitting saturation or high utilization) sleep_after: 1800 - venv: /home/stack/rally-venv/bin/activate + venv: /home/stack/rally-venv plugins: - nova: rally/rally-plugins/nova benchmarks: diff --git a/lib/PerfKit.py b/lib/PerfKit.py index d80135fca..1fbf4f450 100644 --- a/lib/PerfKit.py +++ b/lib/PerfKit.py @@ -89,7 +89,7 @@ class PerfKit(WorkloadBase.WorkloadBase): as perfkit_results_json: for result_count, json_result in enumerate(perfkit_results_json): complete_result_json = {'browbeat_scenario': benchmark_config} - complete_result_json['results'] = {'unit':{}, 'value': {}} + complete_result_json['results'] = {'unit': {}, 'value': {}} single_result = self.elastic.load_json(json_result.strip()) complete_result_json['browbeat_rerun'] = browbeat_rerun complete_result_json['timestamp'] = str(es_ts).replace(" ", "T") @@ -125,9 +125,11 @@ class PerfKit(WorkloadBase.WorkloadBase): # Build command to run if 'enabled' in benchmark_config: del benchmark_config['enabled'] - cmd = ("source /home/stack/overcloudrc; source {0}; " + cmd = ("source {0}/bin/activate; source {1} " "/home/stack/perfkit-venv/PerfKitBenchmarker/pkb.py " - "--cloud={1} --run_uri=browbeat".format(self.config['perfkit']['venv'], cloud_type)) + "--cloud={2} --run_uri=browbeat".format( + self.config['perfkit']['venv'], + self.config['browbeat']['overcloud_credentials'], cloud_type)) for parameter, value in benchmark_config.iteritems(): if not parameter == 'name': self.logger.debug( diff --git a/lib/Rally.py b/lib/Rally.py index e1dbc3371..ded6ca0b3 100644 --- a/lib/Rally.py +++ b/lib/Rally.py @@ -62,7 +62,7 @@ class Rally(WorkloadBase.WorkloadBase): plugin_string = "" if len(plugins) > 0: plugin_string = "--plugin-paths {}".format(",".join(plugins)) - cmd = "source {}; ".format(self.config['rally']['venv']) + cmd = "source {}/bin/activate; ".format(self.config['rally']['venv']) cmd += "rally {} task start {} --task-args \'{}\' 2>&1 | tee {}.log".format( plugin_string, task_file, task_args, test_name) from_time = time.time() @@ -108,18 +108,18 @@ class Rally(WorkloadBase.WorkloadBase): def gen_scenario_html(self, task_ids, test_name): all_task_ids = ' '.join(task_ids) - cmd = "source {}; ".format(self.config['rally']['venv']) + cmd = "source {}/bin/activate; ".format(self.config['rally']['venv']) cmd += "rally task report --task {} --out {}.html".format( all_task_ids, test_name) return self.tools.run_cmd(cmd)['stdout'] def gen_scenario_json(self, task_id): - cmd = "source {}; ".format(self.config['rally']['venv']) + cmd = "source {}/bin/activate; ".format(self.config['rally']['venv']) cmd += "rally task results {}".format(task_id) return self.tools.run_cmd(cmd)['stdout'] def gen_scenario_json_file(self, task_id, test_name): - cmd = "source {}; ".format(self.config['rally']['venv']) + cmd = "source {}/bin/activate; ".format(self.config['rally']['venv']) cmd += "rally task results {} > {}.json".format(task_id, test_name) return self.tools.run_cmd(cmd)['stdout'] diff --git a/lib/Shaker.py b/lib/Shaker.py index 4da4273a3..0266326bc 100644 --- a/lib/Shaker.py +++ b/lib/Shaker.py @@ -357,7 +357,8 @@ class Shaker(WorkloadBase.WorkloadBase): self.logger.info( "The uuid for this shaker scenario is {}".format(shaker_uuid)) cmd_env = ( - "source {}/bin/activate; source /home/stack/overcloudrc").format(venv) + "source {0}/bin/activate; source {1}").format(venv, self.config['browbeat'] + ['overcloud_credentials']) if 'external' in filename and 'external_host' in self.config['shaker']: external_host = self.config['shaker']['external_host'] cmd_shaker = ( diff --git a/lib/validate.yaml b/lib/validate.yaml index 1dc4d5806..14714142a 100644 --- a/lib/validate.yaml +++ b/lib/validate.yaml @@ -15,6 +15,10 @@ mapping: cloud_name: type: str required: True + overcloud_credentials: + type: str + required: True + elasticsearch: required: True