diff --git a/ansible/browbeat/adjustment-workers.yml b/ansible/browbeat/adjustment-workers.yml index 9a933e4c1..a39c39046 100644 --- a/ansible/browbeat/adjustment-workers.yml +++ b/ansible/browbeat/adjustment-workers.yml @@ -34,5 +34,5 @@ roles: - { role: keystone-workers, when: service == "keystone" } - { role: nova-workers, when: service == "nova" } - - { role: neutron-workers, when: service == "neutron" } + - { role: neutron-workers, when: service == "neutron" } - { role: cinder-workers, when: service == "cinder" } diff --git a/ansible/check/site.yml b/ansible/check/site.yml index 62a6508fa..e7dd14d01 100644 --- a/ansible/check/site.yml +++ b/ansible/check/site.yml @@ -57,10 +57,15 @@ ansible_python_interpreter: "/usr/bin/python" name: Generating bug report tasks: - - file: "path={{result_dir}} state=directory" - - template: src=templates/bug_report.j2 dest={{result_dir}}/bug_report.log + - name: Create results dir + file: "path={{result_dir}} state=directory" + - name: Template Bug report + template: src=templates/bug_report.j2 dest={{result_dir}}/bug_report.log become: false - - template: src=templates/mysql_report.j2 dest={{result_dir}}/mysql_report.log + - name: Template MySQL report + template: src=templates/mysql_report.j2 dest={{result_dir}}/mysql_report.log become: false - - replace: dest={{result_dir}}/mysql_report.log regexp='\[([^\s+]+)' replace='' - - replace: dest={{result_dir}}/mysql_report.log regexp='\r' replace='' + - name: Run MySQL report regex + replace: dest={{result_dir}}/mysql_report.log regexp='\[([^\s+]+)' replace='' + - name: Run MySQL report regex + replace: dest={{result_dir}}/mysql_report.log regexp='\r' replace='' diff --git a/ansible/install/roles/grafana_docker/handlers/main.yml b/ansible/install/roles/grafana_docker/handlers/main.yml new file mode 100644 index 000000000..826781eb9 --- /dev/null +++ b/ansible/install/roles/grafana_docker/handlers/main.yml @@ -0,0 +1,3 @@ +- name: restart systemd-grafana + shell: /usr/bin/systemctl daemon-reload && /usr/bin/systemctl enable grafana-server && /usr/bin/systemctl restart grafana-server + become: true diff --git a/ansible/install/roles/grafana_docker/tasks/main.yml b/ansible/install/roles/grafana_docker/tasks/main.yml index 95b0de90e..22fc11704 100644 --- a/ansible/install/roles/grafana_docker/tasks/main.yml +++ b/ansible/install/roles/grafana_docker/tasks/main.yml @@ -65,10 +65,6 @@ group=root mode=0644 become: true - register: systemd_grafana_needs_restart + notify: restart systemd-grafana -- name: bounce systemd and grafana-server container - shell: /usr/bin/systemctl daemon-reload && /usr/bin/systemctl enable grafana-server && /usr/bin/systemctl restart grafana-server - become: true - when: systemd_grafana_needs_restart.changed diff --git a/ansible/install/roles/graphite/handlers/main.yml b/ansible/install/roles/graphite/handlers/main.yml new file mode 100644 index 000000000..3987061b4 --- /dev/null +++ b/ansible/install/roles/graphite/handlers/main.yml @@ -0,0 +1,6 @@ +- name: restart apache + service: name=httpd state=restarted enabled=true + become: true +- name: restart carbon-cache + service: name=carbon-cache state=restarted enabled=true + become: true diff --git a/ansible/install/roles/graphite/tasks/main.yml b/ansible/install/roles/graphite/tasks/main.yml index 5ba52cb2d..105f85135 100644 --- a/ansible/install/roles/graphite/tasks/main.yml +++ b/ansible/install/roles/graphite/tasks/main.yml @@ -29,7 +29,8 @@ shell: /root/setup-graphite-db.exp {{ graphite_username }} {{ graphite_password }} && chown apache:apache /var/lib/graphite-web/graphite.db become: true when: graphite_db_installed.rc != 0 - register: apache_needs_restart + notify: + - restart apache - name: Setup httpd graphite-web config template: @@ -39,7 +40,8 @@ group=root mode=0644 become: true - register: apache_needs_restart + notify: + - restart apache ### begin firewall ### # we need TCP/80 open @@ -110,12 +112,8 @@ - name: Remove httpd welcome config become: true file: path=/etc/httpd/conf.d/welcome.conf state=absent - register: apache_needs_restart - -- name: Bounce Apache - service: name=httpd state=restarted enabled=true - become: true - when: apache_needs_restart.changed + notify: + - restart apache # # setup the python-carbon service @@ -133,7 +131,8 @@ group=root mode=0644 become: true - register: carbon_cache_needs_restart + notify: + - restart carbon-cache - name: copy carbon storage aggregation config copy: @@ -143,7 +142,8 @@ group=root mode=0644 become: true - register: carbon_cache_needs_restart + notify: + - restart carbon-cache - name: copy carbon config copy: @@ -153,10 +153,7 @@ group=root mode=0644 become: true - register: carbon_cache_needs_restart + notify: + - restart carbon-cache -- name: bounce carbon cache - service: name=carbon-cache state=restarted enabled=true - become: true - when: carbon_cache_needs_restart.changed diff --git a/ansible/install/roles/graphite_docker/handlers/main.yml b/ansible/install/roles/graphite_docker/handlers/main.yml new file mode 100644 index 000000000..50049b3dd --- /dev/null +++ b/ansible/install/roles/graphite_docker/handlers/main.yml @@ -0,0 +1,3 @@ +- name: restart systemd-graphite + shell: /usr/bin/systemctl daemon-reload && /usr/bin/systemctl enable graphite-web && /usr/bin/systemctl restart graphite-web + become: true diff --git a/ansible/install/roles/graphite_docker/tasks/main.yml b/ansible/install/roles/graphite_docker/tasks/main.yml index 5e58bf758..cd7f3294d 100644 --- a/ansible/install/roles/graphite_docker/tasks/main.yml +++ b/ansible/install/roles/graphite_docker/tasks/main.yml @@ -87,12 +87,8 @@ group=root mode=0644 become: true - register: systemd_graphite_needs_restart + notify: restart systemd-graphite -- name: bounce systemd and graphite-web container - shell: /usr/bin/systemctl daemon-reload && /usr/bin/systemctl enable graphite-web && /usr/bin/systemctl restart graphite-web - become: true - when: systemd_graphite_needs_restart.changed - name: Setup carbon-cache systemd config template: @@ -102,10 +98,6 @@ group=root mode=0644 become: true - register: systemd_carbon_needs_restart + notify: restart systemd-graphite -- name: bounce systemd and carbon-cache container - shell: /usr/bin/systemctl daemon-reload && /usr/bin/systemctl enable carbon-cache && /usr/bin/systemctl restart carbon-cache - become: true - when: systemd_carbon_needs_restart.changed diff --git a/ansible/install/roles/kibana-visualization/vars/shaker_cloud_comparison_visualizations.yml b/ansible/install/roles/kibana-visualization/vars/shaker_cloud_comparison_visualizations.yml index 75a43c2ef..24a0badfa 100644 --- a/ansible/install/roles/kibana-visualization/vars/shaker_cloud_comparison_visualizations.yml +++ b/ansible/install/roles/kibana-visualization/vars/shaker_cloud_comparison_visualizations.yml @@ -9,7 +9,7 @@ shaker_cloud_comparison_visualizations: markdown: "# Shaker-L2-Flent\\\\n ## Throughput in Mbps\\\\nTo filter based on concurrency please query for key record.concurrency and to filter based based on number of compute nodes, please query for key acommodation.compute_nodes" - title: "Browbeat-Shaker-Cloud-Comprison-L2-Throughput" template: "shaker-cloud-comparison.json.j2" - query: "shaker_test_info.deployment.template: l2.hot AND (result.result_type: tcp_download OR result.result_type: tcp_upload)" + query: "shaker_test_info.deployment.template: l2.hot AND (result.result_type: tcp_download OR result.result_type: tcp_upload)" - title: "Shaker-Cloud-Comparison-L2-Latency" template: "markdown.json.j2" markdown: "# Shaker-L2-Flent\\\\n ## Latency in ms\\\\nTo filter based on concurrency please query for key record.concurrency and to filter based based on number of compute nodes, please query for accommodation.compute_nodes" @@ -21,7 +21,7 @@ shaker_cloud_comparison_visualizations: markdown: "# Shaker-L3-East-West-Flent\\\\n## Throughput in Mbps\\\\nTo filter based on concurrency please query for key record.concurrency and to filter based based on number of compute nodes, please query for key accommodation.compute_nodes" - title: "Browbeat-Shaker-Cloud-Comparison-L3-East-West-Throughput" template: "shaker-cloud-comparison.json.j2" - query: "shaker_test_info.deployment.template: l3_east_west.hot AND (result.result_type: tcp_download OR result.result_type: tcp_upload)" + query: "shaker_test_info.deployment.template: l3_east_west.hot AND (result.result_type: tcp_download OR result.result_type: tcp_upload)" - title: "Shaker-Cloud-Comparison-L3-East-West-Latency" template: "markdown.json.j2" markdown: "# Shaker-L3-East-West-Flent\\\\n## Latency in ms\\\\nTo filter based on concurrency please query for key record.concurrency and to filter based based on number of compute nodes, please query for key accommodation.compute_nodes" diff --git a/browbeat-complete.yaml b/browbeat-complete.yaml index 9acac75d1..cd03d49af 100644 --- a/browbeat-complete.yaml +++ b/browbeat-complete.yaml @@ -5,6 +5,7 @@ browbeat: cloud_name: openstack elasticsearch: enabled: false + regather: false host: 1.1.1.1 port: 9200 metadata_files: diff --git a/ci-scripts/linters/lint-browbeat-config.py b/ci-scripts/linters/lint-browbeat-config.py new file mode 100644 index 000000000..95138bdbb --- /dev/null +++ b/ci-scripts/linters/lint-browbeat-config.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import yaml +import sys +from pykwalify import core as pykwalify_core +from pykwalify import errors as pykwalify_errors +stream = open(sys.argv[1], 'r') +schema = yaml.load(stream) +check = pykwalify_core.Core(sys.argv[2], schema_data=schema) +try: + check.validate(raise_exception=True) + print ("Validation successful") + exit(0) +except pykwalify_errors.SchemaError as e: + print ("Config " + sys.argv[2] + " is not valid!") + raise Exception('File does not conform to schema: {}'.format(e)) diff --git a/conf/browbeat-glance-complete.yaml b/conf/browbeat-glance-complete.yaml index 3065f5953..cfebfa58a 100644 --- a/conf/browbeat-glance-complete.yaml +++ b/conf/browbeat-glance-complete.yaml @@ -5,6 +5,7 @@ browbeat: cloud_name: openstack elasticsearch: enabled: false + regather: false host: 1.1.1.1 port: 9200 metadata_files: diff --git a/conf/browbeat-keystone-complete.yaml b/conf/browbeat-keystone-complete.yaml index beebe3f1e..34b229536 100644 --- a/conf/browbeat-keystone-complete.yaml +++ b/conf/browbeat-keystone-complete.yaml @@ -6,6 +6,7 @@ browbeat: cloud_name: openstack elasticsearch: enabled: false + regather: false host: 1.1.1.1 port: 9200 regather: true diff --git a/conf/browbeat-keystone-minimal.yaml b/conf/browbeat-keystone-minimal.yaml index ba6335945..9b0a926ac 100644 --- a/conf/browbeat-keystone-minimal.yaml +++ b/conf/browbeat-keystone-minimal.yaml @@ -6,6 +6,7 @@ browbeat: cloud_name: openstack elasticsearch: enabled: false + regather: false host: 1.1.1.1 port: 9200 regather: true diff --git a/conf/ceilometer-minimal.yaml b/conf/ceilometer-minimal.yaml index 3a7bbf2e3..c2e793c78 100644 --- a/conf/ceilometer-minimal.yaml +++ b/conf/ceilometer-minimal.yaml @@ -5,6 +5,7 @@ browbeat: cloud_name: openstack elasticsearch: enabled: false + regather: false host: 1.1.1.1 port: 9200 metadata_files: diff --git a/conf/ceilometer-persist-response-timing.yaml b/conf/ceilometer-persist-response-timing.yaml index df6bcab65..b4ab758d0 100644 --- a/conf/ceilometer-persist-response-timing.yaml +++ b/conf/ceilometer-persist-response-timing.yaml @@ -8,6 +8,7 @@ browbeat: cloud_name: openstack elasticsearch: enabled: false + regather: false host: 1.1.1.1 port: 9200 metadata_files: diff --git a/conf/perfkit-benchmarks.yaml b/conf/perfkit-benchmarks.yaml index 6bf32f138..516bb774c 100644 --- a/conf/perfkit-benchmarks.yaml +++ b/conf/perfkit-benchmarks.yaml @@ -10,6 +10,7 @@ browbeat: cloud_name: openstack elasticsearch: enabled: false + regather: false host: 1.1.1.1 port: 9200 metadata_files: diff --git a/conf/telemetry-nova-persist-instances.yaml b/conf/telemetry-nova-persist-instances.yaml index a03479017..66766bea3 100644 --- a/conf/telemetry-nova-persist-instances.yaml +++ b/conf/telemetry-nova-persist-instances.yaml @@ -9,6 +9,7 @@ browbeat: cloud_name: openstack elasticsearch: enabled: false + regather: false host: 1.1.1.1 port: 9200 regather: true diff --git a/tox.ini b/tox.ini index de60a7289..008545e9c 100644 --- a/tox.ini +++ b/tox.ini @@ -1,7 +1,6 @@ [tox] minversion = 2.0 -#envlist = py34-constraints,py27-constraints,pypy-constraints,pep8-constraints -envlist = py27-constraints,pypy-constraints,pep8-constraints +envlist = py27-constraints,pep8-constraints,linters skipsdist = True [testenv] @@ -13,17 +12,23 @@ setenv = VIRTUAL_ENV={envdir} deps = -r{toxinidir}/test-requirements.txt commands = python setup.py test -#commands = python setup.py test --slowest --testr-args='{posargs}' [testenv:common-constraints] install_command = pip install -c{env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt} {opts} {packages} + [testenv:linters] whitelist_externals = bash commands = + pip install pykwalify flake8 -# bash -c "cd ansible; find . -type f -regex '.*.y[a]?ml' -print0 | xargs -t -n1 -0 \ -# ansible-lint" + bash -c "cd ansible; find . -type f -regex '.*.y[a]?ml' -print0 | xargs -t -n1 -0 \ + ansible-lint \ + -x ANSIBLE0013,ANSIBLE0012,ANSIBLE0006,ANSIBLE0007,ANSIBLE0010,ANSIBLE0016" + python ci-scripts/linters/lint-browbeat-config.py lib/validate.yaml browbeat-config.yaml + python ci-scripts/linters/lint-browbeat-config.py lib/validate.yaml browbeat-complete.yaml + bash -c "for config in $(ls conf/); do \ + python ci-scripts/linters/lint-browbeat-config.py lib/validate.yaml conf/$config; done" [testenv:pep8] commands = flake8 {posargs} --exclude=ansible