Merge "Update tox.ini with Ansible linting and Browbeat config linting"

ansible/browbeat/adjustment-workers.yml

@@ -34,5 +34,5 @@
   roles:
     - { role: keystone-workers, when: service == "keystone" }
     - { role: nova-workers, when: service == "nova" }
-    - { role: neutron-workers, when: service == "neutron" } 
+    - { role: neutron-workers, when: service == "neutron" }
     - { role: cinder-workers, when: service == "cinder" }

ansible/check/site.yml

@@ -57,10 +57,15 @@
     ansible_python_interpreter: "/usr/bin/python"
   name: Generating bug report
   tasks:
-    - file: "path={{result_dir}} state=directory"
-    - template: src=templates/bug_report.j2 dest={{result_dir}}/bug_report.log
+    - name: Create results dir
+      file: "path={{result_dir}} state=directory"
+    - name: Template Bug report
+      template: src=templates/bug_report.j2 dest={{result_dir}}/bug_report.log
       become: false
-    - template: src=templates/mysql_report.j2 dest={{result_dir}}/mysql_report.log
+    - name: Template MySQL report
+      template: src=templates/mysql_report.j2 dest={{result_dir}}/mysql_report.log
       become: false
-    - replace: dest={{result_dir}}/mysql_report.log regexp='\[([^\s+]+)' replace=''
-    - replace: dest={{result_dir}}/mysql_report.log regexp='\r' replace=''
+    - name: Run MySQL report regex
+      replace: dest={{result_dir}}/mysql_report.log regexp='\[([^\s+]+)' replace=''
+    - name: Run MySQL report regex
+      replace: dest={{result_dir}}/mysql_report.log regexp='\r' replace=''

ansible/install/roles/grafana_docker/handlers/main.yml

@@ -0,0 +1,3 @@
+- name: restart systemd-grafana
+  shell: /usr/bin/systemctl daemon-reload && /usr/bin/systemctl enable grafana-server && /usr/bin/systemctl restart grafana-server
+  become: true

ansible/install/roles/grafana_docker/tasks/main.yml

@@ -65,10 +65,6 @@
     group=root
     mode=0644
   become: true
-  register: systemd_grafana_needs_restart
+  notify: restart systemd-grafana
 
-- name: bounce systemd and grafana-server container
-  shell: /usr/bin/systemctl daemon-reload && /usr/bin/systemctl enable grafana-server && /usr/bin/systemctl restart grafana-server
-  become: true
-  when: systemd_grafana_needs_restart.changed
 

ansible/install/roles/graphite/handlers/main.yml

@@ -0,0 +1,6 @@
+- name: restart apache
+  service: name=httpd state=restarted enabled=true
+  become: true
+- name: restart carbon-cache
+  service: name=carbon-cache state=restarted enabled=true
+  become: true

ansible/install/roles/graphite/tasks/main.yml

@@ -29,7 +29,8 @@
   shell: /root/setup-graphite-db.exp {{ graphite_username }} {{ graphite_password }} && chown apache:apache /var/lib/graphite-web/graphite.db
   become: true
   when: graphite_db_installed.rc != 0
-  register: apache_needs_restart
+  notify:
+     - restart apache
 
 - name: Setup httpd graphite-web config
   template:
@@ -39,7 +40,8 @@
     group=root
     mode=0644
   become: true
-  register: apache_needs_restart
+  notify:
+     - restart apache
 
 ### begin firewall ###
 # we need TCP/80 open
@@ -110,12 +112,8 @@
 - name: Remove httpd welcome config
   become: true
   file: path=/etc/httpd/conf.d/welcome.conf state=absent
-  register: apache_needs_restart
-
-- name: Bounce Apache
-  service: name=httpd state=restarted enabled=true
-  become: true
-  when: apache_needs_restart.changed
+  notify:
+     - restart apache
 
 #
 # setup the python-carbon service
@@ -133,7 +131,8 @@
     group=root
     mode=0644
   become: true
-  register: carbon_cache_needs_restart
+  notify:
+     - restart carbon-cache
 
 - name: copy carbon storage aggregation config
   copy:
@@ -143,7 +142,8 @@
     group=root
     mode=0644
   become: true
-  register: carbon_cache_needs_restart
+  notify:
+     - restart carbon-cache
 
 - name: copy carbon config
   copy:
@@ -153,10 +153,7 @@
     group=root
     mode=0644
   become: true
-  register: carbon_cache_needs_restart
+  notify:
+     - restart carbon-cache
 
-- name: bounce carbon cache
-  service: name=carbon-cache state=restarted enabled=true
-  become: true
-  when: carbon_cache_needs_restart.changed
 

ansible/install/roles/graphite_docker/handlers/main.yml

@@ -0,0 +1,3 @@
+- name: restart systemd-graphite
+  shell: /usr/bin/systemctl daemon-reload && /usr/bin/systemctl enable graphite-web && /usr/bin/systemctl restart graphite-web
+  become: true

ansible/install/roles/graphite_docker/tasks/main.yml

@@ -87,12 +87,8 @@
     group=root
     mode=0644
   become: true
-  register: systemd_graphite_needs_restart
+  notify: restart systemd-graphite
 
-- name: bounce systemd and graphite-web container
-  shell: /usr/bin/systemctl daemon-reload && /usr/bin/systemctl enable graphite-web && /usr/bin/systemctl restart graphite-web
-  become: true
-  when: systemd_graphite_needs_restart.changed
 
 - name: Setup carbon-cache systemd config
   template:
@@ -102,10 +98,6 @@
     group=root
     mode=0644
   become: true
-  register: systemd_carbon_needs_restart
+  notify: restart systemd-graphite
 
-- name: bounce systemd and carbon-cache container
-  shell: /usr/bin/systemctl daemon-reload && /usr/bin/systemctl enable carbon-cache && /usr/bin/systemctl restart carbon-cache
-  become: true
-  when: systemd_carbon_needs_restart.changed
 

ansible/install/roles/kibana-visualization/vars/shaker_cloud_comparison_visualizations.yml

@@ -9,7 +9,7 @@ shaker_cloud_comparison_visualizations:
     markdown: "# Shaker-L2-Flent\\\\n ## Throughput in Mbps\\\\nTo filter based on concurrency please query for key record.concurrency and to filter based based on number of compute nodes, please query for key acommodation.compute_nodes"
   - title: "Browbeat-Shaker-Cloud-Comprison-L2-Throughput"
     template: "shaker-cloud-comparison.json.j2"
-    query: "shaker_test_info.deployment.template: l2.hot AND (result.result_type: tcp_download OR result.result_type: tcp_upload)" 
+    query: "shaker_test_info.deployment.template: l2.hot AND (result.result_type: tcp_download OR result.result_type: tcp_upload)"
   - title: "Shaker-Cloud-Comparison-L2-Latency"
     template: "markdown.json.j2"
     markdown: "# Shaker-L2-Flent\\\\n ## Latency in ms\\\\nTo filter based on concurrency please query for key record.concurrency and to filter based based on number of compute nodes, please query for accommodation.compute_nodes"
@@ -21,7 +21,7 @@ shaker_cloud_comparison_visualizations:
     markdown: "# Shaker-L3-East-West-Flent\\\\n## Throughput in Mbps\\\\nTo filter based on concurrency please query for key record.concurrency and to filter based based on number of compute nodes, please query for key accommodation.compute_nodes"
   - title: "Browbeat-Shaker-Cloud-Comparison-L3-East-West-Throughput"
     template: "shaker-cloud-comparison.json.j2"
-    query: "shaker_test_info.deployment.template: l3_east_west.hot AND (result.result_type: tcp_download OR result.result_type: tcp_upload)" 
+    query: "shaker_test_info.deployment.template: l3_east_west.hot AND (result.result_type: tcp_download OR result.result_type: tcp_upload)"
   - title: "Shaker-Cloud-Comparison-L3-East-West-Latency"
     template: "markdown.json.j2"
     markdown: "# Shaker-L3-East-West-Flent\\\\n## Latency in ms\\\\nTo filter based on concurrency please query for key record.concurrency and to filter based based on number of compute nodes, please query for key accommodation.compute_nodes"

browbeat-complete.yaml

@@ -5,6 +5,7 @@ browbeat:
   cloud_name: openstack
 elasticsearch:
   enabled: false
+  regather: false
   host: 1.1.1.1
   port: 9200
   metadata_files:

ci-scripts/linters/lint-browbeat-config.py

@@ -0,0 +1,27 @@
+#!/usr/bin/env python
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+import yaml
+import sys
+from pykwalify import core as pykwalify_core
+from pykwalify import errors as pykwalify_errors
+stream = open(sys.argv[1], 'r')
+schema = yaml.load(stream)
+check = pykwalify_core.Core(sys.argv[2], schema_data=schema)
+try:
+    check.validate(raise_exception=True)
+    print ("Validation successful")
+    exit(0)
+except pykwalify_errors.SchemaError as e:
+    print ("Config " + sys.argv[2] + " is not valid!")
+    raise Exception('File does not conform to schema: {}'.format(e))

conf/browbeat-glance-complete.yaml

@@ -5,6 +5,7 @@ browbeat:
   cloud_name: openstack
 elasticsearch:
   enabled: false
+  regather: false
   host: 1.1.1.1
   port: 9200
   metadata_files:

conf/browbeat-keystone-complete.yaml

@@ -6,6 +6,7 @@ browbeat:
   cloud_name: openstack
 elasticsearch:
   enabled: false
+  regather: false
   host: 1.1.1.1
   port: 9200
   regather: true

conf/browbeat-keystone-minimal.yaml

@@ -6,6 +6,7 @@ browbeat:
   cloud_name: openstack
 elasticsearch:
   enabled: false
+  regather: false
   host: 1.1.1.1
   port: 9200
   regather: true

conf/ceilometer-minimal.yaml

@@ -5,6 +5,7 @@ browbeat:
   cloud_name: openstack
 elasticsearch:
   enabled: false
+  regather: false
   host: 1.1.1.1
   port: 9200
   metadata_files:

conf/ceilometer-persist-response-timing.yaml

@@ -8,6 +8,7 @@ browbeat:
   cloud_name: openstack
 elasticsearch:
   enabled: false
+  regather: false
   host: 1.1.1.1
   port: 9200
   metadata_files:

conf/perfkit-benchmarks.yaml

@@ -10,6 +10,7 @@ browbeat:
   cloud_name: openstack
 elasticsearch:
   enabled: false
+  regather: false
   host: 1.1.1.1
   port: 9200
   metadata_files:

conf/telemetry-nova-persist-instances.yaml

@@ -9,6 +9,7 @@ browbeat:
   cloud_name: openstack
 elasticsearch:
   enabled: false
+  regather: false
   host: 1.1.1.1
   port: 9200
   regather: true

tox.ini

@@ -1,7 +1,6 @@
 [tox]
 minversion = 2.0
-#envlist = py34-constraints,py27-constraints,pypy-constraints,pep8-constraints
-envlist = py27-constraints,pypy-constraints,pep8-constraints
+envlist = py27-constraints,pep8-constraints,linters
 skipsdist = True
 
 [testenv]
@@ -13,17 +12,23 @@ setenv =
    VIRTUAL_ENV={envdir}
 deps = -r{toxinidir}/test-requirements.txt
 commands = python setup.py test
-#commands = python setup.py test --slowest --testr-args='{posargs}'
 
 [testenv:common-constraints]
 install_command = pip install -c{env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt} {opts} {packages}
 
+
 [testenv:linters]
 whitelist_externals = bash
 commands =
+  pip install pykwalify
   flake8
-#  bash -c "cd ansible; find . -type f -regex '.*.y[a]?ml' -print0 | xargs -t -n1 -0 \
-#    ansible-lint"
+  bash -c "cd ansible; find . -type f -regex '.*.y[a]?ml' -print0 | xargs -t -n1 -0 \
+    ansible-lint \
+    -x ANSIBLE0013,ANSIBLE0012,ANSIBLE0006,ANSIBLE0007,ANSIBLE0010,ANSIBLE0016"
+  python ci-scripts/linters/lint-browbeat-config.py lib/validate.yaml browbeat-config.yaml
+  python ci-scripts/linters/lint-browbeat-config.py lib/validate.yaml browbeat-complete.yaml
+  bash -c "for config in $(ls conf/); do \
+    python ci-scripts/linters/lint-browbeat-config.py lib/validate.yaml conf/$config; done"
 
 [testenv:pep8]
 commands = flake8 {posargs} --exclude=ansible