Selenium Tests for OSH Infra

This adds scripts using Selenium Webdriver to verify
the dashboards for Gafana, Nagios, and Prometheus are
reachable and functioning as expected. The scripts
create screenshots of each dashboard as well as
pages that can be navigated to.

It also adds the scripts to the gates for the single
and multinode deployments.

Change-Id: I1699e0ba8ff82ce8f59342cc71aad10cff7d2516

playbooks/osh-infra-deploy-selenium.yaml

@@ -0,0 +1,25 @@
+# Copyright 2017 The Openstack-Helm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- hosts: primary
+  vars_files:
+    - vars.yaml
+  vars:
+    work_dir: "{{ zuul.project.src_dir }}/{{ zuul_osh_infra_relative_path | default('') }}"
+  gather_facts: True
+  become: yes
+  roles:
+    - deploy-selenium
+  tags:
+    - deploy-selenium

playbooks/osh-infra-gate-runner.yaml

@@ -14,9 +14,19 @@
 
 - hosts: primary
   tasks:
+    - name: "creating directory for run artifacts"
+      file:
+        path: "/tmp/artifacts"
+        state: directory
     - name: Run gate scripts
       include_role:
         name: osh-run-script
       vars:
         gate_script_path: "{{ item }}"
       with_items: "{{ gate_scripts }}"
+    - name: "Downloads artifacts to executor"
+      synchronize:
+        src: "/tmp/artifacts"
+        dest: "{{ zuul.executor.log_root }}/{{ inventory_hostname }}"
+        mode: pull
+      ignore_errors: True

roles/deploy-selenium/tasks/main.yaml

@@ -0,0 +1,54 @@
+# Copyright 2017 The Openstack-Helm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: "creating selenium configuration directory"
+  file:
+    path: /etc/selenium
+    state: directory
+
+- name: install selenium dependencies
+  when: ansible_distribution == 'Debian' or ansible_distribution == 'Ubuntu'
+  apt:
+    name: "{{ packages }}"
+  vars:
+    packages:
+      - unzip
+      - wget
+      - xvfb
+
+- name: install selenium
+  pip:
+    name: selenium
+    state: latest
+
+- name: Get selenium chrome driver
+  shell: |-
+          set -ex
+          wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add -
+          sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list'
+          wget --directory-prefix=/tmp/ https://chromedriver.storage.googleapis.com/2.44/chromedriver_linux64.zip
+  args:
+    executable: /bin/bash
+
+- name: unarchive selenium chrome driver
+  unarchive:
+    src: /tmp/chromedriver_linux64.zip
+    dest: /etc/selenium
+    remote_src: yes
+
+- name: install google chrome
+  when: ansible_distribution == 'Debian' or ansible_distribution == 'Ubuntu'
+  apt:
+    name: google-chrome-stable
+    update_cache: yes

tools/deployment/common/grafana-selenium.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+export GRAFANA_USER="admin"
+export GRAFANA_PASSWORD="password"
+export GRAFANA_URI="http://grafana.osh-infra.svc.cluster.local"
+python tools/gate/selenium/grafanaSelenium.py

tools/deployment/common/nagios-selenium.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+export NAGIOS_USER="nagiosadmin"
+export NAGIOS_PASSWORD="password"
+export NAGIOS_URI="nagios.osh-infra.svc.cluster.local"
+python tools/gate/selenium/nagiosSelenium.py

tools/deployment/common/prometheus-selenium.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+export PROMETHEUS_USER="admin"
+export PROMETHEUS_PASSWORD="changeme"
+export PROMETHEUS_URI="prometheus.osh-infra.svc.cluster.local"
+python tools/gate/selenium/prometheusSelenium.py

tools/deployment/multinode/grafana-selenium.sh

@@ -0,0 +1 @@
+../common/grafana-selenium.sh

tools/deployment/multinode/nagios-selenium.sh

@@ -0,0 +1 @@
+../common/nagios-selenium.sh

tools/deployment/multinode/prometheus-selenium.sh

@@ -0,0 +1 @@
+../common/prometheus-selenium.sh

tools/deployment/osh-infra-monitoring/grafana-selenium.sh

@@ -0,0 +1 @@
+../common/grafana-selenium.sh

tools/deployment/osh-infra-monitoring/nagios-selenium.sh

@@ -0,0 +1 @@
+../common/nagios-selenium.sh

tools/deployment/osh-infra-monitoring/prometheus-selenium.sh

@@ -0,0 +1 @@
+../common/prometheus-selenium.sh

tools/gate/selenium/grafanaSelenium.py

@@ -0,0 +1,95 @@
+import logging
+import os
+import sys
+from selenium import webdriver
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.chrome.options import Options
+
+# Create logger, console handler and formatter
+logger = logging.getLogger('Grafana Selenium Tests')
+logger.setLevel(logging.DEBUG)
+ch = logging.StreamHandler()
+ch.setLevel(logging.DEBUG)
+formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+
+# Set the formatter and add the handler
+ch.setFormatter(formatter)
+logger.addHandler(ch)
+
+# Get Grafana admin user name
+if "GRAFANA_USER" in os.environ:
+  grafana_user = os.environ['GRAFANA_USER']
+  logger.info('Found Grafana username')
+else:
+  logger.critical('Grafana username environment variable not set')
+  sys.exit(1)
+
+if "GRAFANA_PASSWORD" in os.environ:
+  grafana_password = os.environ['GRAFANA_PASSWORD']
+  logger.info('Found Grafana password')
+else:
+  logger.critical('Grafana password environment variable not set')
+  sys.exit(1)
+
+if "GRAFANA_URI" in os.environ:
+  grafana_uri = os.environ['GRAFANA_URI']
+  logger.info('Found Grafana URI')
+else:
+  logger.critical('Grafana URI environment variable not set')
+  sys.exit(1)
+
+options = Options()
+options.add_argument('--headless')
+options.add_argument('--no-sandbox')
+options.add_argument('--window-size=1920x1080')
+
+browser = webdriver.Chrome('/etc/selenium/chromedriver', chrome_options=options)
+
+browser.get(grafana_uri)
+username = browser.find_element_by_name('username')
+username.send_keys(grafana_user)
+
+password = browser.find_element_by_name('password')
+password.send_keys(grafana_password)
+
+login = browser.find_element_by_css_selector('body > grafana-app > div.main-view > div > div:nth-child(1) > div > div > div.login-inner-box > form > div.login-button-group > button')
+login.click()
+
+el = WebDriverWait(browser, 15).until(
+    EC.presence_of_element_located((By.LINK_TEXT, 'Home'))
+)
+
+homeBtn = browser.find_element_by_link_text('Home')
+homeBtn.click()
+
+
+el = WebDriverWait(browser, 15).until(
+    EC.presence_of_element_located((By.LINK_TEXT, 'Nodes'))
+)
+
+nodeBtn = browser.find_element_by_link_text('Nodes')
+nodeBtn.click()
+
+el = WebDriverWait(browser, 15).until(
+    EC.presence_of_element_located((By.XPATH, '/html/body/grafana-app/div[2]/div/div[1]/div/div/div[1]/dashboard-grid/div/div[1]/div/plugin-component/panel-plugin-graph/grafana-panel/div/div[2]'))
+)
+
+browser.save_screenshot('/tmp/artifacts/Grafana_Nodes.png')
+
+nodeBtn = browser.find_element_by_link_text('Nodes')
+nodeBtn.click()
+
+el = WebDriverWait(browser, 15).until(
+    EC.presence_of_element_located((By.LINK_TEXT, 'Kubernetes Cluster Status'))
+)
+
+healthBtn = browser.find_element_by_link_text('Kubernetes Cluster Status')
+healthBtn.click()
+
+el = WebDriverWait(browser, 15).until(
+    EC.presence_of_element_located((By.XPATH, '/html/body/grafana-app/div[2]/div/div[1]/div/div/div[1]/dashboard-grid/div/div[5]/div/plugin-component/panel-plugin-singlestat/grafana-panel/div'))
+)
+
+browser.save_screenshot('/tmp/artifacts/Grafana_ClusterStatus.png')

tools/gate/selenium/nagiosSelenium.py

@@ -0,0 +1,70 @@
+import os
+import logging
+import sys
+from selenium import webdriver
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.chrome.options import Options
+
+
+# Create logger, console handler and formatter
+logger = logging.getLogger('Nagios Selenium Tests')
+logger.setLevel(logging.DEBUG)
+ch = logging.StreamHandler()
+ch.setLevel(logging.DEBUG)
+formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+
+# Set the formatter and add the handler
+ch.setFormatter(formatter)
+logger.addHandler(ch)
+
+# Get Grafana admin user name
+if "NAGIOS_USER" in os.environ:
+  nagios_user = os.environ['NAGIOS_USER']
+  logger.info('Found Nagios username')
+else:
+  logger.critical('Nagios username environment variable not set')
+  sys.exit(1)
+
+if "NAGIOS_PASSWORD" in os.environ:
+  nagios_password = os.environ['NAGIOS_PASSWORD']
+  logger.info('Found Nagios password')
+else:
+  logger.critical('Nagios password environment variable not set')
+  sys.exit(1)
+
+if "NAGIOS_URI" in os.environ:
+  nagios_uri = os.environ['NAGIOS_URI']
+  logger.info('Found Nagios URI')
+else:
+  logger.critical('Nagios URI environment variable not set')
+  sys.exit(1)
+
+options = Options()
+options.add_argument('--headless')
+options.add_argument('--no-sandbox')
+options.add_argument('--window-size=1920x1080')
+
+browser = webdriver.Chrome('/etc/selenium/chromedriver', chrome_options=options)
+browser.get('http://'+nagios_user+':'+nagios_password+'@'+nagios_uri)
+
+sideFrame = browser.switch_to.frame('side')
+
+services = browser.find_element_by_link_text('Services')
+services.click()
+
+el = WebDriverWait(browser, 15)
+browser.save_screenshot('/tmp/artifacts/Nagios_Services.png')
+
+hostGroups = browser.find_element_by_link_text('Host Groups')
+hostGroups.click()
+
+el = WebDriverWait(browser, 15)
+browser.save_screenshot('/tmp/artifacts/Nagios_HostGroups.png')
+
+hosts = browser.find_element_by_link_text('Hosts')
+hosts.click()
+
+el = WebDriverWait(browser, 15)
+browser.save_screenshot('/tmp/artifacts/Nagios_Hosts.png')

tools/gate/selenium/prometheusSelenium.py

@@ -0,0 +1,79 @@
+import os
+import logging
+import sys
+from selenium import webdriver
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.chrome.options import Options
+
+# Create logger, console handler and formatter
+logger = logging.getLogger('Prometheus Selenium Tests')
+logger.setLevel(logging.DEBUG)
+ch = logging.StreamHandler()
+ch.setLevel(logging.DEBUG)
+formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+
+# Set the formatter and add the handler
+ch.setFormatter(formatter)
+logger.addHandler(ch)
+
+# Get Grafana admin user name
+if "PROMETHEUS_USER" in os.environ:
+  prometheus_user = os.environ['PROMETHEUS_USER']
+  logger.info('Found Prometheus username')
+else:
+  logger.critical('Prometheus username environment variable not set')
+  sys.exit(1)
+
+if "PROMETHEUS_PASSWORD" in os.environ:
+  prometheus_password = os.environ['PROMETHEUS_PASSWORD']
+  logger.info('Found Prometheus password')
+else:
+  logger.critical('Prometheus password environment variable not set')
+  sys.exit(1)
+
+if "PROMETHEUS_URI" in os.environ:
+  prometheus_uri = os.environ['PROMETHEUS_URI']
+  logger.info('Found Prometheus URI')
+else:
+  logger.critical('Prometheus URI environment variable not set')
+  sys.exit(1)
+
+options = Options()
+options.add_argument('--headless')
+options.add_argument('--no-sandbox')
+options.add_argument('--window-size=1920x1080')
+
+browser = webdriver.Chrome('/etc/selenium/chromedriver', chrome_options=options)
+
+browser.get("http://"+prometheus_user+":"+prometheus_password+"@"+prometheus_uri)
+
+el = WebDriverWait(browser, 15).until(
+    EC.presence_of_element_located((By.NAME, 'submit'))
+)
+
+browser.save_screenshot('/tmp/artifacts/Prometheus_Dash.png')
+
+
+statusBtn = browser.find_element_by_link_text('Status')
+statusBtn.click()
+
+browser.find_element_by_link_text('Runtime & Build Information').click()
+
+el = WebDriverWait(browser, 15).until(
+    EC.presence_of_element_located((By.XPATH, '/html/body/div/table[1]'))
+)
+
+browser.save_screenshot('/tmp/artifacts/Prometheus_RuntimeInfo.png')
+
+statusBtn = browser.find_element_by_link_text('Status')
+statusBtn.click()
+
+browser.find_element_by_link_text('Command-Line Flags').click()
+
+el = WebDriverWait(browser, 15).until(
+    EC.presence_of_element_located((By.XPATH, '/html/body/div/table'))
+)
+
+browser.save_screenshot('/tmp/artifacts/Prometheus_CommandLineFlags.png')

zuul.d/jobs.yaml

@@ -37,6 +37,7 @@
     pre-run:
       - playbooks/osh-infra-upgrade-host.yaml
       - playbooks/osh-infra-deploy-docker.yaml
+      - playbooks/osh-infra-deploy-selenium.yaml
       - playbooks/osh-infra-build.yaml
       - playbooks/osh-infra-deploy-k8s.yaml
     run: playbooks/osh-infra-gate-runner.yaml
@@ -61,6 +62,9 @@
         - ./tools/deployment/multinode/120-elasticsearch.sh
         - ./tools/deployment/multinode/130-fluent-logging.sh
         - ./tools/deployment/multinode/140-kibana.sh
+        - ./tools/deployment/multinode/grafana-selenium.sh
+        - ./tools/deployment/multinode/nagios-selenium.sh
+        - ./tools/deployment/multinode/prometheus-selenium.sh
 
 - job:
     name: openstack-helm-infra-tenant-ceph
@@ -70,6 +74,7 @@
     pre-run:
       - playbooks/osh-infra-upgrade-host.yaml
       - playbooks/osh-infra-deploy-docker.yaml
+      - playbooks/osh-infra-deploy-selenium.yaml
       - playbooks/osh-infra-build.yaml
       - playbooks/osh-infra-deploy-k8s.yaml
     run: playbooks/osh-infra-gate-runner.yaml
@@ -104,7 +109,9 @@
     name: openstack-helm-infra-aio-logging
     parent: openstack-helm-infra-functional
     timeout: 7200
-    pre-run: playbooks/osh-infra-upgrade-host.yaml
+    pre-run:
+      - playbooks/osh-infra-upgrade-host.yaml
+      - playbooks/osh-infra-deploy-selenium.yaml
     run: playbooks/osh-infra-gate-runner.yaml
     post-run: playbooks/osh-infra-collect-logs.yaml
     nodeset: openstack-helm-single-node
@@ -126,7 +133,9 @@
     name: openstack-helm-infra-aio-monitoring
     parent: openstack-helm-infra-functional
     timeout: 7200
-    pre-run: playbooks/osh-infra-upgrade-host.yaml
+    pre-run:
+      - playbooks/osh-infra-upgrade-host.yaml
+      - playbooks/osh-infra-deploy-selenium.yaml
     run: playbooks/osh-infra-gate-runner.yaml
     post-run: playbooks/osh-infra-collect-logs.yaml
     nodeset: openstack-helm-single-node
@@ -146,12 +155,17 @@
         - ./tools/deployment/osh-infra-monitoring/090-process-exporter.sh
         - ./tools/deployment/osh-infra-monitoring/100-grafana.sh
         - ./tools/deployment/osh-infra-monitoring/110-nagios.sh
+        - ./tools/deployment/osh-infra-monitoring/grafana-selenium.sh
+        - ./tools/deployment/osh-infra-monitoring/prometheus-selenium.sh
+        - ./tools/deployment/osh-infra-monitoring/nagios-selenium.sh
 
 - job:
     name: openstack-helm-infra-aio-network-policy
     parent: openstack-helm-infra-functional
     timeout: 7200
-    pre-run: playbooks/osh-infra-upgrade-host.yaml
+    pre-run:
+      - playbooks/osh-infra-upgrade-host.yaml
+      - playbooks/osh-infra-deploy-selenium.yaml
     run: playbooks/osh-infra-gate-runner.yaml
     post-run: playbooks/osh-infra-collect-logs.yaml
     nodeset: openstack-helm-single-node