Benchmark requests during upgrade testing

During upgrade testing, move Keystone containers behind an HAProxy
load balancer and run a benchmarking tool to make continuous requests
against the keystone API, providing summary output at the end of the
play. This should help ensure the upgrade process between releases
remains without downtime.

To avoid service interruptions because of files being replaced during
installations:
  - set the 'keystone_venv_tag' var so that it changes between releases
  - perform the 'Create WSGI symlinks' task only after the files being
    linked to exist

Benchmarking scripts were provided by:
https://github.com/lbragstad/keystone-performance-upgrade

Co-Authored-By: Lance Bragstad <lbragstad@gmail.com>

Change-Id: I462e6496f125c7e263bbab188e86c45e1f4f7f1e

tasks/keystone_install.yml

@@ -152,18 +152,6 @@
   when:
     - keystone_developer_mode | bool
 
-- name: Create WSGI symlinks
-  file:
-    src: "{{ item.src }}"
-    dest: "/var/www/cgi-bin/keystone/{{ item.dest }}"
-    state: link
-    force: yes
-  with_items:
-    - src: "{{ keystone_bin }}/keystone-wsgi-admin"
-      dest: admin
-    - src: "{{ keystone_bin }}/keystone-wsgi-public"
-      dest: main
-
 - name: Create developer mode constraint file
   copy:
     dest: "/opt/developer-pip-constraints.txt"
@@ -251,3 +239,15 @@
   command: >
     virtualenv-tools --update-path=auto --reinitialize {{ keystone_bin | dirname }}
   when: keystone_get_venv | changed
+
+- name: Create WSGI symlinks
+  file:
+    src: "{{ item.src }}"
+    dest: "/var/www/cgi-bin/keystone/{{ item.dest }}"
+    state: link
+    force: yes
+  with_items:
+    - src: "{{ keystone_bin }}/keystone-wsgi-admin"
+      dest: admin
+    - src: "{{ keystone_bin }}/keystone-wsgi-public"
+      dest: main

tests/ansible-role-requirements.yml

@@ -30,6 +30,10 @@
   src: https://git.openstack.org/openstack/openstack-ansible-galera_server
   scm: git
   version: master
+- name: haproxy_server
+  src: https://git.openstack.org/openstack/openstack-ansible-haproxy_server
+  scm: git
+  version: master
 - name: rabbitmq_server
   src: https://git.openstack.org/openstack/openstack-ansible-rabbitmq_server
   scm: git

tests/inventory

@@ -29,3 +29,6 @@ keystone2
 
 [utility_all]
 keystone1
+
+[haproxy_all]
+localhost

tests/keystone-test-vars.yml

@@ -0,0 +1,48 @@
+---
+# Copyright 2017, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+haproxy_default_services:
+  - service:
+      haproxy_service_name: galera
+      haproxy_backend_nodes: "{{ [groups['galera_all'][0]] | default([]) }}"  # list expected
+      haproxy_backup_nodes: "{{ groups['galera_all'][1:] | default([]) }}"
+      haproxy_port: 3306
+      haproxy_balance_type: tcp
+      haproxy_timeout_client: 5000s
+      haproxy_timeout_server: 5000s
+      haproxy_backend_options:
+        - "mysql-check user {{ galera_monitoring_user }}"
+      haproxy_whitelist_networks:
+        - 192.168.0.0/16
+        - 172.16.0.0/12
+        - 10.0.0.0/8
+  - service:
+      haproxy_service_name: keystone_service
+      haproxy_backend_nodes: "{{ groups['keystone_all'] | default([])  }}"
+      haproxy_port: 5000
+      haproxy_ssl: "{{ haproxy_ssl }}"
+      haproxy_balance_type: "http"
+      haproxy_backend_options:
+        - "httpchk HEAD /"
+  - service:
+      haproxy_service_name: keystone_admin
+      haproxy_backend_nodes: "{{ groups['keystone_all'] | default([])  }}"
+      haproxy_port: 35357
+      haproxy_balance_type: "http"
+      haproxy_backend_options:
+        - "httpchk HEAD /"
+
+ssl_cipher_suite: "ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:!aNULL:!MD5:!DSS"
+haproxy_ssl: false

tests/os_keystone-overrides.yml

@@ -25,3 +25,7 @@ tempest_plugins:
 
 tempest_test_whitelist:
   - keystone_tempest_plugin.tests.api.identity*
+
+external_lb_vip_address: 10.1.0.1
+internal_lb_vip_address: 10.1.0.1
+test_keystone_host: "{{ (keystone_upgrade is defined) | ternary(external_lb_vip_address, hostvars[groups[test_keystone_group][0]]['ansible_host']) }}"

tests/templates/auth.json

@@ -0,0 +1,26 @@
+{
+    "auth": {
+        "identity": {
+            "methods": [
+                "password"
+            ],
+            "password": {
+                "user": {
+                    "domain": {
+                        "id": "default"
+                    },
+                    "password": "{{ keystone_auth_admin_password }}",
+                    "name": "admin"
+                }
+            }
+        },
+        "scope": {
+            "project": {
+                "domain": {
+                    "id": "default"
+                },
+                "name": "admin"
+            }
+        }
+    }
+}

tests/templates/locustfile.py

@@ -0,0 +1,25 @@
+import json
+
+import locust
+
+TOKEN_PATH = 'http://{{ external_lb_vip_address }}:5000/v3/auth/tokens'
+
+
+class Task(locust.TaskSet):
+    @locust.task
+    def authenticate(self):
+        # authenticate for a token and validate it
+        with open('/opt/auth.json', 'r') as f:
+            body = json.loads(f.read())
+        headers = {'Content-Type': 'application/json'}
+        response = self.client.post(TOKEN_PATH, json=body, headers=headers)
+        token = response.headers.get('X-Subject-Token')
+        headers = {
+            'X-Subject-Token': token,
+            'X-Auth-Token': token
+        }
+        self.client.get(TOKEN_PATH, headers=headers)
+
+
+class MyLocust(locust.HttpLocust):
+    task_set = Task

tests/test-benchmark-keystone-upgrade-results.yml

@@ -0,0 +1,27 @@
+---
+# Copyright 2017, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: Playbook for deploying upgrade benchmarking
+  hosts: localhost
+  user: root
+  tasks:
+    - name: Kill upgrade benchmarking process
+      command: pkill locust
+    - name: Register upgrade benchmarking result
+      async_status: jid={{ locust_benchmark.ansible_job_id }}
+      register: locust_benchmark_result
+    - name: Display upgrade benchmarking result
+      debug:
+        msg: "{{ locust_benchmark_result.stderr }}"

tests/test-benchmark-keystone-upgrade.yml

@@ -0,0 +1,39 @@
+---
+# Copyright 2017, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: Playbook for deploying upgrade benchmarking
+  hosts: localhost
+  user: root
+  tasks:
+    - name: Install locust
+      pip:
+        name: locustio
+    - name: Drop benchmarking config files
+      template:
+        src: "{{ item.src }}"
+        dest: "{{ item.dest }}"
+        owner: "root"
+        group: "root"
+        mode: "0644"
+      with_items:
+        - { src: "auth.json" , dest: "/opt/auth.json" }
+        - { src: "locustfile.py" , dest: "/opt/locustfile.py" }
+    - name: Run locust
+      command: locust -f /opt/locustfile.py --no-web --clients=4 --hatch-rate=1 --only-summary --host=http://localhost --port=5000 --logfile=/var/log/locust.log
+      async: 1000
+      poll: 0
+      register: locust_benchmark
+  vars_files:
+    - common/test-vars.yml

tests/test-install-haproxy.yml

@@ -0,0 +1,22 @@
+---
+# Copyright 2017, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- name: Install haproxy
+  hosts: localhost
+  roles:
+    - role: "haproxy_server"
+      haproxy_service_configs: "{{ haproxy_default_services }}"
+  vars_files:
+    - keystone-test-vars.yml

tests/test-install-previous-keystone.yml

@@ -18,6 +18,9 @@
   user: root
   gather_facts: true
   pre_tasks:
+    - name: Set keystone_venv_tag fact
+      set_fact:
+        keystone_venv_tag: testing-previous
     - name: Set keystone_messaging fact
       set_fact:
         keystone_messaging_enabled: "{{ groups['rabbitmq_all'] is defined }}"
@@ -31,5 +34,9 @@
       db_password: "{{ keystone_container_mysql_password }}"
   roles:
     - role: "os_previous_keystone"
+  post_tasks:
+    - name: Reset keystone_venv_tag fact
+      set_fact:
+        keystone_venv_tag: testing
   vars_files:
     - common/previous/test-vars.yml

tests/test-upgrade.yml

@@ -22,12 +22,18 @@
 # Prepare the containers
 - include: common/test-prepare-containers.yml
 
+# Install haproxy
+- include: test-install-haproxy.yml
+
 # Install RabbitMQ/MariaDB
 - include: common/test-install-infra.yml
 
 # Install previous Keystone
 - include: test-install-previous-keystone.yml
 
+# Install upgrade benchmarking
+- include: test-benchmark-keystone-upgrade.yml
+
 # Install Keystone
 - include: common/test-install-keystone.yml
 
@@ -36,3 +42,6 @@
 
 # Test Keystone
 - include: test-keystone-functional.yml
+
+# Test upgrade benchmarking results
+- include: test-benchmark-keystone-upgrade-results.yml

tox.ini

@@ -115,6 +115,7 @@ deps =
 setenv =
     {[testenv]setenv}
     TEST_PLAYBOOK={toxinidir}/tests/test-upgrade.yml
+    ANSIBLE_PARAMETERS=-e keystone_upgrade=True
 commands =
     {[testenv:tests_clone]commands}
     bash -c "if [ ! -d "{toxinidir}/tests/common/previous" ]; then \