Add initial .zuul file, devstack and packstack job

* Create a structure for Zuul v3 jobs.
* Add tempest jobs running on devstack and packstack environment with admin
  and demo credentials.

Depends-On: I6f19d2b8128fbb0f2034892a11127bce73d102f9
Change-Id: Id52e7e1c42e7e93bece7907b8f4b05e5bfa2a7f9

.zuul.yaml

@@ -4,9 +4,17 @@
     check:
       jobs:
         - python-tempestconf-tox-cover
+        - python-tempestconf-tempest-devstack-admin
+        - python-tempestconf-tempest-devstack-demo
+        - python-tempestconf-tempest-packstack-admin
+        - python-tempestconf-tempest-packstack-demo
         - tripleo-ci-centos-7-containers-multinode
     gate:
       jobs:
+        - python-tempestconf-tempest-devstack-admin
+        - python-tempestconf-tempest-devstack-demo
+        - python-tempestconf-tempest-packstack-admin
+        - python-tempestconf-tempest-packstack-demo
         - tripleo-ci-centos-7-containers-multinode
 
 - job:
@@ -19,3 +27,99 @@
       - ^releasenotes/.*$
     vars:
       tox_envlist: cover
+
+- job:
+    name: python-tempestconf-tempest-devstack-admin
+    parent: devstack
+    description: |
+      Tempest job for python-tempestconf on a devstack environment as the admin user.
+    required-projects:
+      - openstack/python-tempestconf
+      - openstack/tempest
+      - openstack-dev/devstack
+    roles:
+      - zuul: openstack/python-tempestconf
+      - zuul: openstack/tempest
+      - zuul: openstack-dev/devstack
+    run: playbooks/python-tempestconf-tempest-devstack.yaml
+    vars:
+      user: "admin"
+      cloud_user: "devstack-admin"
+    irrelevant-files:
+      - config_tempest/tests/.*$
+      - ^doc/.*$
+      - ^releasenotes/.*$
+      - ^.*\.rst$
+
+- job:
+    name: python-tempestconf-tempest-devstack-demo
+    parent: devstack
+    description: |
+      Tempest job for python-tempestconf on a devstack environment as the demo user.
+    required-projects:
+      - openstack/python-tempestconf
+      - openstack/tempest
+      - openstack-dev/devstack
+    roles:
+      - zuul: openstack/python-tempestconf
+      - zuul: openstack/tempest
+      - zuul: openstack-dev/devstack
+    run: playbooks/python-tempestconf-tempest-devstack.yaml
+    vars:
+      user: "demo"
+      cloud_user: "devstack"
+    irrelevant-files:
+      - config_tempest/tests/.*$
+      - ^doc/.*$
+      - ^releasenotes/.*$
+      - ^.*\.rst$
+
+- job:
+    name: python-tempestconf-tempest-packstack-admin
+    parent: packstack-base
+    description: |
+      Tempest job for python-tempestconf on a packstack environment as the admin user.
+    required-projects:
+      - openstack/packstack
+      - openstack/python-tempestconf
+      - openstack/tempest
+      - openstack-dev/devstack
+    roles:
+      - zuul: openstack/packstack
+      - zuul: openstack/python-tempestconf
+      - zuul: openstack/tempest
+      - zuul: openstack-dev/devstack
+    run: playbooks/python-tempestconf-tempest-packstack.yaml
+    vars:
+      scenario: scenario000
+      user: "admin"
+    irrelevant-files:
+      - config_tempest/tests/.*$
+      - ^doc/.*$
+      - ^releasenotes/.*$
+      - ^.*\.rst$
+
+- job:
+    name: python-tempestconf-tempest-packstack-demo
+    parent: packstack-base
+    description: |
+      Tempest job for python-tempestconf on a packstack environment as the demo user.
+    required-projects:
+      - openstack/packstack
+      - openstack/python-tempestconf
+      - openstack/tempest
+      - openstack-dev/devstack
+    roles:
+      - zuul: openstack/packstack
+      - zuul: openstack/python-tempestconf
+      - zuul: openstack/tempest
+      - zuul: openstack-dev/devstack
+    run: playbooks/python-tempestconf-tempest-packstack.yaml
+    vars:
+      scenario: scenario000
+      user: "demo"
+    irrelevant-files:
+      - config_tempest/tests/.*$
+      - ^doc/.*$
+      - ^releasenotes/.*$
+      - ^.*\.rst$

playbooks/python-tempestconf-tempest-devstack.yaml

@@ -0,0 +1,39 @@
+- hosts: all
+  roles:
+    # the role is inherited from openstack-dev/devstack project
+    - run-devstack
+
+- hosts: tempest
+  vars:
+    # It's important that OS_AUTH_URL is in this format:
+    # <protocol>://<host>/identity/v3
+    # In some cases it can be only: <protocol>://<host>:5000
+    # https://github.com/openstack-dev/devstack/blob/2c9343e5db44fa7a41ca6924737331dd9088ef8f/openrc#L87-L89
+    # (mkopec) That would end up with an HTTP MaxRetryError to <host>:5000,
+    # therefor OS_AUTH_URL is defined manually
+    set_auth_url: "OS_AUTH_URL=$SERVICE_PROTOCOL://$SERVICE_HOST/identity/v3"
+  tasks:
+    # setup-tempest-* and acl-devstack-files roles are inherited from
+    # openstack/tempest project
+    - name: Setup Tempest Run Directory
+      include_role:
+        name: setup-tempest-run-dir
+    - name: Setup Tempest Data Directory
+      include_role:
+        name: setup-tempest-data-dir
+    - name: ACL devstack files
+      include_role:
+        name: acl-devstack-files
+    - name: Generate tempest configuration file
+      include_role:
+        name: generate-tempestconf-file
+      vars:
+        source_credentials_commands: "export HOST_IP={{ ansible_default_ipv4.address }}; source {{ devstack_base_dir }}/devstack/openrc {{ user }} {{ user }}; {{ set_auth_url }}"
+        aditional_tempestconf_params: "auth.tempest_roles Member"
+    - name: Generate tempest configuration file based on cloud credentials
+      include_role:
+        name: generate-tempestconf-file-cloud
+    # run-tempest role is inherited from openstack/tempest project
+    - name: Run Tempest Tests
+      include_role:
+        name: run-tempest

playbooks/python-tempestconf-tempest-packstack.yaml

@@ -0,0 +1,41 @@
+- hosts: all
+  tasks:
+    # packstack-integration-tempest role is inherited from
+    # openstack/packstack project
+    - name: Install packstack allinone
+      include_role:
+        name: packstack-integration-tempest
+    # setup-tempest-user, setup-stack-user and setup-devstack-source-dirs
+    # are inherited from openstack-dev/devstack
+    - name: Setup Stack user
+      include_role:
+        name: setup-stack-user
+    - name: Setup Tempest user
+      include_role:
+        name: setup-tempest-user
+    - name: Setup Tempest user
+      include_role:
+        name: setup-devstack-source-dirs
+    # setup-tempest-* and acl-devstack-files roles are inherited from
+    # openstack/tempest project
+    - name: Setup Tempest Run Directory
+      include_role:
+        name: setup-tempest-run-dir
+    - name: Setup Tempest Data Directory
+      include_role:
+        name: setup-tempest-data-dir
+    - name: ACL devstack files
+      include_role:
+        name: acl-devstack-files
+    - name: Prepare keystonerc credentials generated by packstack
+      include_role:
+        name: create-keystonerc-files
+    - name: Generate configuration file for python-tempestconf
+      include_role:
+        name: generate-tempestconf-file
+      vars:
+        source_credentials_commands: "source {{ ansible_user_dir }}/keystonerc_{{ user }}"
+    # run-tempest role is inherited from openstack/tempest project
+    - name: Run Tempest Tests
+      include_role:
+        name: run-tempest

roles/create-keystonerc-files/README.rst

@@ -0,0 +1,59 @@
+Create keystonerc files
+
+Creates keystonerc files for tempest, print them to the output and copy them
+to the wanted destination.
+
+**Role Variables**
+
+.. zuul:rolevar:: admin_user
+   :type: dict
+
+    A dictionary of admin user credentials.
+
+   .. zuul:rolevar:: username
+      :default: admin
+
+   .. zuul:rolevar:: password
+      :default: packstack
+
+   .. zuul:rolevar:: project_name
+      :default: admin
+
+   .. zuul:rolevar:: user_domain_name
+      :default: Default
+
+   .. zuul:rolevar:: project_domain_name
+      :default: Default
+
+   .. zuul:rolevar:: identity_api_version
+      :default: 3
+
+   .. zuul:rolevar:: keystonerc_destination
+      :default: {{ ansible_user_dir }}/keystonerc_admin
+
+.. zuul:rolevar:: demo_user
+   :type: dict
+
+    A dictionary of demo user credentials.
+
+   .. zuul:rolevar:: username
+      :default: demo
+
+   .. zuul:rolevar:: password
+      :default: packstack
+
+   .. zuul:rolevar:: project_name
+      :default: demo
+
+   .. zuul:rolevar:: user_domain_name
+      :default: Default
+
+   .. zuul:rolevar:: project_domain_name
+      :default: Default
+
+   .. zuul:rolevar:: identity_api_version
+      :default: 3
+
+   .. zuul:rolevar:: keystonerc_destination
+      :default: {{ ansible_user_dir }}/keystonerc_demo
+

roles/create-keystonerc-files/defaults/main.yaml

@@ -0,0 +1,18 @@
+admin_user:
+    username: admin
+    password: packstack
+    project_name: admin
+    user_domain_name: Default
+    project_domain_name: Default
+    identity_api_version: "3"
+    keystonerc_destination: "{{ ansible_user_dir }}/keystonerc_admin"
+
+demo_user:
+    username: demo
+    password: packstack
+    project_name: demo
+    user_domain_name: Default
+    project_domain_name: Default
+    identity_api_version: "3"
+    keystonerc_destination: "{{ ansible_user_dir }}/keystonerc_demo"
+

roles/create-keystonerc-files/tasks/main.yaml

@@ -0,0 +1,39 @@
+- debug:
+      var: ansible_all_ipv4_addresses
+      var: ansible_default_ipv4.address
+
+- name: Create keystonerc_admin file
+  shell: |
+      echo """
+      unset OS_SERVICE_TOKEN
+      export OS_USERNAME={{ admin_user.username }}
+      export OS_PASSWORD={{ admin_user.password }}
+      export OS_AUTH_URL=http://{{ ansible_default_ipv4.address }}:5000/v3
+
+      export PS1='[\u@\h \W(keystone_admin)]\$ '
+      export OS_PROJECT_NAME={{ admin_user.project_name }}
+      export OS_USER_DOMAIN_NAME={{ admin_user.user_domain_name }}
+      export OS_PROJECT_DOMAIN_NAME={{ admin_user.project_domain_name }}
+      export OS_IDENTITY_API_VERSION={{ admin_user.identity_api_version }}
+      """ > {{ admin_user.keystonerc_destination }}
+      cat {{ admin_user.keystonerc_destination }}
+  args:
+      executable: /bin/bash
+
+- name: Create keystonerc_demo file
+  shell: |
+      echo """
+      unset OS_SERVICE_TOKEN
+      export OS_USERNAME={{ demo_user.username }}
+      export OS_PASSWORD={{demo_user.password }}
+      export OS_AUTH_URL=http://{{ ansible_default_ipv4.address }}:5000/v3
+
+      export PS1='[\u@\h \W(keystone_admin)]\$ '
+      export OS_PROJECT_NAME={{ demo_user.project_name }}
+      export OS_USER_DOMAIN_NAME={{ demo_user.user_domain_name }}
+      export OS_PROJECT_DOMAIN_NAME={{ demo_user.project_domain_name }}
+      export OS_IDENTITY_API_VERSION={{ demo_user.identity_api_version }}
+      """ > {{ demo_user.keystonerc_destination }}
+      cat {{ demo_user.keystonerc_destination }}
+  args:
+      executable: /bin/bash

roles/generate-tempestconf-file-cloud/README.rst

@@ -0,0 +1,37 @@
+Generate configuration file for tempest from cloud credentials
+
+Installs python-tempestconf cloned from git and generates tempest.conf with
+credentials saved in clouds.yaml file. The tempest configuration file is printed
+to the output.
+
+**Role Variables**
+
+.. zuul:rolevar:: cloud_user
+   :type: string
+   :default: devstack
+
+   Named cloud with demo user credentials as a default value.
+
+.. zuul:rolevar:: virtualenvs
+   :type: dict
+
+    A dictionary of paths to virtual environments.
+
+   .. zuul:rolevar:: tempestconf
+      :default: ~/.virtualenvs/.tempestconf
+
+       The path to the virtual environment of python-tempestconf tool.
+
+.. zuul:rolevar:: url_cirros_image
+   :type: string
+   :default: http://download.cirros-cloud.net/0.3.5/cirros-0.3.5-x86_64-disk.img
+
+    A URL address of the cirros image.
+
+.. zuul:rolevar:: tempestconf_src_relative_path
+   :type: string
+
+    A relative path to a python-tempestconf project which is by default cloned
+    to the Zuul home directory. Value of the variable is set in the role to
+    that default path. If needed, the variable can be overridden from the
+    playbook where the role is called.

roles/generate-tempestconf-file-cloud/defaults/main.yaml

@@ -0,0 +1,4 @@
+cloud_user: "devstack"
+virtualenvs:
+    tempestconf: "~/.virtualenvs/.tempestconf"
+url_cirros_image: "http://download.cirros-cloud.net/0.3.5/cirros-0.3.5-x86_64-disk.img"

roles/generate-tempestconf-file-cloud/tasks/main.yaml

@@ -0,0 +1,75 @@
+- block:
+  - name: Create python-tempestconf venv with latest pip, setuptools and pbr
+    pip:
+        virtualenv: "{{ virtualenvs.tempestconf }}"
+        name: "{{ item }}"
+        state: latest
+    with_items:
+        - pip
+        - setuptools
+        - pbr
+
+  - name: Debug, list tempetsconf dir
+    shell: |
+        set -ex
+        ls -all .
+        pwd
+    args:
+        chdir: "{{ tempestconf_src_relative_path }}"
+
+  - name: Install python-tempestconf
+    pip:
+        name: "."
+        virtualenv: "{{ virtualenvs.tempestconf }}"
+        chdir: "{{ tempestconf_src_relative_path }}"
+
+  - name: Cat clouds.yaml file
+    shell: |
+        set -ex
+        cat /etc/openstack/clouds.yaml
+    ignore_errors: True
+    args:
+       executable: /bin/bash
+
+  - name: "Workaround for AUTH URL in clouds.yaml"
+    replace:
+        path: /etc/openstack/clouds.yaml
+        regexp: "auth_url:.*"
+        replace: "auth_url: http://{{ ansible_default_ipv4.address }}/identity/v3"
+    become: true
+    ignore_errors: True
+
+  - name: Cat edited clouds.yaml file
+    shell: |
+        set -ex
+        cat /etc/openstack/clouds.yaml
+    ignore_errors: True
+    args:
+       executable: /bin/bash
+
+  - name: Generate tempest configuration file
+    shell: |
+        set -ex
+        export PATH=$PATH:/usr/local/sbin:/usr/sbin
+        source {{ virtualenvs.tempestconf }}/bin/activate
+        printenv
+        discover-tempest-config \
+          --out etc/cloud_tempest.conf \
+          --debug \
+          -v \
+          --create \
+          --os-cloud {{ cloud_user }} \
+          auth.tempest_roles Member \
+          service_available.swift False \
+          image.http_image {{ url_cirros_image }}
+    args:
+        chdir: "{{ tempestconf_src_relative_path }}"
+        executable: /bin/bash
+
+  - name: Print generated tempest.conf
+    shell: |
+        set -ex
+        cat {{ tempestconf_src_relative_path }}/etc/cloud_tempest.conf
+
+  vars:
+    tempestconf_src_relative_path: "{{ zuul._projects['git.openstack.org/openstack/python-tempestconf'].src_dir }}"

roles/generate-tempestconf-file/README.rst

@@ -0,0 +1,60 @@
+Generate configuration file for tempest
+
+Installs python-tempestconf cloned from git and generates tempest.conf which
+is then copied to tempest directory.
+
+**Role Variables**
+
+.. zuul:rolevar:: devstack_base_dir
+   :type: string
+   :default: /opt/stack
+
+    The devstack base directory.
+
+.. zuul:rolevar:: virtualenvs
+   :type: dict
+
+    A dictionary of paths to virtual environments.
+
+   .. zuul:rolevar:: tempestconf
+      :default: ~/.virtualenvs/.tempestconf
+
+       The path to the virtual environment of python-tempestconf tool.
+
+.. zuul:rolevar:: source_credentials_commands
+   :type: string
+
+   Commands divided by a semicolon which defines sourcing credentials for
+   running python-tempestconf tool with. They need to be defined in the
+   playbook the role is called from.
+   For example for devstack it's used ({{ devstack_base_dir}} is the location
+   where run-devstack role copies devstack source files):
+
+   `source {{ devstack_base_dir }}/devstack/openrc {{ user }} {{ user }}`
+
+   and for packstack:
+
+   `source {{ ansible_user_dir }}/keystonerc_{{ user }}`
+
+.. zuul:rolevar:: url_cirros_image
+   :type: string
+   :default: http://download.cirros-cloud.net/0.3.5/cirros-0.3.5-x86_64-disk.img
+
+    A URL address of the cirros image.
+
+.. zuul:rolevar:: tempestconf_src_relative_path
+   :type: string
+
+    A relative path to a python-tempestconf project which is by default cloned
+    to the Zuul home directory. Value of the variable is set in the role to
+    that default path. If needed, the variable can be overridden from the
+    playbook where the role is called.
+
+
+.. zuul:rolevar:: aditional_tempestconf_params
+   :type: string
+   :default: ""
+
+   Additional parameters for tempestconf if more specific parameters are
+   needed, different from the ones which are the same for devstack and tempest.
+

roles/generate-tempestconf-file/defaults/main.yaml

@@ -0,0 +1,5 @@
+devstack_base_dir: /opt/stack
+virtualenvs:
+    tempestconf: ~/.virtualenvs/.tempestconf
+url_cirros_image: "http://download.cirros-cloud.net/0.3.5/cirros-0.3.5-x86_64-disk.img"
+aditional_tempestconf_params: ""

roles/generate-tempestconf-file/tasks/main.yaml

@@ -0,0 +1,67 @@
+- block:
+  - debug:
+      var: source_credentials_commands
+
+  - name: Create python-tempestconf venv with latest pip, setuptools and pbr
+    pip:
+        virtualenv: "{{ virtualenvs.tempestconf }}"
+        name: "{{ item }}"
+        state: latest
+    with_items:
+        - pip
+        - setuptools
+        - pbr
+
+  - name: Debug, list tempetsconf dir
+    shell: |
+        set -ex
+        ls -all .
+        pwd
+    args:
+        chdir: "{{ tempestconf_src_relative_path }}"
+
+  - name: Install python-tempestconf
+    pip:
+        name: "."
+        virtualenv: "{{ virtualenvs.tempestconf }}"
+        chdir: "{{ tempestconf_src_relative_path }}"
+
+  - name: "Cat keystonerc_ file (only in packstack case)"
+    shell: |
+        set -ex
+        cat {{ ansible_user_dir }}/keystonerc_{{ user }}
+    ignore_errors: True
+    args:
+       executable: /bin/bash
+
+  - name: Generate tempest configuration file
+    shell: |
+        set -ex
+        export PATH=$PATH:/usr/local/sbin:/usr/sbin
+        source {{ virtualenvs.tempestconf }}/bin/activate
+        {{ source_credentials_commands }}
+        printenv
+        discover-tempest-config \
+          --debug \
+          -v \
+          --create \
+          identity.uri $OS_AUTH_URL \
+          identity.admin_password $OS_PASSWORD \
+          service_available.swift False \
+          {{ aditional_tempestconf_params }} \
+          image.http_image {{ url_cirros_image }}
+    args:
+        chdir: "{{ tempestconf_src_relative_path }}"
+        executable: /bin/bash
+
+  - name: Print generated tempest.conf and copy it to tempest directory
+    shell: |
+        set -x
+        cat {{ tempestconf_src_relative_path }}/etc/tempest.conf
+        ls /opt/stack/
+        ls /opt/stack/tempest
+        ls /opt/stack/tempest/etc
+        cp {{ tempestconf_src_relative_path }}/etc/tempest.conf {{ devstack_base_dir }}/tempest/etc/tempest.conf
+
+  vars:
+    tempestconf_src_relative_path: "{{ zuul._projects['git.openstack.org/openstack/python-tempestconf'].src_dir }}"