[ci] Fix existing users job

Change-Id: I964d37efbc47ee56e29cfcaa5317506f79a2bbf1

.zuul.d/zuul.yaml

@@ -8,9 +8,7 @@
         - rally-tox-py38
         - rally-dsvm-tox-functional
         - rally-docker-check
-        - rally-task-basic-with-existing-users:
-            # use_existing_users key did not trigger proper ansible tasks
-            voting: false
+        - rally-task-basic-with-existing-users
         - rally-task-simple-job
         - rally-task-barbican:
             files:
@@ -83,6 +81,7 @@
         - rally-tox-py38
         - rally-dsvm-tox-functional
         - rally-docker-check
+        - rally-task-basic-with-existing-users
         - rally-task-simple-job
         - rally-task-barbican:
             files:

rally-jobs/basic-with-existing-users.yaml

@@ -1,11 +1,11 @@
 {% set flavor_name = "m1.tiny" %}
 {% set image_name = "^cirros.*-disk$" %}
-{% set cirros_image_url = "http://download.cirros-cloud.net/0.3.5/cirros-0.3.5-x86_64-disk.img" %}
+{% set cirros_image_url = "https://github.com/cirros-dev/cirros/releases/download/0.3.5/cirros-0.3.5-x86_64-disk.img" %}
 {% set smoke = 0 %}
 
 ---
   version: 2
-  title: rally-neutron-existing-users.yaml
+  title: Rally task that is called for an evironment with existing users
   description: >
     The task contains various scenarios that do not require admin user
   subtasks:
@@ -72,22 +72,7 @@
           sla:
             failure_rate:
               max: 0
-    -
-      title: Test main Glance actions
-      workloads:
-        -
-          scenario:
-            GlanceImages.create_and_delete_image:
-              image_location: "{{ cirros_image_url }}"
-              container_format: "bare"
-              disk_format: "qcow2"
-          runner:
-            constant:
-              times: 1
-              concurrency: 1
-          sla:
-            failure_rate:
-              max: 100
+
     -
       title: Test main Neutron actions
       workloads:

tests/ci/playbooks/roles/prepare-for-rally-task/defaults/main.yaml

@@ -1,7 +1,5 @@
-existing_user_name_1: "rally-test-user-1"
-existing_user_password_1: "rally-test-password-1"
-existing_user_project_1: "rally-test-project-1"
-existing_user_name_2: "rally-test-user-2"
-existing_user_password_2: "rally-test-password-2"
-existing_user_project_2: "rally-test-project-2"
-RALLY_OSPROFILER_CHART: "osprofiler_reports"
+rally_use_existing_users: False
+existing_users_env_spec: '{{ rally_home_dir }}/env-with-existing-users-config'
+projects_count: 2
+users_per_project: 1
+RALLY_OSPROFILER_CHART: "osprofiler_reports"

tests/ci/playbooks/roles/prepare-for-rally-task/library/make_env_spec_with_existing_users.py

@@ -0,0 +1,143 @@
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import copy
+import json
+import uuid
+
+from ansible.module_utils.basic import AnsibleModule
+
+from rally import api
+from rally.env import env_mgr
+from rally import plugins
+
+from rally_openstack.common import consts
+from rally_openstack.common import credential
+
+
+def fetch_parent_env_and_admin_creds(env_name):
+    """Fetch parent environment spec and openstack admin creds from it."""
+
+    env_data = env_mgr.EnvManager.get(env_name).data
+
+    openstack_platform = env_data["platforms"]["openstack"]
+    admin_creds = credential.OpenStackCredential(
+        permission=consts.EndpointPermission.ADMIN,
+        **openstack_platform["platform_data"]["admin"])
+
+    return env_data["spec"], admin_creds
+
+
+def create_projects_and_users(admin_creds, projects_count, users_per_project):
+    """Create new projects and users via 'users@openstack' context.
+
+    :param admin_creds: admin credentials to use for creating new entities
+    :param projects_count: The number of keystone projects to create.
+    :param users_per_project: The number of keystone users to create per one
+        keystone project.
+    """
+
+    # it should be imported after calling rally.api.API that setups oslo_config
+    from rally_openstack.task.contexts.keystone import users as users_ctx
+
+    ctx = {
+        "env": {
+            "platforms": {
+                "openstack": {
+                    "admin": admin_creds.to_dict(),
+                    "users": []
+                }
+            }
+        },
+        "task": {
+            "uuid": str(uuid.uuid4())
+        },
+        "config": {
+            "users@openstack": {
+                "tenants": projects_count,
+                "users_per_tenant": users_per_project
+            }
+        }
+    }
+
+    users_ctx.UserGenerator(ctx).setup()
+
+    users = []
+    for user in ctx["users"]:
+        users.append({
+            "username": user["credential"]["username"],
+            "password": user["credential"]["password"],
+            "project_name": user["credential"]["tenant_name"]
+        })
+
+        for optional in ("domain_name",
+                         "user_domain_name",
+                         "project_domain_name"):
+            if user["credential"][optional]:
+                users[-1][optional] = user["credential"][optional]
+
+    return users
+
+
+def store_a_new_spec(original_spec, users, path_for_new_spec):
+    new_spec = copy.deepcopy(original_spec)
+    del new_spec["existing@openstack"]["admin"]
+    new_spec["existing@openstack"]["users"] = users
+    with open(path_for_new_spec, "w") as f:
+        f.write(json.dumps(new_spec, indent=4))
+
+
+@plugins.ensure_plugins_are_loaded
+def ansible_main():
+    module = AnsibleModule(argument_spec=dict(
+        projects_count=dict(
+            type="int",
+            default=1,
+            required=False
+        ),
+        users_per_project=dict(
+            type="int",
+            default=1,
+            required=False
+        ),
+        parent_env_name=dict(
+            type="str",
+            required=True
+        ),
+        path_for_new_spec=dict(
+            type="str",
+            required=True
+        )
+    ))
+
+    # init Rally API as it makes all work for logging and config initialization
+    api.API()
+
+    original_spec, admin_creds = fetch_parent_env_and_admin_creds(
+        module.params["parent_env_name"]
+    )
+
+    users = create_projects_and_users(
+        admin_creds,
+        projects_count=module.params["projects_count"],
+        users_per_project=module.params["users_per_project"]
+    )
+
+    store_a_new_spec(original_spec, users, module.params["path_for_new_spec"])
+
+    module.exit_json(changed=True)
+
+
+if __name__ == "__main__":
+    ansible_main()

tests/ci/playbooks/roles/prepare-for-rally-task/tasks/main.yaml

@@ -110,57 +110,12 @@
           openstack network list
       fi
 
-- name: Create new projects and users
+- name: Print Rally environment config
   become: True
   become_user: stack
-  shell:
-    executable: /bin/sh
-    cmd: |
-      set -e
+  command: "rally env show --only-spec"
 
-      . {{ rally_home_dir }}/openrc admin admin
-
-      openstack --version
-
-      openstack project create {{ existing_user_project_1 }}
-      openstack user create --project {{ existing_user_project_1 }} --password {{ existing_user_password_1 }} {{ existing_user_name_1 }}
-      openstack role add --project {{ existing_user_project_1 }} --user {{ existing_user_name_1 }} Member
-
-      openstack project create {{ existing_user_project_2 }}
-      openstack user create --project {{ existing_user_project_2 }} --password {{ existing_user_password_2 }} {{ existing_user_name_2 }}
-      openstack role add --project {{ existing_user_project_2 }} --user {{ existing_user_name_2 }} Member
-
-      set +e
-      NEUTRON_EXISTS=$(openstack --os-interface admin service list | grep neutron)
-      set -e
-      if [ "$NEUTRON_EXISTS" ]; then
-          OS_QUOTA_STR="--networks -1 --subnets -1 --routers -1 --floating-ips -1 --subnetpools -1 --secgroups -1 --secgroup-rules -1 --ports -1"
-          openstack --debug quota set $OS_QUOTA_STR {{ existing_user_project_1 }}
-          openstack --debug quota show {{ existing_user_project_1 }}
-          openstack --debug quota set $OS_QUOTA_STR {{ existing_user_project_2 }}
-          openstack --debug quota show {{ existing_user_project_2 }}
-      fi
-  when: rally_use_existing_users == True
-
-- name: Capture Keystone auth URL
-  become: True
-  become_user: stack
-  shell: ". {{ rally_home_dir }}/openrc admin admin > /dev/null && echo $OS_AUTH_URL"
-  register: keystone_auth_url
-  when: rally_use_existing_users == True
-
-- name: Make Rally Environment spec with existing users
-  become: True
-  become_user: stack
-  template:
-    src: env.yaml.j2
-    dest: "{{ rally_existing_users_config }}"
-  when: rally_use_existing_users == True
-
-- name: Create new projects and users
-  become: True
-  become_user: stack
-  shell: rally env create --name devstask-with-users --spec "{{ rally_existing_users_config }}"
+- include_tasks: prepare-env-with-existing-users.yaml
   when: rally_use_existing_users == True
 
 - name: Check Environment works
@@ -168,11 +123,6 @@
   become_user: stack
   command: "rally --debug env check"
 
-- name: Print Rally deployment config
-  become: True
-  become_user: stack
-  command: "rally deployment config"
-
 - name: Print Environment info
   become: True
   become_user: stack

tests/ci/playbooks/roles/prepare-for-rally-task/tasks/prepare-env-with-existing-users.yaml

@@ -0,0 +1,18 @@
+- name: Create keystone projects & users for a new rally environment
+  become: yes
+  become_user: stack
+  make_env_spec_with_existing_users:
+    projects_count: "{{ projects_count }}"
+    users_per_project: "{{ users_per_project }}"
+    parent_env_name: "devstack"
+    path_for_new_spec: "{{ existing_users_env_spec }}"
+
+- name: Create a new Rally environment
+  become: True
+  become_user: stack
+  shell: rally env create --name devstask-with-users --spec "{{ existing_users_env_spec }}"
+
+- name: Print new Rally env spec
+  become: True
+  become_user: stack
+  command: "rally env show --only-spec"

tests/ci/playbooks/roles/prepare-for-rally-task/templates/env.yaml.j2

@@ -1,19 +0,0 @@
-{
-    "openstack": {
-        "users": [
-            {"username": "{{ existing_user_name_1 }}",
-             "password": "{{ existing_user_password_1 }}",
-             "project_name": "{{ existing_user_project_1 }}",
-             "user_domain_name": "Default",
-             "project_domain_name": "Default"
-            },
-            {"username": "{{ existing_user_name_2 }}",
-             "password": "{{ existing_user_password_2 }}",
-             "project_name": "{{ existing_user_project_2 }}",
-             "user_domain_name": "Default",
-             "project_domain_name": "Default"
-            }],
-        "auth_url": " {{ keystone_auth_url.stdout }}",
-        "region_name": "RegionOne"
-    }
-}

tests/ci/playbooks/run-rally-task.yaml

@@ -6,8 +6,6 @@
   vars:
     rally_home_dir: '/opt/stack/.rally'
     rally_fake_image_path: '{{ rally_home_dir }}/extra/fake-image.img'
-    rally_use_existing_users: false
-    rally_existing_users_config: '{{ rally_home_dir }}/with-existing-users-config'
     rally_results_dir: '{{ rally_home_dir }}/results'
     rally_resources_at_start: '{{ rally_results_dir }}/resources_at_start.txt'
     rally_task_args_file: "100-percent-not-exist-file"

tests/ci/playbooks/run-rally-tox.yaml

@@ -6,8 +6,6 @@
   vars:
     rally_home_dir: '/opt/stack/.rally'
     rally_fake_image_path: '{{ rally_home_dir }}/extra/fake-image.img'
-    rally_use_existing_users: false
-    rally_existing_users_config: '{{ rally_home_dir }}/with-existing-users-config'
     rally_task_args_file: "100-percent-not-exist-file"
     # this task will not be launched, but we need to specify something real to
     # pass a check at 'prepare-for-rally-task' role.