[ci] Temporary copy-paste docker jobs content from rally

openstack/rally project is not registered as a trusted place for Zuul.
It means that we cannot transmit secret and reuse docker job from there.

This is temporary workaround that should be removed asap to reduce code
duplication.

Change-Id: I19d8d40d72ed46d4bf576afab8e93cdbb9df9540

.zuul.d/docker-jobs.yaml

@@ -24,29 +24,38 @@
           pm95hJSPjVDW6LhujIKIWL21swKyeSrJGB9f3MlaYkiQn3vZ1Dy/7rE/fRN9a9Vur/KNU
           eh7c/tXVG3k0n2Y+7O6gMnLpT6lHM18jlbjyNfvEWLSfDVoRWG87L2BDeVTZtY=
 
+
 - job:
     name: rally-openstack-docker-build
-    parent: rally-docker-build
+    parent: build-docker-image
+    nodeset: ubuntu-bionic
+    run: tests/ci/playbooks/docker-build-and-check.yaml
+    post-run: tests/ci/playbooks/fetch-html-and-json-reports.yaml
+    timeout: 1800
     required-projects:
       - name: openstack/rally
       - name: openstack/rally-openstack
     vars:
-      docker_repository: "xrally/xrally-openstack"
       docker_image_version: latest
+      docker_repository: "xrally/xrally-openstack"
       rally_plugin_name: "Dummy.openstack"
       rally_package_name: "rally-openstack"
 
 - job:
     name: rally-openstack-docker-build-and-push
-    parent: rally-docker-build-and-push
+    parent: build-docker-image
+    nodeset: ubuntu-bionic
+    run: tests/ci/playbooks/docker-build-check-and-push.yaml
+    post-run: tests/ci/playbooks/fetch-html-and-json-reports.yaml
+    timeout: 1800
     required-projects:
-        - name: openstack/rally
-        - name: openstack/rally-openstack
+      - name: openstack/rally
+      - name: openstack/rally-openstack
+    secrets:
+      name: docker_credentials
+      secret: rally-openstack-dockerhub-credentials
+      pass-to-parent: false
     vars:
       docker_repository: "xrally/xrally-openstack"
       rally_plugin_name: "Dummy.openstack"
       rally_package_name: "rally-openstack"
-    secrets:
-        name: dockerhub_credentials
-        secret: rally-openstack-dockerhub-credentials
-        pass-to-parent: true

tests/ci/playbooks/docker-build-and-check.yaml

@@ -0,0 +1,4 @@
+- hosts: all
+  name: Build docker image for Rally and check it (calling the similar workload as `tox -e self`)
+  roles:
+    - docker-build-image

tests/ci/playbooks/docker-build-check-and-push.yaml

@@ -0,0 +1,5 @@
+- hosts: all
+  name: Build and push docker image for Rally
+  roles:
+    - docker-build-image
+    - docker-push-image

tests/ci/playbooks/fetch-html-and-json-reports.yaml

@@ -0,0 +1,42 @@
+- hosts: all
+  vars:
+    results_dir: "{{ zuul.project.src_dir }}/.test_results/"
+    html_report: "{{ tox_env | default('self') }}_report.html"
+    json_report: "{{ tox_env | default('self') }}_report.json"
+  tasks:
+    - shell: "ls {{ results_dir }}"
+      register: results_dir_stat
+      ignore_errors: True
+
+    - name: Save results
+      become: yes
+      when: results_dir_stat.rc == 0
+      synchronize:
+        src: "{{ results_dir }}"
+        dest: '{{ zuul.executor.log_root }}/'
+        mode: pull
+        copy_links: true
+        verify_host: true
+        rsync_opts:
+        - --include=/**
+        - --include=*/
+        - --exclude=*
+        - --prune-empty-dirs
+
+    - name: Return artifact to Zuul
+      when: html_report in results_dir_stat.stdout
+      zuul_return:
+        data:
+          zuul:
+            artifacts:
+              - name: "HTML report"
+                url: "{{ html_report }}"
+
+    - name: Return artifact to Zuul
+      when: json_report in results_dir_stat.stdout
+      zuul_return:
+        data:
+          zuul:
+            artifacts:
+              - name: "JSON report"
+                url: "{{ json_report }}"

tests/ci/playbooks/roles/docker-build-image/defaults/main.yaml

@@ -0,0 +1,6 @@
+docker_repository: xrally/xrally
+docker_image_version: "{{ zuul.tag }}"
+docker_image_tag: "{{ docker_repository }}:{{ docker_image_version }}"
+rally_project_path: "{{ zuul.projects['opendev.org/openstack/rally'].src_dir }}"
+rally_plugin_name: "Dummy.dummy"
+rally_package_name: ""

tests/ci/playbooks/roles/docker-build-image/tasks/main.yaml

@@ -0,0 +1,42 @@
+- name: "Build {{ docker_image_tag }} docker image"
+  shell:
+    chdir: '{{ zuul.project.src_dir }}'
+    cmd: 'docker build -t {{ docker_image_tag }} ./'
+
+- name: Get image size
+  shell: >
+    set -e
+    SIZE=$(docker image inspect {{ docker_image_tag }} {% raw %}--format='{{.Size}}'{% endraw %} )
+    SIZE=$(expr $SIZE / 1000 / 1000)
+    echo "Size of {{ docker_image_tag }}: "$SIZE" Mbyte"
+
+- name: Prepare directories to mount to the docker container for persistency
+  shell:
+    chdir: '{{ zuul.project.src_dir }}'
+    cmd: |
+      mkdir -p .test_results
+      sudo chown 65500 .test_results
+
+- name: List Rally version
+  shell: docker run {{ docker_image_tag }} --version
+  register: rally_version_info
+
+- name: Check plugin package is discoverable
+  fail:
+    msg: "{{ rally_package_name }} is not discoverable."
+  when:
+   - '"{{ rally_package_name }}" != ""'
+   - '"{{ rally_package_name }}" not in rally_version_info.stdout'
+
+- name: "Check availability of {{ rally_plugin_name }} plugin"
+  shell: docker run {{ docker_image_tag }} plugin show {{ rally_plugin_name }}
+
+- name: Execute the similar wokrloads as `tox -e self`
+  shell:
+    cmd: >
+      python3 {{ rally_project_path }}/tests/ci/rally_self_job.py
+      --task /rally/source/rally-jobs/self-rally.yaml
+      --plugins-path /rally/source/rally-jobs/plugins
+      --rally-cmd 'docker run -v '$(realpath {{ zuul.project.src_dir }})'/.test_results:/home/rally/.rally {{ docker_image_tag }}'
+      --results-dir /home/rally/.rally
+      --without-tmp-sqlite

tests/ci/playbooks/roles/docker-push-image/defaults/main.yaml

@@ -0,0 +1,4 @@
+docker_repository: xrally/xrally
+docker_image_version: latest
+docker_image_tag: "{{ docker_repository }}:{{ docker_image_version }}"
+docker_file: "{{ zuul.project.src_dir }}/DOCKER_README.md"

tests/ci/playbooks/roles/docker-push-image/tasks/main.yaml

@@ -0,0 +1,42 @@
+- name: Log in to dockerhub
+  command: "docker login -u '{{ docker_credentials.username }}' -p '{{ docker_credentials.password }}'"
+  no_log: true
+
+- name: Upload image to dockerhub
+  command: "docker push {{ docker_image_tag }}"
+  register: result
+  until: result.rc == 0
+  retries: 3
+  delay: 30
+
+- name: Read README file for docker
+  command: "cat {{ docker_file }}"
+  register: docker_readme
+
+- name: Get dockerhub JWT token
+  no_log: true
+  uri:
+    url: "https://hub.docker.com/v2/users/login/"
+    body_format: json
+    body:
+      username: '{{ docker_credentials.username }}'
+      password: '{{ docker_credentials.password }}'
+  register: jwt_token
+  delay: 5
+  retries: 3
+  until: jwt_token and jwt_token.status==200
+
+- name: Update README at Docker HUB
+  no_log: true
+  uri:
+    url: "https://hub.docker.com/v2/repositories/{{ docker_repository }}/"
+    method: "PATCH"
+    body_format: json
+    headers:
+      Authorization: "JWT {{ jwt_token.json.token }}"
+    body:
+      full_description: "{{ docker_readme.stdout }}"
+  register: repository_metadata
+  delay: 5
+  retries: 3
+  until: repository_metadata and repository_metadata.status==200