fetch-sphinx-tarball: introduce zuul_use_fetch_output

This change enables using fetch-sphinx-tarball role along with
the fetch-output role. By default the role still synchronizes
artifacts back to the executor.

Change-Id: I7452f34bccdca49e256053f9630f77953b27f99c

roles/fetch-sphinx-tarball/README.rst

@@ -23,3 +23,11 @@ archive into the log root for viewing.
    A list of file names of PDF files to collect.
    By default, the list contains as entry only
    ``doc-{{ zuul.project.short_name }}.pdf``.
+
+.. zuul:rolevar:: zuul_use_fetch_output
+   :default: false
+
+   Whether to synchronize files to the executor work dir, or to copy them
+   on the test instance.
+   When set to false, the role synchronizes the file to the executor.
+   When set to true, the job needs to use the fetch-output role later.

roles/fetch-sphinx-tarball/defaults/main.yaml

@@ -1,5 +1,7 @@
 ---
 zuul_work_dir: "{{ zuul.project.src_dir }}"
+zuul_output_dir: "{{ ansible_user_dir }}/zuul-output"
+zuul_use_fetch_output: "{{ zuul_site_use_fetch_output|default(false) }}"
 sphinx_build_dir: doc/build
 sphinx_pdf_files:
   - "doc-{{ zuul.project.short_name }}.pdf"

roles/fetch-sphinx-tarball/tasks/html.yaml

@@ -9,27 +9,50 @@
   args:
     warn: false
 
-- name: Fetch archive HTML
+- block:
-  synchronize:
+    - name: Fetch archive HTML
-    dest: "{{ zuul.executor.log_root }}/docs-html.tar.gz"
+      synchronize:
-    mode: pull
+        dest: "{{ zuul.executor.log_root }}/docs-html.tar.gz"
-    src: "{{ html_archive.path }}"
+        mode: pull
-    verify_host: true
+        src: "{{ html_archive.path }}"
+        verify_host: true
 
-- name: Create browseable HTML directory
+    - name: Create browseable HTML directory
-  delegate_to: localhost
+      delegate_to: localhost
-  file:
+      file:
-    path: "{{ zuul.executor.log_root }}/docs"
+        path: "{{ zuul.executor.log_root }}/docs"
-    state: directory
+        state: directory
 
-- name: Extract archive HTML
+    - name: Extract archive HTML
-  delegate_to: localhost
+      delegate_to: localhost
-  unarchive:
+      unarchive:
-    src: "{{ zuul.executor.log_root }}/docs-html.tar.gz"
+        src: "{{ zuul.executor.log_root }}/docs-html.tar.gz"
-    dest: "{{ zuul.executor.log_root }}/docs"
+        dest: "{{ zuul.executor.log_root }}/docs"
-    remote_src: true
+        remote_src: true
-    extra_opts:
+        extra_opts:
-      - "--no-same-owner"
+          - "--no-same-owner"
+  when: not zuul_use_fetch_output
+
+- block:
+    - name: Copy archive HTML
+      copy:
+        dest: "{{ zuul_output_dir }}/logs/docs-html.tar.gz"
+        src: "{{ html_archive.path }}"
+        remote_src: true
+
+    - name: Create browseable HTML directory
+      file:
+        path: "{{ zuul_output_dir }}/logs/docs"
+        state: directory
+
+    - name: Extract archive HTML
+      unarchive:
+        src: "{{ zuul_output_dir }}/logs/docs-html.tar.gz"
+        dest: "{{ zuul_output_dir }}/logs/docs"
+        remote_src: true
+        extra_opts:
+          - "--no-same-owner"
+  when: zuul_use_fetch_output
 
 - name: Return artifact to Zuul
   zuul_return:

roles/fetch-sphinx-tarball/tasks/pdf.yaml

@@ -28,9 +28,10 @@
 # Now loop...
 
 - name: Grab PDF files
-  when: pdf_files_found
+  when:
+    - pdf_files_found
+    - not zuul_use_fetch_output
   block:
-
     - name: Create PDF directory
       delegate_to: localhost
       file:
@@ -48,16 +49,38 @@
         loop_var: zj_pdf
       when: zj_pdf.stat.exists
 
-    - name: Return PDF artifact to Zuul
+- name: Copy PDF files
-      zuul_return:
+  when:
-        data:
+    - pdf_files_found
-          zuul:
+    - zuul_use_fetch_output
-            artifacts:
+  block:
-              - name: "Docs PDF: {{ zj_pdf.zj_sphinx_pdf }}"
+    - name: Create local PDF directory
-                url: "pdf/{{ zj_pdf.zj_sphinx_pdf }}"
+      file:
-                metadata:
+        path: "{{ zuul_output_dir }}/logs/pdf"
-                  type: docs_pdf
+        state: directory
+
+    - name: Copy PDF files
+      copy:
+        dest: "{{ zuul_output_dir }}/logs/pdf/{{ zj_pdf.zj_sphinx_pdf }}"
+        src: "{{ zj_pdf.stat.path }}"
+        remote_src: true
       with_items: "{{ pdf_file_stat.results }}"
       loop_control:
         loop_var: zj_pdf
       when: zj_pdf.stat.exists
+
+- name: Return PDF artifact to Zuul
+  zuul_return:
+    data:
+      zuul:
+        artifacts:
+          - name: "Docs PDF: {{ zj_pdf.zj_sphinx_pdf }}"
+            url: "pdf/{{ zj_pdf.zj_sphinx_pdf }}"
+            metadata:
+              type: docs_pdf
+  with_items: "{{ pdf_file_stat.results }}"
+  loop_control:
+    loop_var: zj_pdf
+  when:
+    - pdf_files_found
+    - zj_pdf.stat.exists

test-playbooks/python/fetch-sphinx-tarball.yaml

@@ -0,0 +1,45 @@
+- hosts: all
+  pre_tasks:
+    # Run ensure-output-dirs now as it is not performed speculatively
+    - import_role:
+        name: ensure-output-dirs
+
+    - name: Create fake sphinx output
+      shell: |
+        mkdir -p {{ zuul.project.src_dir }}/doc/build/pdf
+        mkdir -p {{ zuul.project.src_dir }}/doc/build/html
+        echo "%PDF-1.2" > {{ zuul.project.src_dir }}/doc/build/pdf/doc-{{ zuul.project.short_name }}.pdf
+        echo "<body>Hello</body>" > {{ zuul.project.src_dir }}/doc/build/html/index.html
+
+  tasks:
+    - import_role:
+        name: fetch-sphinx-tarball
+
+    - import_role:
+        name: fetch-output
+      when: zuul_use_fetch_output
+
+    - import_role:
+        name: merge-output-to-logs
+      when: zuul_use_fetch_output
+
+  post_tasks:
+    - name: Check for artifact on the test instance
+      stat:
+        path: "{{ ansible_user_dir }}/zuul-output/logs/{{ item }}"
+      register: _test_artifact
+      failed_when: not _test_artifact.stat.exists
+      with_items:
+        - "pdf/doc-{{ zuul.project.short_name }}.pdf"
+        - docs/index.html
+      when: zuul_use_fetch_output
+
+    - name: Check for artifact on the executor
+      stat:
+        path: "{{ zuul.executor.log_root }}/{{ item }}"
+      delegate_to: localhost
+      register: _executor_artifact
+      failed_when: not _executor_artifact.stat.exists
+      with_items:
+        - "pdf/doc-{{ zuul.project.short_name }}.pdf"
+        - docs/index.html

zuul-tests.d/python-jobs.yaml

@@ -478,6 +478,26 @@
     vars:
       zuul_use_fetch_output: false
 
+- job:
+    name: zuul-jobs-test-fetch-sphinx-tarball-with-zuul-output
+    description: Test the fetch-sphinx-tarball
+    files:
+      - roles/ensure-output-dirs/.*
+      - roles/fetch-sphinx-tarball/.*
+      - roles/fetch-output/.*
+    run: test-playbooks/python/fetch-sphinx-tarball.yaml
+    vars:
+      zuul_use_fetch_output: true
+
+- job:
+    name: zuul-jobs-test-fetch-sphinx-tarball-synchronize
+    description: Test the fetch-sphinx-tarball
+    files:
+      - roles/fetch-sphinx-tarball/.*
+    run: test-playbooks/python/fetch-sphinx-tarball.yaml
+    vars:
+      zuul_use_fetch_output: false
+
 - project:
     check:
       jobs:
@@ -525,6 +545,8 @@
         - zuul-jobs-test-fetch-subunit-output-synchronize
         - zuul-jobs-test-fetch-sphinx-output
         - zuul-jobs-test-fetch-sphinx-output-synchronize
+        - zuul-jobs-test-fetch-sphinx-tarball-with-zuul-output
+        - zuul-jobs-test-fetch-sphinx-tarball-synchronize
     gate:
       jobs:
         - zuul-jobs-test-ensure-pip-centos-7
@@ -568,3 +590,5 @@
         - zuul-jobs-test-fetch-subunit-output-synchronize
         - zuul-jobs-test-fetch-sphinx-output
         - zuul-jobs-test-fetch-sphinx-output-synchronize
+        - zuul-jobs-test-fetch-sphinx-tarball-with-zuul-output
+        - zuul-jobs-test-fetch-sphinx-tarball-synchronize