Import ansible version of the image building job

This patch starts "ansiblization" of the image building script, while
also cleaning it up. More importantly, it adds check and gate jobs
that verify the same playbooks as used in the post job.

Also correct source-repository for ironic-python-agent to use the
project name, not the old element name.

Change-Id: I9a404b24f95e47c84b22d4739fcf2b5b800886f9

.zuul.yaml

@@ -1,3 +1,58 @@
+- job:
+    name: ironic-python-agent-build-image-base
+    parent: publish-openstack-artifacts
+    pre-run: playbooks/ironic-python-agent-build-image/pre.yaml
+    run: playbooks/ironic-python-agent-build-image/run.yaml
+    post-run: playbooks/ironic-python-build-image/post.yaml
+    timeout: 1800
+    required-projects:
+      - openstack/requirements
+      - openstack/ironic-python-agent
+      - openstack/ironic-python-agent-builder
+
+- job:
+    name: ironic-python-agent-build-image-tinyipa
+    parent: ironic-python-agent-build-image-base
+    vars:
+      image_type: 'tinyipa'
+
+- job:
+    name: ironic-python-agent-build-image-dib-centos7
+    parent: ironic-python-agent-build-image-base
+    required-projects:
+      # NOTE(dtantsur): used for bindep only
+      - openstack/diskimage-builder
+    vars:
+      image_type: 'dib'
+      image_distro: 'centos7'
+
+- job:
+    name: ironic-python-agent-check-image-base
+    parent: base
+    pre-run: playbooks/ironic-python-agent-build-image/pre.yaml
+    run: playbooks/ironic-python-agent-build-image/run.yaml
+    timeout: 1800
+    required-projects:
+      - openstack/requirements
+      - openstack/ironic-python-agent
+      - openstack/ironic-python-agent-builder
+
+- job:
+    name: ironic-python-agent-check-image-tinyipa
+    parent: ironic-python-agent-check-image-base
+    vars:
+      image_type: 'tinyipa'
+
+- job:
+    name: ironic-python-agent-check-image-dib-centos7
+    parent: ironic-python-agent-check-image-base
+    required-projects:
+      # NOTE(dtantsur): used for bindep only
+      - openstack/diskimage-builder
+    vars:
+      image_type: 'dib'
+      image_distro: 'centos7'
+
 - project:
     templates:
       - publish-openstack-docs-pti
@@ -11,6 +66,8 @@
         - ipa-tempest-wholedisk-bios-ipmi-direct-tinyipa-src
         - ipa-tempest-wholedisk-bios-ipmi-iscsi-tinyipa-src
         - ipa-tempest-ironic-inspector-src
+        - ironic-python-agent-check-image-tinyipa
+        - ironic-python-agent-check-image-dib-centos7
     gate:
       jobs:
         - openstack-tox-pep8
@@ -19,13 +76,10 @@
         - ipa-tempest-wholedisk-bios-ipmi-direct-tinyipa-src
         - ipa-tempest-wholedisk-bios-ipmi-iscsi-tinyipa-src
         - ipa-tempest-ironic-inspector-src
+        - ironic-python-agent-check-image-tinyipa
+        - ironic-python-agent-check-image-dib-centos7
     post:
       jobs:
         - publish-openstack-python-branch-tarball
-        - ipa-builder-buildimage-tinyipa
-
-- job:
-    name: ipa-builder-buildimage-tinyipa
-    parent: ironic-python-agent-buildimage-base
-    vars:
-      image_name: 'tinyipa'
+        - ironic-python-agent-build-image-tinyipa
+        - ironic-python-agent-build-image-dib-centos7

dib/ironic-python-agent-ramdisk/install.d/ironic-agent-source-install/60-ironic-agent-install

@@ -7,7 +7,7 @@ set -eu
 set -o pipefail
 
 SCRIPTDIR=$(dirname $0)
-IPADIR=/usr/share/ironic-python-agent
+IPADIR=/tmp/ironic-python-agent
 
 # Generate upper-constraints
 $IPADIR/imagebuild/common/generate_upper_constraints.sh $IPADIR/upper-constraints.txt

dib/ironic-python-agent-ramdisk/source-repository-ironic-agent

@@ -1 +0,0 @@
-ironic-agent git /usr/share/ironic-python-agent https://opendev.org/openstack/ironic-python-agent

dib/ironic-python-agent-ramdisk/source-repository-ironic-python-agent

@@ -0,0 +1 @@
+ironic-python-agent git /tmp/ironic-python-agent https://opendev.org/openstack/ironic-python-agent

playbooks/ironic-python-agent-build-image/post.yaml

@@ -0,0 +1,47 @@
+- hosts: all
+
+  vars:
+    ipa_raw_dir: '{{ ansible_user_dir }}/src/opendev.org/openstack/ironic-python-agent/UPLOAD_RAW'
+    ipa_tar_dir: '{{ ansible_user_dir }}/src/opendev.org/openstack/ironic-python-agent/UPLOAD_TAR'
+
+  tasks:
+
+    - name: Ensure artifacts directory exists
+      file:
+        path: '{{ zuul.executor.work_root }}/artifacts/{{ image_name }}/files'
+        state: directory
+      delegate_to: localhost
+
+    - name: Copy files from {{ ansible_user_dir }}/src/opendev.org/openstack/ironic-python-agent on node
+      synchronize:
+        src: '{{ ipa_raw_dir }}'
+        dest: '{{ zuul.executor.work_root }}/artifacts/{{ image_name }}/files/'
+        mode: pull
+        copy_links: true
+        verify_host: true
+        rsync_opts:
+          - --include=/*
+          - --prune-empty-dirs
+
+    - name: Copy files from {{ ansible_user_dir }}/src/opendev.org/openstack/ironic-python-agent on node
+      synchronize:
+        src: '{{ ipa_tar_dir }}'
+        dest: '{{ zuul.executor.work_root }}/artifacts/{{ image_name }}/'
+        mode: pull
+        copy_links: true
+        verify_host: true
+        rsync_opts:
+          - --include=/*
+          - --prune-empty-dirs
+
+    - name: Copy files from {{ ansible_user_dir }}/src/opendev.org/openstack/ironic-python-agent on node
+      synchronize:
+        src: '{{ ansible_user_dir }}/src/opendev.org/openstack/ironic-python-agent/'
+        dest: '{{ zuul.executor.log_root }}'
+        mode: pull
+        copy_links: true
+        verify_host: true
+        rsync_opts:
+          - --include=*/
+          - --exclude=*
+          - --prune-empty-dirs

playbooks/ironic-python-agent-build-image/pre.yaml

@@ -0,0 +1,7 @@
+- hosts: all
+  tasks:
+
+    - import_role:
+        name: ipa-build-dib-image
+        tasks_from: install.yaml
+      when: image_type == 'dib'

playbooks/ironic-python-agent-build-image/run.yaml

@@ -0,0 +1,25 @@
+- hosts: all
+
+  vars:
+    ipa_raw_dir: '{{ ansible_user_dir }}/src/opendev.org/openstack/ironic-python-agent/UPLOAD_RAW'
+    ipa_tar_dir: '{{ ansible_user_dir }}/src/opendev.org/openstack/ironic-python-agent/UPLOAD_TAR'
+
+  tasks:
+
+    - name: Create a target directory for raw images
+      file:
+        path: '{{ ipa_raw_dir }}'
+        state: directory
+
+    - name: Create a target directory for tarballs
+      file:
+        path: '{{ ipa_tar_dir }}'
+        state: directory
+
+    - import_role:
+        name: ipa-build-tinyipa-image
+      when: image_type == 'tinyipa'
+
+    - import_role:
+        name: ipa-build-dib-image
+      when: image_type == 'dib'

roles/ipa-build-dib-image/defaults/main.yaml

@@ -0,0 +1,5 @@
+image_distro: centos7
+ipa_branch_path: '{{ zuul.branch | replace("/", "-") }}'
+ipa_source_path: '{{ ansible_user_dir }}/src/opendev.org/openstack/ironic-python-agent'
+ipa_raw_dir: '{{ ansible_user_dir }}/src/opendev.org/openstack/ironic-python-agent/UPLOAD_RAW'
+ipa_tar_dir: '{{ ansible_user_dir }}/src/opendev.org/openstack/ironic-python-agent/UPLOAD_TAR'

roles/ipa-build-dib-image/tasks/install.yaml

@@ -0,0 +1,16 @@
+- name: Install binary dependencies from diskimage-builder
+  include_role:
+    name: bindep
+  vars:
+    bindep_dir: "{{ ansible_user_dir }}/{{ zuul.projects['opendev.org/openstack/diskimage-builder'].src_dir }}"
+
+- name: Install requirements for ironic-python-agent-builder
+  pip:
+    requirements: "{{ ansible_user_dir }}/{{ zuul.projects['opendev.org/openstack/ironic-python-agent-builder'].src_dir }}/requirements.txt"
+    extra_args: -c "{{ ansible_user_dir }}/{{ zuul.projects['opendev.org/openstack/requirements'].src_dir }}/upper-constraints.txt"
+  become: true
+
+- name: Install ironic-python-agent-builder
+  pip:
+    name: "{{ ansible_user_dir }}/{{ zuul.projects['opendev.org/openstack/ironic-python-agent-builder'].src_dir }}"
+  become: true

roles/ipa-build-dib-image/tasks/main.yaml

@@ -0,0 +1,16 @@
+- name: Generate image name
+  set_fact:
+    image_name: ipa-{{ image_distro }}-{{ ipa_branch_path }}
+
+- name: Build a DIB image
+  command: |
+    ironic-python-agent-builder -e dhcp-all-interfaces -o {{ image_name }} {{ image_distro }}
+  environment:
+    # Use repositories checked out by Zuul
+    DIB_REPOLOCATION_ironic_python_agent: '{{ ipa_source_path }}'
+    DIB_REPOREF_ironic_python_agent: HEAD
+
+- name: Move the resulting files
+  shell: |
+    tar -czf "{{ ipa_tar_dir }}/{{ image_name }}.tar.gz" {{ image_name }}*
+    mv {{ image_name }}* "{{ ipa_raw_dir }}"

roles/ipa-build-tinyipa-image/defaults/main.yaml

@@ -0,0 +1,4 @@
+ipa_branch_path: '{{ zuul.branch | replace("/", "-") }}'
+ipa_raw_dir: '{{ ansible_user_dir }}/src/opendev.org/openstack/ironic-python-agent/UPLOAD_RAW'
+ipa_tar_dir: '{{ ansible_user_dir }}/src/opendev.org/openstack/ironic-python-agent/UPLOAD_TAR'
+tinyipa_dir: '{{ ansible_user_dir }}/src/opendev.org/openstack/ironic-python-agent-builder/tinyipa'

roles/ipa-build-tinyipa-image/tasks/main.yaml

@@ -0,0 +1,23 @@
+- name: Build a TinyIPA image
+  command: make
+  args:
+    chdir: '{{ tinyipa_dir }}'
+  environment:
+    BRANCH_PATH: '{{ ipa_branch_path }}'
+    BUILD_AND_INSTALL_TINYIPA: True
+    IPA_SOURCE_DIR: '{{ ansible_user_dir }}/src/opendev.org/openstack/ironic-python-agent'
+
+- name: Move resulting files
+  shell: |
+    mv tinyipa*.tar.gz* "{{ ipa_tar_dir }}"
+    mv tinyipa*.* "{{ ipa_raw_dir }}"
+    mv tiny-instance-uec*.tar.gz "{{ ipa_tar_dir }}"
+  args:
+    chdir: '{{ tinyipa_dir }}'
+
+# NOTE(sambetts) Must clean up chroot directories before
+# publisher rsync command tries and fails to read them
+- name: Clean up the environment
+  command: make clean
+  args:
+    chdir: '{{ tinyipa_dir }}'