Allow IPA images to be built locally

This will be performed by kayobe overcloud post configure if ipa_build_images
is True.

ansible/group_vars/all/inspector

@@ -8,10 +8,10 @@ inspector_extra_kernel_options: >
      ipa_kernel_options_inspection_collectors_all }}
 
 # URL of Ironic Python Agent (IPA) kernel image.
-inspector_ipa_kernel_upstream_url: "{{ ipa_images_kernel_url }}"
+inspector_ipa_kernel_upstream_url: "{{ ipa_kernel_upstream_url }}"
 
 # URL of Ironic Python Agent (IPA) ramdisk image.
-inspector_ipa_ramdisk_upstream_url: "{{ ipa_images_ramdisk_url }}"
+inspector_ipa_ramdisk_upstream_url: "{{ ipa_ramdisk_upstream_url }}"
 
 ###############################################################################
 # Ironic inspector processing configuration.

ansible/group_vars/all/ipa

@@ -1,6 +1,25 @@
 ---
 # Ironic Python Agent (IPA) configuration.
 
+###############################################################################
+# Ironic Python Agent (IPA) image build configuration.
+
+# Whether to build IPA images from source.
+ipa_build_images: False
+
+# URL of IPA source repository.
+ipa_build_source_url: "https://github.com/openstack/ironic-python-agent"
+
+# Version of IPA source repository.
+ipa_build_source_version: "stable/ocata"
+
+# URL of IPA upper constraints file.
+ipa_build_upper_constraints_file_url:
+
+# Custom python package version constraints for IPA. Dict mapping package name
+# to upper version constraint.
+ipa_build_custom_upper_constraints: []
+
 ###############################################################################
 # Ironic Python Agent (IPA) images configuration.
 
@@ -8,13 +27,13 @@
 ipa_images_kernel_name: "ipa.vmlinuz"
 
 # URL of Ironic deployment kernel image to download.
-ipa_images_kernel_url: "https://tarballs.openstack.org/ironic-python-agent/coreos/files/coreos_production_pxe-stable-ocata.vmlinuz"
+ipa_kernel_upstream_url: "https://tarballs.openstack.org/ironic-python-agent/coreos/files/coreos_production_pxe-stable-ocata.vmlinuz"
 
 # Name of Ironic deployment ramdisk image to register in Glance.
 ipa_images_ramdisk_name: "ipa.initramfs"
 
 # URL of Ironic deployment ramdisk image to download.
-ipa_images_ramdisk_url: "https://tarballs.openstack.org/ironic-python-agent/coreos/files/coreos_production_pxe_image-oem-stable-ocata.cpio.gz"
+ipa_ramdisk_upstream_url: "https://tarballs.openstack.org/ironic-python-agent/coreos/files/coreos_production_pxe_image-oem-stable-ocata.cpio.gz"
 
 ###############################################################################
 # Ironic Python Agent (IPA) deployment configuration.

ansible/ipa-build.yml

@@ -0,0 +1,14 @@
+---
+- name: Ensure Ironic Python Agent images are built
+  hosts: controllers[0]
+  vars:
+    # This can be set to True to force rebuilding images.
+    ipa_build_force: False
+  roles:
+    - role: ipa-build
+      ipa_build_venv: "{{ ansible_env['PWD'] }}/ipa-build-venv"
+      ipa_build_image_cache_path: "{{ image_cache_path }}"
+      ipa_build_source_checkout_path: "{{ source_checkout_path }}"
+      ipa_build_kernel_name: "{{ ipa_images_kernel_name }}"
+      ipa_build_ramdisk_name: "{{ ipa_images_ramdisk_name }}"
+      when: "{{ ipa_build_images | bool }}"

ansible/ipa-images.yml

@@ -15,9 +15,35 @@
       tags:
         - config-validation
 
+    - block:
+        - name: Check for the presence of locally built Ironic Python Agent (IPA) images
+          stat:
+            path: "{{ image_cache_path }}/{{ item }}"
+            get_md5: False
+            get_checksum: False
+            mime: False
+          with_items:
+            - "{{ ipa_images_kernel_name }}"
+            - "{{ ipa_images_ramdisk_name }}"
+          register: ipa_image_stat
+
+        - name: Validate the presence of locally built Ironic Python Agent (IPA) images
+          fail:
+            msg: >
+              Expected locally built Ironic Python Agent (IPA) image
+              {{ item.item }} was not present in {{ image_cache_path }}.
+          with_items: "{{ ipa_image_stat.results }}"
+          when: "{{ not item.stat.exists }}"
+      when: "{{ ipa_build_images | bool }}"
+      tags:
+        - config-validation
+
   roles:
     - role: ipa-images
       ipa_images_venv: "{{ ansible_env['PWD'] }}/shade-venv"
       ipa_images_openstack_auth_type: "{{ openstack_auth_type }}"
       ipa_images_openstack_auth: "{{ openstack_auth }}"
       ipa_images_cache_path: "{{ image_cache_path }}"
+      # Don't pass the kernel and ramdisk image URLs if using built images.
+      ipa_images_kernel_url: "{{ ipa_build_images | ternary(None, ipa_kernel_upstream_url) }}"
+      ipa_images_ramdisk_url: "{{ ipa_build_images | ternary(None, ipa_ramdisk_upstream_url) }}"

ansible/roles/ipa-build/README.md

@@ -0,0 +1,64 @@
+Ironic Python Agent (IPA) Image Build
+=====================================
+
+This role can be used to build kernel and ramdisk images for OpenStack Ironic
+Python Agent (IPA).
+
+Requirements
+------------
+
+None
+
+Role Variables
+--------------
+
+`ipa_build_venv` is a path to a directory in which to create a virtualenv.
+
+`ipa_build_install_epel`: Whether to install EPEL repository package.
+
+`ipa_build_install_package_dependencies`: Whether to install package
+dependencies.
+
+`ipa_build_cache_path`: Path to directory in which to store built images.
+
+`ipa_build_source_url`: URL of IPA source repository.
+
+`ipa_build_source_version`: Version of IPA source repository.
+
+`ipa_build_upper_constraints_file_url`: URL of IPA upper constraints file.
+
+`ipa_build_custom_upper_constraints`: Custom python package version constraints
+for IPA. Dict mapping package name to upper version constraint.
+
+`ipa_build_kernel_name`: Name of kernel image to save.
+
+`ipa_build_ramdisk_name`: Name of ramdisk image to save.
+
+`ipa_build_force`: Whether to force rebuilding images when they already exist.
+
+Dependencies
+------------
+
+None
+
+Example Playbook
+----------------
+
+The following playbook installs openstackclient in a virtualenv.
+
+    ---
+    - name: Ensure Ironic Python Agent (IPA) images are built
+      hosts: localhost
+      roles:
+        - role: ipa-build
+          ipa_build_venv: "~/ipa-build-venv"
+          ipa_build_cache_path: "~/ipa-build-cache"
+          ipa_build_source_url: "https://github.com/openstack/ironic-python-agent"
+          ipa_build_source_version: "master"
+          ipa_build_kernel_name: "ipa.vmlinuz"
+          ipa_build_ramdisk_name: "ipa.initramfs"
+
+Author Information
+------------------
+
+- Mark Goddard (<mark@stackhpc.com>)

ansible/roles/ipa-build/defaults/main.yml

@@ -0,0 +1,37 @@
+---
+# Path to a directory in which to create a virtualenv.
+ipa_build_venv:
+
+# Path to a directory in which to clone the IPA source repository.
+ipa_build_source_checkout_path:
+
+# Whether to install EPEL repository package.
+ipa_build_install_epel: True
+
+# Whether to install package dependencies.
+ipa_build_install_package_dependencies: True
+
+# Path to directory in which to store built images.
+ipa_build_cache_path:
+
+# URL of IPA source repository.
+ipa_build_source_url:
+
+# Version of IPA source repository.
+ipa_build_source_version:
+
+# URL of IPA upper constraints file.
+ipa_build_upper_constraints_file_url:
+
+# Custom python package version constraints for IPA. Dict mapping package name
+# to upper version constraint.
+ipa_build_custom_upper_constraints:
+
+# Name of kernel image to save.
+ipa_build_kernel_name:
+
+# Name of ramdisk image to save.
+ipa_build_ramdisk_name:
+
+# Whether to force rebuilding images when they already exist.
+ipa_build_force: False

ansible/roles/ipa-build/tasks/main.yml

@@ -0,0 +1,104 @@
+---
+- name: Ensure EPEL repo is installed
+  yum:
+    name: epel-release
+    state: installed
+  become: True
+  when: "{{ ipa_build_install_epel | bool }}"
+
+- name: Ensure required packages are installed
+  yum:
+    name: "{{ item }}"
+    state: installed
+  become: True
+  with_items:
+    - gcc
+    - libffi-devel
+    - openssl-devel
+    - python-devel
+    - python-pip
+    - python-virtualenv
+  when: "{{ ipa_build_install_package_dependencies | bool }}"
+
+- name: Ensure source code checkout and image cache directories exist
+  file:
+    path: "{{ ipa_build_source_checkout_path }}"
+    state: directory
+    owner: "{{ ansible_user }}"
+    group: "{{ ansible_user }}"
+    recurse: True
+  with_items:
+    - "{{ ipa_build_source_checkout_path }}"
+    - "{{ ipa_build_image_cache_path }}"
+  become: True
+
+- name: Ensure Ironic Python Agent (IPA) source code checkout exists
+  git:
+    repo: "{{ ipa_build_source_url }}"
+    dest: "{{ ipa_build_source_checkout_path }}/ironic-python-agent"
+    version: "{{ ipa_build_source_version }}"
+
+- name: Ensure the latest version of pip is installed
+  pip:
+    name: "{{ item.name }}"
+    state: latest
+    virtualenv: "{{ ipa_build_venv or omit }}"
+  with_items:
+    - { name: pip }
+
+- name: Ensure required Python packages are installed
+  pip:
+    requirements: "{{ ipa_build_source_checkout_path }}/ironic-python-agent/requirements.txt"
+    state: present
+    virtualenv: "{{ ipa_build_venv or omit }}"
+
+- name: Ensure upper constraints file is downloaded
+  uri:
+    url: "{{ ipa_build_upper_constraints_file_url }}"
+    return_content: yes
+  when: "{{ ipa_build_upper_constraints_file_url != None }}"
+  register: upper_constraints
+
+- name: Ensure custom upper constraints are set
+  copy:
+    content: |
+      {% for line in upper_constraints.content.splitlines() %}
+      {% set package, _, version = line.partition('===') %}
+      {{ package }}==={{ ipa_build_custom_upper_constraints.get(package, version) }}
+      {% endfor %}
+    dest: "{{ ipa_build_source_checkout_path }}/ironic-python-agent/upper-constraints.txt"
+  when: "{{ ipa_build_upper_constraints_file_url != None }}"
+
+- name: Ensure existing Ironic Python Agent (IPA) images are removed when forcing rebuild
+  file:
+    path: "{{ ipa_build_source_checkout_path }}/ironic-python-agent/imagebuild/coreos/UPLOAD/{{ item }}"
+    state: absent
+  with_items:
+    - "coreos_production_pxe.vmlinuz"
+    - "coreos_production_pxe_image-oem.cpio.gz"
+  when: "{{ ipa_build_force | bool }}"
+
+- name: Ensure Ironic Python Agent (IPA) images are built
+  shell: >
+    {% if ipa_build_venv | bool %}
+    source {{ ipa_build_venv }}/bin/activate &&
+    {% endif %}
+    {% if ipa_build_upper_constraints_file_url is defined %}
+    export UPPER_CONSTRAINTS_FILE="{{ ipa_build_source_checkout_path }}/ironic-python-agent/upper-constraints.txt" &&
+    {% endif %}
+    make clean &&
+    make
+  args:
+    chdir: "{{ ipa_build_source_checkout_path }}/ironic-python-agent/imagebuild/coreos"
+    creates: "{{ ipa_build_source_checkout_path }}/ironic-python-agent/imagebuild/coreos/UPLOAD/coreos_production_pxe_image-oem.cpio.gz"
+
+- name: Ensure built Ironic Python Agent (IPA) images are cached
+  copy:
+    src: "{{ ipa_build_source_checkout_path }}/ironic-python-agent/imagebuild/coreos/UPLOAD/{{ item.src }}"
+    dest: "{{ ipa_build_image_cache_path }}/{{ item.dest }}"
+    remote_src: True
+  with_items:
+    - src: "coreos_production_pxe.vmlinuz"
+      dest: "{{ ipa_build_kernel_name }}"
+    - src: "coreos_production_pxe_image-oem.cpio.gz"
+      dest: "{{ ipa_build_ramdisk_name }}"

ansible/roles/ipa-images/defaults/main.yml

@@ -16,11 +16,13 @@ ipa_images_cache_path:
 # Name of Ironic deployment kernel image to register in Glance.
 ipa_images_kernel_name:
 
-# URL of Ironic deployment kernel image to download.
+# URL of Ironic deployment kernel image to download. If unset, an existing
+# image in ipa_images_cache_path will be used.
 ipa_images_kernel_url:
 
 # Name of Ironic deployment ramdisk image to register in Glance.
 ipa_images_ramdisk_name:
 
-# URL of Ironic deployment ramdisk image to download.
+# URL of Ironic deployment ramdisk image to download. If unset, an existing
+# image in ipa_images_cache_path will be used.
 ipa_images_ramdisk_url:

ansible/roles/ipa-images/tasks/main.yml

@@ -10,10 +10,24 @@
 - name: Ensure Ironic Python Agent (IPA) images are downloaded
   get_url:
     url: "{{ item }}"
-    dest: "{{ ipa_images_cache_path }}"
+    dest: "{{ ipa_images_cache_path }}/{{ item.filename }}"
   with_items:
-    - "{{ ipa_images_kernel_url }}"
+    - url: "{{ ipa_images_kernel_url }}"
-    - "{{ ipa_images_ramdisk_url }}"
+      filename: "{{ ipa_images_kernel_name }}"
+    - url: "{{ ipa_images_ramdisk_url }}"
+      filename: "{{ ipa_images_ramdisk_name }}"
+  when: "{{ item.url != None }}"
+
+- name: Compute the MD5 checksum of the Ironic Python Agent (IPA) images
+  stat:
+    path: "{{ ipa_images_cache_path }}/{{ item }}"
+    get_md5: True
+    get_checksum: False
+    mime: False
+  with_items:
+    - "{{ ipa_images_kernel_name }}"
+    - "{{ ipa_images_ramdisk_name }}"
+  register: ipa_images_checksum
 
 # Note that setting this via a play or task variable seems to not
 # evaluate the Jinja variable reference, so we use set_fact.
@@ -21,6 +35,48 @@
   set_fact:
     ansible_python_interpreter: "{{ ipa_images_venv }}/bin/python"
 
+# To support updating the IPA image, we check the MD5 sum of the cached image
+# files, and compare with the images in Glance (if there are any).
+
+- name: Gather facts about Ironic Python Agent (IPA) kernel image
+  os_image_facts:
+    auth_type: "{{ ipa_images_openstack_auth_type }}"
+    auth: "{{ ipa_images_openstack_auth }}"
+    image: "{{ ipa_images_kernel_name }}"
+
+- name: Set a fact containing the Ironic Python Agent (IPA) kernel image checksum
+  set_fact:
+    ipa_images_kernel_checksum: "{{ openstack_image.checksum }}"
+  when: "{{ openstack_image != None }}"
+
+- name: Gather facts about Ironic Python Agent (IPA) ramdisk image
+  os_image_facts:
+    auth_type: "{{ ipa_images_openstack_auth_type }}"
+    auth: "{{ ipa_images_openstack_auth }}"
+    image: "{{ ipa_images_ramdisk_name }}"
+
+- name: Set a fact containing the Ironic Python Agent (IPA) ramdisk image checksum
+  set_fact:
+    ipa_images_ramdisk_checksum: "{{ openstack_image.checksum }}"
+  when: "{{ openstack_image != None }}"
+
+- name: Ensure Ironic Python Agent (IPA) images are removed from Glance
+  os_image:
+    auth_type: "{{ ipa_images_openstack_auth_type }}"
+    auth: "{{ ipa_images_openstack_auth }}"
+    name: "{{ item.name }}"
+    state: absent
+  with_items:
+    - name: "{{ ipa_images_kernel_name }}"
+      checksum: "{{ ipa_images_checksum.results[0].stat.md5 }}"
+      glance_checksum: "{{ ipa_images_kernel_checksum | default }}"
+    - name: "{{ ipa_images_ramdisk_name }}"
+      checksum: "{{ ipa_images_checksum.results[1].stat.md5 }}"
+      glance_checksum: "{{ ipa_images_ramdisk_checksum | default }}"
+  when:
+    - "{{ item.glance_checksum != None }}"
+    - "{{ item.checksum != item.glance_checksum }}"
+
 - name: Ensure Ironic Python Agent (IPA) images are registered with Glance
   os_image:
     auth_type: "{{ ipa_images_openstack_auth_type }}"
@@ -29,13 +85,11 @@
     container_format: "{{ item.format }}"
     disk_format: "{{ item.format }}"
     state: present
-    filename: "{{ ipa_images_cache_path }}/{{ item.filename }}"
+    filename: "{{ ipa_images_cache_path }}/{{ item.name }}"
   with_items:
     - name: "{{ ipa_images_kernel_name }}"
-      filename: "{{ ipa_images_kernel_url | basename }}"
       format: aki
     - name: "{{ ipa_images_ramdisk_name }}"
-      filename: "{{ ipa_images_ramdisk_url | basename }}"
       format: ari
 
 # This variable is unset before we set it, and it does not appear to be

etc/kayobe/ipa.yml

@@ -1,6 +1,25 @@
 ---
 # Ironic Python Agent (IPA) configuration.
 
+###############################################################################
+# Ironic Python Agent (IPA) image build configuration.
+
+# Whether to build IPA images from source.
+#ipa_build_images:
+
+# URL of IPA source repository.
+#ipa_build_source_url:
+
+# Version of IPA source repository.
+#ipa_build_source_version:
+
+# URL of IPA upper constraints file.
+#ipa_build_upper_constraints_file_url:
+
+# Custom python package version constraints for IPA. Dict mapping package name
+# to upper version constraint.
+#ipa_build_custom_upper_constraints:
+
 ###############################################################################
 # Ironic Python Agent (IPA) images configuration.
 
@@ -8,13 +27,13 @@
 #ipa_images_kernel_name:
 
 # URL of Ironic deployment kernel image to download.
-#ipa_images_kernel_url:
+#ipa_kernel_upstream_url:
 
 # Name of Ironic deployment ramdisk image to register in Glance.
 #ipa_images_ramdisk_name:
 
 # URL of Ironic deployment ramdisk image to download.
-#ipa_images_ramdisk_url:
+#ipa_ramdisk_upstream_url:
 
 ###############################################################################
 # Ironic Python Agent (IPA) deployment configuration.

kayobe/cli/commands.py

@@ -465,7 +465,7 @@ class OvercloudPostConfigure(KayobeAnsibleMixin, Command):
     def take_action(self, parsed_args):
         self.app.LOG.debug("Performing post-deployment configuration")
         playbooks = _build_playbook_list(
-            "ipa-images", "overcloud-introspection-rules",
+            "ipa-build", "ipa-images", "overcloud-introspection-rules",
             "overcloud-introspection-rules-dell-lldp-workaround",
             "provision-net")
         self.run_kayobe_playbooks(parsed_args, playbooks)