Add tinyipa to IPA imagebuild directory

TinyCoreLinux is a small 10mb base linux distribution which can easily
be built into a ramdisk for PXE and ISO booting. This patch adds scripts
and other required files to build a TinyCore based ramdisk which runs
the Ironic Python Agent.

Change-Id: I79fbec75de19b3d6feb19f87a3a0f662eae42bf3

imagebuild/README.rst

@@ -3,3 +3,6 @@ ironic-python-agent images
 
 coreos - Builds a CoreOS Ramdisk and Kernel suitable for running
 ironic-python-agent
+
+tinyipa - Builds a TinyCoreLinux Ramdisk and Kernel suitable for running
+ironic-python-agent

imagebuild/tinyipa/.gitignore

@@ -0,0 +1,12 @@
+*.gz
+*.zip
+*.tar.gz
+tinyipabuild
+tinyipafinal
+*.vmlinuz
+vmlinuz64
+*.iso
+*.tcz*
+squashfs-root
+syslinux*
+newiso

imagebuild/tinyipa/Makefile

@@ -0,0 +1,31 @@
+default: build finalise
+
+all: build finalise iso
+
+build:
+	./build-tinyipa.sh
+
+finalise:
+	./finalise-tinyipa.sh
+
+iso:
+	./build-iso.sh
+
+clean: clean_build clean_iso
+
+clean_build:
+	sudo -v
+	sudo rm -rf tinyipabuild
+	sudo rm -rf tinyipafinal
+	rm -f tinyipa.vmlinuz
+	rm -f tinyipa.gz
+	rm -f build_files/corepure64.gz
+	rm -f build_files/vmlinuz64
+	rm -f build_files/*.tcz
+	rm -f build_files/*.tcz.*
+
+clean_iso:
+	rm -rf newiso
+	rm -f build_files/syslinux-4.06.tar.gz
+	rm -rf build_files/syslinux-4.06
+	rm -f tinyipa.iso

imagebuild/tinyipa/README.rst

@@ -0,0 +1,81 @@
+=============================
+Tiny Core Ironic Python Agent
+=============================
+
+.. WARNING::
+  This is experimental! Build tested on Ubuntu Server 14.04 and Debian Jessie
+
+Build script requirements
+-------------------------
+For the main build script:
+
+* wget
+* pip
+* unzip
+* sudo
+* awk
+
+For building an ISO you'll also need:
+
+* genisoimage
+
+Instructions:
+-------------
+To create a new ramdisk, run::
+
+  make
+
+or::
+
+  ./build-tinyipa.sh && ./finalise-tinyipa.sh
+
+This will create two new files once completed:
+
+* tinyipa.vmlinuz
+* tinyipa.gz
+
+These are your two files to upload to glance for use with Ironic.
+
+Building an ISO from a previous make run:
+-----------------------------------------
+Once you've built tinyipa it is possible to pack it into an ISO if required. To
+create a bootable ISO, run::
+
+  make iso
+
+or::
+
+./build-iso.sh
+
+This will create one new file once completed:
+
+* tinyipa.iso
+
+To build a fresh ramdisk and build an iso from it:
+--------------------------------------------------
+Run::
+
+  make all
+
+To clean up the whole build environment run:
+--------------------------------------------
+Run::
+
+  make clean
+
+For cleaning up just the iso or just the ramdisk build::
+
+  make clean_iso
+
+or::
+
+  make clean_tinyipa
+
+Advanced options
+----------------
+
+If you want the build script to preinstall everything into the ramdisk,
+instead of loading some things at runtime (this results in a slightly bigger
+ramdisk), before running make or build-tinyipa.sh run::
+
+  export BUILD_AND_INSTALL_TINYIPA=true

imagebuild/tinyipa/build-iso.sh

@@ -0,0 +1,16 @@
+#!/bin/sh
+
+set -ex
+WORKDIR=$(realpath $0 | xargs dirname)
+
+cd $WORKDIR/build_files
+wget -N https://www.kernel.org/pub/linux/utils/boot/syslinux/syslinux-4.06.tar.gz && tar zxf syslinux-4.06.tar.gz
+
+cd $WORKDIR
+rm -rf newiso
+mkdir -p newiso/boot/isolinux
+cp build_files/syslinux-4.06/core/isolinux.bin newiso/boot/isolinux/.
+cp build_files/isolinux.cfg newiso/boot/isolinux/.
+cp tinyipa.gz newiso/boot/corepure64.gz
+cp tinyipa.vmlinuz newiso/boot/vmlinuz64
+genisoimage -l -r -J -R -V TC-custom -no-emul-boot -boot-load-size 4 -boot-info-table -b boot/isolinux/isolinux.bin -c boot/isolinux/boot.cat -o tinyipa.iso newiso

imagebuild/tinyipa/build-tinyipa.sh

@@ -0,0 +1,84 @@
+#!/bin/sh
+
+set -ex
+WORKDIR=$(readlink -f $0 | xargs dirname)
+BUILDDIR="$WORKDIR/tinyipabuild"
+BUILD_AND_INSTALL_TINYIPA=${BUILD_AND_INSTALL_TINYIPA:-false}
+
+CHROOT_PATH="/usr/local/sbin:/usr/local/bin:/apps/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+CHROOT_CMD="sudo chroot $BUILDDIR /usr/bin/env PATH=$CHROOT_PATH"
+
+# Ensure we have an extended sudo to prevent the need to enter a password over
+# and over again.
+sudo -v
+
+# If an old build directory exists remove it
+if [ -d "$BUILDDIR" ]; then
+    sudo rm -rf "$BUILDDIR"
+fi
+
+##############################################
+# Download and Cache Tiny Core Files
+##############################################
+
+cd $WORKDIR/build_files
+wget -N http://distro.ibiblio.org/tinycorelinux/6.x/x86_64/release/distribution_files/corepure64.gz
+wget -N http://distro.ibiblio.org/tinycorelinux/6.x/x86_64/release/distribution_files/vmlinuz64
+cd $WORKDIR
+
+########################################################
+# Build Required Python Dependecies in a Build Directory
+########################################################
+
+# Make directory for building in
+mkdir "$BUILDDIR"
+
+# Extract rootfs from .gz file
+( cd "$BUILDDIR" && zcat $WORKDIR/build_files/corepure64.gz | sudo cpio -i -H newc -d )
+
+# Download get-pip into ramdisk
+( cd "$BUILDDIR/tmp" && wget https://bootstrap.pypa.io/get-pip.py )
+
+# Download TGT and Qemu-utils source
+git clone https://github.com/fujita/tgt.git $BUILDDIR/tmp/tgt --depth=1 --branch v1.0.62
+git clone git://git.qemu-project.org/qemu.git $BUILDDIR/tmp/qemu --depth=1 --branch v2.5.0
+
+# Create directory for python local mirror
+mkdir -p "$BUILDDIR/tmp/localpip"
+
+# Download IPA and requirements
+cd ../..
+rm -rf *.egg-info
+python setup.py sdist --dist-dir "$BUILDDIR/tmp/localpip" --quiet
+cd $WORKDIR
+
+sudo cp /etc/resolv.conf $BUILDDIR/etc/resolv.conf
+sudo mount --bind /proc $BUILDDIR/proc
+$CHROOT_CMD mkdir /etc/sysconfig/tcedir
+$CHROOT_CMD chmod a+rwx /etc/sysconfig/tcedir
+$CHROOT_CMD touch /etc/sysconfig/tcuser
+$CHROOT_CMD chmod a+rwx /etc/sysconfig/tcuser
+
+while read line; do
+    sudo chroot --userspec=tc:staff $BUILDDIR /usr/bin/env PATH=$CHROOT_PATH tce-load -wci $line
+done < $WORKDIR/build_files/buildreqs.lst
+
+sudo umount $BUILDDIR/proc
+
+# Build python wheels
+$CHROOT_CMD python /tmp/get-pip.py
+$CHROOT_CMD pip install pbr
+$CHROOT_CMD pip wheel --wheel-dir /tmp/wheels setuptools
+$CHROOT_CMD pip wheel --wheel-dir /tmp/wheels pip
+$CHROOT_CMD pip wheel --pre --wheel-dir /tmp/wheels --find-links=/tmp/localpip ironic-python-agent
+
+# Build tgt
+rm -rf $WORKDIR/build_files/tgt.tcz
+$CHROOT_CMD /bin/sh -c "cd /tmp/tgt && make && make install-programs install-conf install-scripts DESTDIR=/tmp/tgt-installed"
+cd $WORKDIR/build_files && mksquashfs $BUILDDIR/tmp/tgt-installed tgt.tcz && md5sum tgt.tcz > tgt.tcz.md5.txt
+# Build qemu-utils
+rm -rf $WORKDIR/build_files/qemu-utils.tcz
+$CHROOT_CMD /bin/sh -c "cd /tmp/qemu && ./configure --disable-system --disable-user --disable-linux-user --disable-bsd-user --disable-guest-agent && make && make install DESTDIR=/tmp/qemu-utils"
+cd $WORKDIR/build_files && mksquashfs $BUILDDIR/tmp/qemu-utils qemu-utils.tcz && md5sum qemu-utils.tcz > qemu-utils.tcz.md5.txt
+# Create qemu-utils.tcz.dep
+echo "glib2.tcz" > qemu-utils.tcz.dep

imagebuild/tinyipa/build_files/bootlocal.sh

@@ -0,0 +1,21 @@
+#!/bin/sh
+# put other system startup commands here
+
+#exec > /tmp/installlogs 2>&1
+set -x
+
+echo "Starting bootlocal script:"
+date
+
+export HOME=/root
+
+# Maybe save some RAM?
+#rm -rf /tmp/builtin
+
+# Install IPA and dependecies
+if ! type "ironic-python-agent" > /dev/null ; then
+  python /tmp/get-pip.py --no-wheel --no-index --find-links=file:///tmp/wheelhouse ironic_python_agent
+fi
+
+# Run IPA
+ironic-python-agent

imagebuild/tinyipa/build_files/buildreqs.lst

@@ -0,0 +1,18 @@
+bash.tcz
+compiletc.tcz
+coreutils.tcz
+dmidecode.tcz
+gdisk.tcz
+hdparm.tcz
+parted.tcz
+python.tcz
+python-dev.tcz
+raid-dm-3.16.6-tinycore64.tcz
+scsi-3.16.6-tinycore64.tcz
+udev-lib.tcz
+util-linux.tcz
+pkg-config.tcz
+zlib_base-dev.tcz
+glib2-dev.tcz
+pixman-dev.tcz
+binutils.tcz

imagebuild/tinyipa/build_files/finalreqs.lst

@@ -0,0 +1,12 @@
+bash.tcz
+coreutils.tcz
+dmidecode.tcz
+gdisk.tcz
+hdparm.tcz
+parted.tcz
+python.tcz
+raid-dm-3.16.6-tinycore64.tcz
+scsi-3.16.6-tinycore64.tcz
+udev-lib.tcz
+util-linux.tcz
+glib2.tcz

imagebuild/tinyipa/build_files/isolinux.cfg

@@ -0,0 +1,9 @@
+display boot.msg
+default corepure64
+
+label corepure64
+        kernel /boot/vmlinuz64
+        initrd /boot/corepure64.gz
+        append loglevel=3 syslog showapps
+
+implicit 0

imagebuild/tinyipa/finalise-tinyipa.sh

@@ -0,0 +1,87 @@
+#!/bin/sh
+
+set -ex
+WORKDIR=$(readlink -f $0 | xargs dirname)
+BUILDDIR="$WORKDIR/tinyipabuild"
+FINALDIR="$WORKDIR/tinyipafinal"
+BUILD_AND_INSTALL_TINYIPA=${BUILD_AND_INSTALL_TINYIPA:-false}
+
+CHROOT_PATH="/usr/local/sbin:/usr/local/bin:/apps/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+CHROOT_CMD="sudo chroot $FINALDIR /usr/bin/env PATH=$CHROOT_PATH"
+TC_CHROOT_CMD="sudo chroot --userspec=tc:staff $FINALDIR /usr/bin/env PATH=$CHROOT_PATH"
+
+sudo -v
+
+if [ -d "$FINALDIR" ]; then
+    sudo rm -rf "$FINALDIR"
+fi
+
+mkdir "$FINALDIR"
+
+# Extract rootfs from .gz file
+( cd "$FINALDIR" && zcat $WORKDIR/build_files/corepure64.gz | sudo cpio -i -H newc -d )
+
+# Download get-pip into ramdisk
+( cd "$FINALDIR/tmp" && wget https://bootstrap.pypa.io/get-pip.py )
+
+#####################################
+# Setup Final Dir
+#####################################
+
+sudo cp /etc/resolv.conf $FINALDIR/etc/resolv.conf.old
+sudo cp /etc/resolv.conf $FINALDIR/etc/resolv.conf
+
+# Modify ldconfig for x86-64
+$CHROOT_CMD cp /sbin/ldconfig /sbin/ldconfigold
+printf '/sbin/ldconfigold $@ | sed "s/unknown/libc6,x86-64/"' | $CHROOT_CMD tee -a /sbin/ldconfignew
+$CHROOT_CMD cp /sbin/ldconfignew /sbin/ldconfig
+$CHROOT_CMD chmod u+x /sbin/ldconfig
+
+# Copy python wheels from build to final dir
+cp -Rp "$BUILDDIR/tmp/wheels" "$FINALDIR/tmp/wheelhouse"
+
+mkdir -p $FINALDIR/tmp/builtin/optional
+$CHROOT_CMD chown -R tc.staff /tmp/builtin
+$CHROOT_CMD chmod -R a+w /tmp/builtin
+$CHROOT_CMD ln -sf /tmp/builtin /etc/sysconfig/tcedir
+echo "tc" | $CHROOT_CMD tee -a /etc/sysconfig/tcuser
+
+cp $WORKDIR/build_files/tgt.* $FINALDIR/tmp/builtin/optional
+cp $WORKDIR/build_files/qemu-utils.* $FINALDIR/tmp/builtin/optional
+
+# Mount /proc for chroot commands
+sudo mount --bind /proc $FINALDIR/proc
+
+while read line; do
+    $TC_CHROOT_CMD tce-load -wi $line
+done < $WORKDIR/build_files/finalreqs.lst
+
+echo "tgt.tcz" | $TC_CHROOT_CMD tee -a /tmp/builtin/onboot.lst
+echo "qemu-utils.tcz" | $TC_CHROOT_CMD tee -a /tmp/builtin/onboot.lst
+
+# If flag is set install the python now
+if $BUILD_AND_INSTALL_TINYIPA ; then
+    $CHROOT_CMD python /tmp/get-pip.py --no-wheel --no-index --find-links=file:///tmp/wheelhouse ironic_python_agent
+    rm -rf $FINALDIR/tmp/wheelhouse
+fi
+
+# Unmount /proc and clean up everything
+sudo umount $FINALDIR/proc
+sudo umount $FINALDIR/tmp/tcloop/*
+sudo rm -rf $FINALDIR/tmp/tcloop
+sudo rm -rf $FINALDIR/usr/local/tce.installed
+sudo mv $FINALDIR/etc/resolv.conf.old $FINALDIR/etc/resolv.conf
+sudo rm $FINALDIR/etc/sysconfig/tcuser
+sudo rm $FINALDIR/etc/sysconfig/tcedir
+
+# Copy bootlocal.sh to opt
+sudo cp "$WORKDIR/build_files/bootlocal.sh" "$FINALDIR/opt/."
+
+# Disable ZSwap
+sudo sed -i '/# Main/a NOZSWAP=1' "$FINALDIR/etc/init.d/tc-config"
+
+# Rebuild build directory into gz file
+( cd "$FINALDIR" && sudo find | sudo cpio -o -H newc | gzip -9 > "$WORKDIR/tinyipa.gz" )
+
+# Copy vmlinuz to new name
+cp "$WORKDIR/build_files/vmlinuz64" "$WORKDIR/tinyipa.vmlinuz"

ironic_python_agent/shell/copy_configdrive_to_disk.sh

@@ -105,6 +105,9 @@ else
     # Find partition we just created
     # Dump all partitions, ignore empty ones, then get the last partition ID
     ISO_PARTITION=`sfdisk --dump $DEVICE | grep -v ' 0,' | tail -n1 | awk '{print $1}'` || fail "finding ISO partition created on ${DEVICE}"
+
+    # Wait for udev to pick up the partition
+    udevadm settle --exit-if-exists=$ISO_PARTITION
   fi
 fi