Add redhat-common element

Add an element intended for use in both Fedora and RHEL.  This
allows them to share install steps that are common to both.

Change-Id: Ie4e820a7b777b8701514351b1f802cfe57c3812e

elements/redhat-common/README.md

@@ -0,0 +1,13 @@
+Image installation steps common to RHEL and Fedora.
+
+Overrides:
+
+ * To use a non-default URL for downloading base cloud images,
+   use the environment variable DIB_CLOUD_IMAGES
+ * To download a non-default release of cloud images, use the
+   environment variable DIB_RELEASE
+ * Alternatively, set DIB_LOCAL_IMAGE to the local path of a qcow2 cloud
+   image. This is useful in that you can use a customized or previously built
+   cloud image from diskimage-builder as input. The cloud image does not have
+   to have been built by diskimage-builder. It should be a full disk image,
+   not just a filesystem image.

elements/redhat-common/bin/extract-image

@@ -0,0 +1,87 @@
+#!/bin/bash
+
+# Intended to be called from the root.d cloud-image script as follows:
+# $TMP_HOOKS_PATH/bin/extract-image $BASE_IMAGE_FILE $BASE_IMAGE_TAR $IMAGE_LOCATION $CACHED_IMAGE
+
+
+set -eu
+
+BASE_IMAGE_FILE=$1
+BASE_IMAGE_TAR=$2
+IMAGE_LOCATION=$3
+CACHED_IMAGE=$4
+
+CACHED_TAR=$DIB_IMAGE_CACHE/$BASE_IMAGE_TAR
+DIB_LOCAL_IMAGE=${DIB_LOCAL_IMAGE:-""}
+
+if [ -n "$DIB_OFFLINE" -a -f "$CACHED_TAR" ] ; then
+    echo "Not checking freshness of cached $CACHED_TAR."
+else
+
+    if [ -z "$DIB_LOCAL_IMAGE" ]; then
+        echo "Fetching Base Image"
+
+        # There seems to be some bad Fedora mirrors returning http 404's for the cloud image.
+        # If the image fails to download due to a 404 we retry once.
+        set +e
+        $TMP_HOOKS_PATH/bin/cache-url $IMAGE_LOCATION $CACHED_IMAGE
+        RV=$?
+        set -e
+
+        if [ "$RV" == "44" ] ; then
+            $TMP_HOOKS_PATH/bin/cache-url $IMAGE_LOCATION $CACHED_IMAGE
+        elif [ "$RV" != "0" ] ; then
+            exit 1
+        fi
+    fi
+
+    if [ ! -f $CACHED_TAR -o \
+        $CACHED_IMAGE -nt $CACHED_TAR ] ; then
+        echo "Repacking base image as tarball."
+        WORKING=$(mktemp -d)
+        EACTION="rm -r $WORKING"
+        trap "$EACTION" EXIT
+        echo "Working in $WORKING"
+        RAW_FILE=$(basename $BASE_IMAGE_FILE)
+        RAW_FILE=${RAW_FILE#.qcow2}.raw
+        qemu-img convert -f qcow2 -O raw $CACHED_IMAGE $WORKING/$RAW_FILE
+        # WARNING: The mattdm image has the root filesystem on the second
+        #          partition (p2). If he changes the image the MAGIC_BIT
+        #          might also need to change.
+        # UPDATE to above warning alluding to Fedora18:
+        # F19 images have the rootfs partition on p1
+        MAGIC_BIT=p1
+        # kpartx fails if no /dev/loop* exists, "losetup -f" prints first unused
+        # loop device and creates it if it doesn't exist
+        sudo losetup -f
+        # XXX: Parsing stdout is dangerous, would like a better way to discover
+        #      the device used for the image.
+        # NOTE: On F17 (parted-3.0-10.fc17.x86_64), partprobe of
+        #       /dev/loop0 does not create /dev/loop0p2, while kpartx at
+        #       least creates /dev/mapper/loop0p2.
+        LOOPDEV=$(sudo kpartx -av $WORKING/$RAW_FILE | awk "/loop[0-9]+$MAGIC_BIT/ {print \$3}")
+        if ! timeout 5 sh -c "while ! [ -e /dev/mapper/$LOOPDEV ]; do sleep 1; done"; then
+            echo "Error: Could not find /dev/mapper/$LOOPDEV"
+            exit 1
+        fi
+        EACTION="sudo kpartx -d $WORKING/$RAW_FILE;$EACTION"
+        trap "$EACTION" EXIT
+        mkdir $WORKING/mnt
+        sudo mount /dev/mapper/$LOOPDEV $WORKING/mnt
+        EACTION="sudo umount -f $WORKING/mnt;$EACTION"
+        trap "$EACTION" EXIT
+        # Chroot in so that we get the correct uid/gid
+        sudo chroot $WORKING/mnt bin/tar -cz . > $WORKING/tmp.tar
+        mv $WORKING/tmp.tar $CACHED_TAR
+    else
+        echo "Using cached tar from $CACHED_TAR"
+    fi
+
+fi
+# Extract the base image (use --numeric-owner to avoid UID/GID mismatch between
+# image tarball and host OS e.g. when building Fedora image on an openSUSE host)
+echo "Extracting base root image from $CACHED_TAR"
+sudo tar -C $TARGET_ROOT --numeric-owner -xzf $CACHED_TAR
+if [ -e "$TARGET_ROOT/lost+found" ]; then
+  sudo rmdir $TARGET_ROOT/lost+found
+fi

elements/redhat-common/bin/map-packages

@@ -19,7 +19,7 @@ import sys
 
 # Manually maintained for brevity; consider making this compiled from
 # distromatch or other rich data sources.
-# Debian name on the left, Fedora on the right.
+# Debian name on the left, Fedora/RHEL on the right.
 package_map = {
     'apache2': 'httpd',
     'arping': 'iputils',

elements/redhat-common/bin/map-services

@@ -0,0 +1,81 @@
+#!/usr/bin/env python
+
+# Copyright 2012 Hewlett-Packard Development Company, L.P.
+# Copyright 2014 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import os
+import sys
+
+# Manually maintained for brevity; consider making this compiled from
+# distromatch or other rich data sources.
+# TripleO service name on the left, Fedora/RHEL on the right.
+service_map = {
+    'cinder-api': 'openstack-cinder-api',
+    'cinder-scheduler': 'openstack-cinder-scheduler',
+    'cinder-volume': 'openstack-cinder-volume',
+    'glance-api': 'openstack-glance-api',
+    'glance-reg': 'openstack-glance-registry',
+    'heat-api': 'openstack-heat-api',
+    'heat-api-cfn': 'openstack-heat-api-cfn',
+    'heat-api-cloudwatch': 'openstack-heat-api-cloudwatch',
+    'heat-engine': 'openstack-heat-engine',
+    'keystone': 'openstack-keystone',
+    'libvirt-bin': 'libvirtd',
+    'mysql': ['mysqld', 'mariadb'],
+    'nova-conductor': 'openstack-nova-conductor',
+    'nova-api': 'openstack-nova-api',
+    'nova-cert': 'openstack-nova-cert',
+    'nova-scheduler': 'openstack-nova-scheduler',
+    'nova-consoleauth': 'openstack-nova-consoleauth',
+    'nova-compute': 'openstack-nova-compute',
+    'rsync': 'rsyncd',
+    'swift-proxy': 'openstack-swift-proxy',
+    'swift-account': 'openstack-swift-account',
+    'swift-account-auditor': 'openstack-swift-account-auditor',
+    'swift-account-reaper': 'openstack-swift-account-reaper',
+    'swift-account-replicator': 'openstack-swift-account-replicator',
+    'swift-container': 'openstack-swift-container',
+    'swift-container-auditor': 'openstack-swift-container-auditor',
+    'swift-container-replicator': 'openstack-swift-container-replicator',
+    'swift-container-updater': 'openstack-swift-container-updater',
+    'swift-object': 'openstack-swift-object',
+    'swift-object-auditor': 'openstack-swift-object-auditor',
+    'swift-object-replicator': 'openstack-swift-object-replicator',
+    'swift-object-updater': 'openstack-swift-object-updater',
+    'tgt': 'tgtd',
+}
+
+for arg in sys.argv[1:]:
+    # We need to support the service name being different when installing from
+    # source vs. packages. So, if the requested service file already exists,
+    # just use that.
+    if os.path.exists('/lib/systemd/system/%s.service' % arg):
+        print(arg)
+    else:
+        mapping = service_map.get(arg, arg)
+        # Handle cases where a service may map to multiple names depending on
+        # which specific distribution we're using.
+        if isinstance(mapping, list):
+            for name in mapping:
+                if os.path.exists('/lib/systemd/system/%s.service' % name):
+                    print(name)
+                    break
+            else:
+                # We didn't find a match for any of the mappings.
+                print(arg)
+        else:
+            print(mapping)
+sys.exit(0)

elements/redhat-common/finalise.d/01-clean-old-kernels.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+install-packages yum-utils
+
+package-cleanup --oldkernels -y --count=1

elements/redhat-common/finalise.d/99-cleanup-tmp-grub

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+set -eux
+
+rm -rf /tmp/grub

elements/redhat-common/finalise.d/99-setup-first-boot

@@ -0,0 +1,35 @@
+#!/bin/bash
+set -e
+
+if [ -d /etc/first-boot.d ]; then
+    rc_local=/etc/rc.d/rc.local
+
+    FILE_EXISTED=
+    if [ -f $rc_local ]
+    then
+        FILE_EXISTED=1
+        mv $rc_local $rc_local.REAL
+    fi
+
+    dd of=$rc_local <<EOF
+#!/bin/bash
+set -e
+set -o xtrace
+
+dib-first-boot
+EOF
+
+    if [ $FILE_EXISTED ]
+    then
+        echo "mv $rc_local.REAL $rc_local" >> $rc_local
+    else
+        echo "rm \$0" >> $rc_local
+    fi
+
+    echo "exit 0" >> $rc_local
+
+    chmod 755 $rc_local
+
+    # Enable the service
+    systemctl enable rc-local.service
+fi

elements/redhat-common/install.d/01-install-deps

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+# Install any packages in this file that may not be in the base cloud
+# image but could reasonably be expected
+
+install-packages which tcpdump traceroute

elements/redhat-common/pre-install.d/00-usr-local-bin-secure-path

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+sed -i '/secure_path/ s/$/:\/usr\/local\/bin/' /etc/sudoers

elements/redhat-common/pre-install.d/02-lsb

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+set -eu
+
+install -m 0755 -o root -g root /opt/stack/lsb-release/lsb_release /usr/local/bin

elements/redhat-common/pre-install.d/15-remove-grub

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+set -e
+
+yum remove -y grub2
+# Install grub2 dependencies to minimise packages installed during finalise.
+install-packages grub2-tools gettext os-prober system-logos
+
+# Remove all old versions of grub2 from the yum cache and then ensure the
+# latest version is in the cache.
+basearch=$(cat /etc/yum/vars/basearch)
+find /tmp/yum/$basearch -regex ".*/grub2-[0-9].*\.rpm" -exec rm -f {} \;
+install-packages -d grub2
+
+# Copy grub2 rpm out of mounted yum cache for install during finalise
+mkdir -p /tmp/grub
+cp $(find /tmp/yum/$basearch -regex ".*/grub2-[0-9].*\.rpm") /tmp/grub
+echo "rpm -i /tmp/grub/*.rpm" > /tmp/grub/install
+
+#GRUB_CFG=/boot/grub2/grub.cfg
+
+#[ -f "$GRUB_CFG" ]
+
+# Update the config to have the search UUID of the image being built.
+# When partition staging is moved to a separate stage, this will need to happen
+# there. This generates a non-UUID config, which is irrelevant for booting with
+# hypervisor kernel + ramdisk, and fixed up by 51-grub for vm images.
+#GRUB_DISABLE_LINUX_UUID=true grub2-mkconfig -o $GRUB_CFG