tools/toCOPY/generate-centos-repo.sh

#!/bin/bash
#
# SPDX-License-Identifier: Apache-2.0
#
# Copyright (C) 2019 Intel Corporation
#

GENERATE_CENTOS_REPO_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}" )" )"

source $GENERATE_CENTOS_REPO_DIR/lst_utils.sh

mirror_dir=""
layer_dirs=""

CREATEREPO=$(which createrepo_c)
if [ $? -ne 0 ]; then
    CREATEREPO="createrepo"
fi

usage () {
    echo
    echo "Create a virtual rpm repo containing only rpms listed in various lst files."
    echo "The virtual repo contains only symlinks to to previously downloaded or built rpms."
    echo
    echo "Usage"
    echo
    echo "$0 [Options] [ --mirror-dir=<mirror-path> | <mirror-path> ]"
    echo
    echo "Commin Options:"
    echo "  --distro=<distro>:  Create repo for the designated distro."
    echo "                      Default 'centos'"
    echo "  --layer=<layer>:    Create a smaller repo, sufficient to build"
    echo "                      only the given layer."
    echo "                      Default: use the LAYER environmnet valiable, or 'all'."
    echo "  --mirror-dir=<dir>: Set the mirror directory.  This is where"
    echo "                      the previously downloaded rpms are located."
    echo
    echo "Override options: For use when working on a multi-layer change"
    echo "  --config-dir=<dir>: Use an alternate config directory rather than the"
    echo "                      system defined one"
    echo "  --layer-inc-url=<lower_layer>,<build_type>,<url>:"
    echo "                      Override the url for the image include file of a lower"
    echo "                      layer's build type.  Normally the url(s) is read from"
    echo "                      <config_dir>/<distro>/<layer>/required_layer_iso_inc.cfg"
    echo "                      This option can be used more than once."
    echo "  --layer-pkg-url=<lower_layer>,<build_type>,<url>:"
    echo "                      Override the url for the package list of a lower"
    echo "                      layer's build type.  Normally the url(s) is read from"
    echo "                      <config_dir>/<distro>/<layer>/required_layer_pkgs.cfg."
    echo "                      This option can be used more than once."
    echo "  --layer-wheels-inc-url=<lower_layer>,<stream>,<url>:"
    echo "                      Override the url for the image include file of a lower"
    echo "                      layer's build type.  Normally the url(s) is read from"
    echo "                      <config_dir>/<distro>/<layer>/required_layer_wheel_inc.cfg"
    echo "                      This option can be used more than once."
    echo "  --layer-dir=<dir>:  Look in provided dir for packages to link to."
    echo "                      This option can be used more than once."
    echo
}

cleanup () {
    if [ -e "${mirror_content}" ]; then
        \rm -f ${mirror_content}
    fi
    if [ -e "${TMP_LST_DIR}" ]; then
        \rm -rf ${TMP_LST_DIR}
    fi
}

trap "cleanup ; exit 1" INT

if [ -z "$MY_REPO" ]; then
    echo "\$MY_REPO is not set. Ensure you are running this script"
    echo "from the container and \$MY_REPO points to the root of"
    echo "your folder tree."
    exit -1
fi

TEMP=$(getopt -o h --long help,config-dir:,distro:,layer:,layer-dir:,layer-inc-url:,layer-pkg-url:,layer-wheels-inc-url:,mirror-dir: -n 'generate-centos-repo' -- "$@")
if [ $? -ne 0 ]; then
    echo "getopt error"
    usage
    exit 1
fi
eval set -- "$TEMP"


while true ; do
    case "$1" in
        --mirror-dir)     mirror_dir=$2 ; shift 2 ;;
        --layer-dir)      layer_dirs+=" ${2/,/ }" ; shift 2 ;;
        --layer-inc-url)  set_layer_image_inc_urls "${2}" ; shift 2 ;;
        --layer-wheels-inc-url)  set_layer_wheels_inc_urls "${2}" ; shift 2 ;;
        --layer-pkg-url)  set_layer_pkg_urls "${2}" ; shift 2 ;;
        --config-dir)     set_and_validate_config_dir "${2}"; shift 2 ;;
        --distro)         set_and_validate_distro "${2}"; shift 2 ;;
        --layer)          set_and_validate_layer "${2}"; shift 2 ;;
        -h|--help)        echo "help"; usage; exit 0 ;;
        --)               shift ; break ;;
        *)                usage; exit 1 ;;
    esac
done

if [ "$mirror_dir" == "" ]; then
    if [ $# -ne 1 ]; then
        usage
        exit -1
    fi

    mirror_dir=$1
fi

echo "mirror_dir=${mirror_dir}"
echo "config_dir=${config_dir}"
echo "distro=${distro}"
echo "layer=${layer}"
echo
echo "layer_pkg_urls=${layer_pkg_urls[@]}"
echo
echo "layer_image_inc_urls=${layer_image_inc_urls[@]}"
echo
echo "layer_wheels_inc_urls=${layer_wheels_inc_urls[@]}"
echo

dest_dir=$MY_REPO/centos-repo
timestamp="$(date +%F_%H%M)"
mock_cfg_file=$MY_REPO/build-tools/repo_files/mock.cfg.proto
comps_xml_file=$MY_REPO/build-tools/repo_files/comps.xml
mock_cfg_dest_file=$MY_REPO/centos-repo/mock.cfg.proto
comps_xml_dest_file=$MY_REPO/centos-repo/Binary/comps.xml

TMP_LST_DIR=$(mktemp -d /tmp/tmp_lst_dir_XXXXXX)
mkdir -p $TMP_LST_DIR
lst_file_dir="$TMP_LST_DIR"
inc_file_dir="${dest_dir}/layer_image_inc"
wheels_file_dir="${dest_dir}/layer_wheels_inc"
build_info_file_dir="${dest_dir}/layer_build_info"

rpm_lst_files="rpms_3rdparties.lst rpms_centos3rdparties.lst rpms_centos.lst"
rpm_lst_files_rt=""
other_lst_file="other_downloads.lst"

for template in $rpm_lst_files $other_lst_file; do
    lst="$lst_file_dir/${template}"
    merge_lst ${config_dir} ${distro} ${template} > ${lst}
done

missing_rpms_file=missing.txt

\rm -f ${missing_rpms_file}

# Strip trailing / from mirror_dir if it was specified...
mirror_dir=$(readlink -f ${mirror_dir} | sed "s%/$%%")

if [[ ( ! -d ${mirror_dir}/Binary ) || ( ! -d ${mirror_dir}/Source ) ]]; then
    echo "The mirror ${mirror_dir} doesn't has the Binary and Source"
    echo "folders. Please provide a valid mirror"
    exit -1
fi

for layer_dir in ${layer_dirs}; do
    if [ ! -d ${layer_dir} ]; then
        echo "The layer-dir ${layer_dir} doesn't exist"
        exit -1
    fi
done

if [ ! -d "${dest_dir}" ]; then
    mkdir -p "${dest_dir}"
fi

for t in "Binary" "Source" ; do
    target_dir=${dest_dir}/$t
    if [ ! -d "$target_dir" ]; then
        mkdir -p "$target_dir"
    else
        mv -f "$target_dir" "$target_dir-backup-$timestamp"
        mkdir -p "$target_dir"
    fi
done

#
# Dowload image inc files from layer_image_inc_urls
#
\rm -rf ${inc_file_dir}
mkdir -p ${inc_file_dir}
for key in "${!layer_image_inc_urls[@]}"; do
    lower_layer="${key%,*}"
    inc_type="${key#*,}"
    url="${layer_image_inc_urls[${key}]}"
    name_from_url=$(url_to_file_name "${url}")

    if [ "${inc_type}" == "std" ]; then
        ideal_name="${lower_layer}_${image_inc_from_layer_build_template}"
    elif [ "${inc_type}" == "dev" ]; then
        ideal_name="${lower_layer}_${dev_image_inc_from_layer_build_template}"
    else
        ideal_name="${lower_layer}_${inc_type}_${image_inc_from_layer_build_template}"
    fi

    list="${ideal_name}"

    for f in $(find -L ${layer_dirs} ${mirror_dir} -type f -name "${name_from_url}"); do
        cp $f ${inc_file_dir}/${list}
        if [ $? -ne 0 ]; then
            echo "WARNING: Failed to copy from cached file '$f' to satisfy url '${url}'"
        fi
    done

    if [ ! -f ${inc_file_dir}/${list} ]; then
        curl --silent --fail ${url} > ${inc_file_dir}/${list}
        if [ $? -ne 0 ]; then
            echo "ERROR: Failed to download from url '${url}'"
            exit 1
        fi
    fi
done

#
# Dowload wheels inc files from layer_wheels_inc_urls
#
\rm -rf ${wheels_file_dir}
mkdir -p ${wheels_file_dir}
for key in "${!layer_wheels_inc_urls[@]}"; do
    lower_layer="${key%,*}"
    stream="${key#*,}"
    url="${layer_wheels_inc_urls[${key}]}"
    name_from_url=$(url_to_file_name "${url}")

    ideal_name="${lower_layer}_${distro}_${stream}_${wheels_inc_from_layer_build_template}"

    list="${ideal_name}"

    for f in $(find -L ${layer_dirs} ${mirror_dir} -type f -name "${name_from_url}"); do
        cp $f ${wheels_file_dir}/${list}
        if [ $? -ne 0 ]; then
            echo "WARNING: Failed to copy from cached file '$f' to satisfy url '${url}'"
        fi
    done

    if [ ! -f ${wheels_file_dir}/${list} ]; then
        curl --silent --fail ${url} > ${wheels_file_dir}/${list}
        if [ $? -ne 0 ]; then
            echo "ERROR: Failed to download from url '${url}'"
            exit 1
        fi
    fi
done

#
# Dowload build info files
#
build_info_from_layer_build_template="BUILD_INFO"
\rm -rf ${build_info_file_dir}
mkdir -p ${build_info_file_dir}
for key in "${!layer_image_inc_urls[@]}"; do
    lower_layer="${key%,*}"
    inc_type="${key#*,}"

    if [ "${inc_type}" != "std" ]; then
        continue
    fi

    if [ "$(basename ${layer_image_inc_urls[${key}]})" != "image.inc" ]; then
        continue
    fi

    url=$( echo ${layer_image_inc_urls[${key}]} | sed 's#image.inc$#BUILD_INFO#' )
    name_from_url=$(url_to_file_name "${url}")
    ideal_name="${lower_layer}_${build_info_from_layer_build_template}"

    list="${ideal_name}"

    for f in $(find -L ${layer_dirs} ${mirror_dir} -type f -name "${name_from_url}"); do
        cp $f ${build_info_file_dir}/${list}
        if [ $? -ne 0 ]; then
            echo "WARNING: Failed to copy from cached file '$f' to satisfy url '${url}'"
        fi
    done

    if [ ! -f ${build_info_file_dir}/${list} ]; then
        curl --silent --fail ${url} > ${build_info_file_dir}/${list}
        if [ $? -ne 0 ]; then
            echo "WARNING: Failed to download from url '${url}'"
        fi
    fi
done


#
# Dowload lst files from layer_pkg_urls
#
for key in "${!layer_pkg_urls[@]}"; do
    lower_layer="${key%,*}"
    build_type="${key#*,}"
    url="${layer_pkg_urls[${key}]}"
    name_from_url=$(url_to_file_name "${url}")
    ideal_name="${lower_layer}_${build_type}_${rpms_from_layer_build_template}"
    list="${ideal_name}"

    for f in $(find -L ${layer_dirs} ${mirror_dir} -type f -name "${name_from_url}"); do
        cp $f ${lst_file_dir}/${list}
        if [ $? -ne 0 ]; then
            echo "WARNING: Failed to copy from cached file '$f' to satisfy url '${url}'"
        fi
    done

    if [ ! -f ${lst_file_dir}/${list} ]; then
        curl --silent --fail ${url} > ${lst_file_dir}/${list}
        if [ $? -ne 0 ]; then
            echo "ERROR: Failed to download from url '${url}'"
            exit 1
        fi
    fi

    if [ "${build_type}" == "rt" ]; then
        rpm_lst_files_rt+=" ${list}"
    else
        rpm_lst_files+=" ${list}"
    fi

    url_type=${url%%:*}
    if [ "${url_type}" == "file" ]; then
        url_dir=$(dirname ${url#file://})
        if [ ! -d ${url_dir} ]; then
            echo "ERROR: No such directory '${url_dir}' derived from url: ${url}"
            exit 1
        fi
        layer_dirs+=" ${url_dir}"
    fi
done

echo "rpm_lst_files=${rpm_lst_files}"
echo "rpm_lst_files_rt=${rpm_lst_files_rt}"
echo "layer_dirs=${layer_dirs}"

mirror_content=$(mktemp -t centos-repo-XXXXXX)
find -L ${layer_dirs} ${mirror_dir} -type f -name '*.rpm' > ${mirror_content}

sed_expression=""
for d in ${mirror_dir} ${layer_dirs}; do
    sed_expression+=" -e s%^${d}/%%"
done


process_lst_file () {
    local lst_file="${1}"
    local dest_dir="${2}"

    grep -v "^#" ${lst_file_dir}/${lst_file} | while IFS="#" read rpmname extrafields; do
        if [ -z "${rpmname}" ]; then
            continue
        fi

        mirror_file=$(grep "/${rpmname}$" ${mirror_content} | head -n 1)
        if [ -z "${mirror_file}" ]; then
            echo "Error -- could not find requested ${rpmname} in ${mirror_dir}"
            echo ${rpmname} >> ${missing_rpms_file}
            continue
        fi

        # Great, we found the file!  Let's strip the mirror_dir prefix from it...
        ff=$(echo ${mirror_file} | sed ${sed_expression})
        f_name=$(basename "$ff")
        arch=$(echo ${f_name} | rev | cut -d '.' -f 2 | rev)
        if [ "${arch}" == "src" ]; then
            sub_dir="Source"
        else
            sub_dir="Binary/${arch}"
        fi

        # Make sure we have a subdir (so we don't symlink the first file as
        # the subdir name)
        mkdir -p ${dest_dir}/${sub_dir}

        # Link it!
        echo "Creating symlink for ${dest_dir}/${sub_dir}/${f_name}"
        ln -sf "${mirror_file}" "${dest_dir}/${sub_dir}/${f_name}"
        if [ $? -ne 0 ]; then
            echo "Failed ${mirror_file}: ln -sf \"${mirror_file}\" \"${dest_dir}/${sub_dir}\""
        fi
    done
}

for lst_file in ${rpm_lst_files} ; do
    process_lst_file "${lst_file}" "${dest_dir}" || exit 1
done

for lst_file in ${rpm_lst_files_rt} ; do
    process_lst_file "${lst_file}" "${dest_dir}/rt" || exit 1
done


echo "Copying comps.xml file."

if [ ! -f "$comps_xml_file" ]; then
    echo "Cannot find comps.xml file!"
    exit 1
fi

if [ -f "$comps_xml_dest_file" ]; then
    \cp -f "$comps_xml_dest_file" "$comps_xml_dest_file-backup-$timestamp"
fi
cp "$comps_xml_file" "$comps_xml_dest_file"


echo "Createing yum repodata."

for subdir in Source Binary; do
    repo_dir="${dest_dir}/${subdir}"
    mkdir -p "${repo_dir}"
    if [ -f "${repo_dir}/comps.xml" ]; then
        ${CREATEREPO} -g "${repo_dir}/comps.xml" -d "${repo_dir}"
    else
        ${CREATEREPO} -d "${repo_dir}"
    fi

    repo_dir="${dest_dir}/rt/${subdir}"
    mkdir -p "${repo_dir}"
    if [ -f "${repo_dir}/comps.xml" ]; then
        ${CREATEREPO} -g "${repo_dir}/comps.xml" -d "${repo_dir}"
    else
        ${CREATEREPO} -d "${repo_dir}"
    fi
done


echo "Copying mock.cfg.proto file."

if [ ! -f "$mock_cfg_file" ]; then
    echo "Cannot find mock.cfg.proto file!"
    exit 1
fi

if [ -f "$mock_cfg_dest_file" ]; then
    \cp -f "$mock_cfg_dest_file" "$mock_cfg_dest_file-backup-$timestamp"
fi
cp "$mock_cfg_file" "$mock_cfg_dest_file"


# Populate the contents from other list files
cat ${lst_file_dir}/${other_lst_file} | grep -v "#" | while IFS=":" read targettype item extrafields; do
    if [ "${targettype}" == "folder" ]; then
        echo "Creating folder ${item}"
        mkdir -p $MY_REPO/centos-repo/Binary/${item}
    fi

    if [ "${targettype}" == "file" ]; then
        mkdir -p $MY_REPO/centos-repo/Binary/$(dirname ${item})
        echo "Creating symlink for $MY_REPO/centos-repo/Binary/${item}"
        ln -sf ${mirror_dir}/Binary/${item} $MY_REPO/centos-repo/Binary/${item}
    fi
done

echo "Done creating repo directory"
declare -i missing_rpms_file_count=$(wc -l ${missing_rpms_file} 2>/dev/null | awk '{print $1}')
if [ ${missing_rpms_file_count} -gt 0 ]; then
    echo "WARNING: Some targets could not be found.  Your repo may be incomplete."
    echo "Missing targets:"
    cat ${missing_rpms_file}
    exit 1
fi