From 1062b035ce91016ea85fc7bce8f00c3376c2db91 Mon Sep 17 00:00:00 2001 From: Hirotaka Wakabayashi Date: Thu, 9 Feb 2023 13:58:31 +0900 Subject: [PATCH] Allows users to specify a MariaDB version in Dockerfile This PR allows users to select MariaDB versions. The original problem is that Trove fails to create backup of MariaDB- -10.7 instances because Trove's backup script is from MariaDB-10.4[1], which is generally incompatible with MariaDB-10.7[2]. 1: https://opendev.org/openstack/trove/src/branch/master/backup/install.sh#L26 2: https://mariadb.com/kb/en/mariabackup-overview/#compatibility-of-mariabackup-releases-with-mariadb-server-releases Story: 2010566 Task: 47275 Change-Id: I03e6b58a7b1746fef8c08d92acd7b1ccac9a46eb --- backup/Dockerfile | 5 +- backup/install.sh | 151 ++++++++++++++---- devstack/plugin.sh | 8 +- playbooks/image-build/docker-registry.yaml | 8 +- ...-backup-docker-image-7e3106446dcb4dd0.yaml | 6 + 5 files changed, 136 insertions(+), 42 deletions(-) create mode 100644 releasenotes/notes/support-new-mariadb-versions-in-backup-docker-image-7e3106446dcb4dd0.yaml diff --git a/backup/Dockerfile b/backup/Dockerfile index 0d542db22d..598eaa2350 100644 --- a/backup/Dockerfile +++ b/backup/Dockerfile @@ -1,7 +1,8 @@ FROM ubuntu:20.04 LABEL maintainer="anlin.kong@gmail.com" -ARG DATASTORE="mysql5.7" +ARG DATASTORE="mysql" +ARG DATASTORE_VERSION="5.7" ARG APTOPTS="-y -qq --no-install-recommends --allow-unauthenticated" RUN export DEBIAN_FRONTEND="noninteractive" \ @@ -15,7 +16,7 @@ RUN apt-get update \ COPY . /opt/trove/backup WORKDIR /opt/trove/backup -RUN ./install.sh $DATASTORE +RUN ./install.sh --datastore $DATASTORE --datastore-version $DATASTORE_VERSION RUN apt-get update \ && apt-get install $APTOPTS build-essential python3-setuptools python3-all python3-all-dev python3-pip libffi-dev libssl-dev libxml2-dev libxslt1-dev libyaml-dev libpq-dev \ diff --git a/backup/install.sh b/backup/install.sh index 26f5a51e4a..5e2132f6c6 100755 --- a/backup/install.sh +++ b/backup/install.sh @@ -3,39 +3,126 @@ set -e export APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1 APTOPTS="-y -qq --no-install-recommends --allow-unauthenticated" +OS_RELEASE_CODENAME=$(lsb_release -sc) -case "$1" in -"mysql5.7") - curl -sSL https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb -o percona-release.deb - dpkg -i percona-release.deb - percona-release enable-only tools release - apt-get update - apt-get install $APTOPTS percona-xtrabackup-24 - rm -f percona-release.deb - ;; -"mysql8.0") - curl -sSL https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb -o percona-release.deb - dpkg -i percona-release.deb - percona-release enable-only tools release - apt-get update - apt-get install $APTOPTS percona-xtrabackup-80 - rm -f percona-release.deb - ;; -"mariadb") - apt-key adv --fetch-keys 'https://mariadb.org/mariadb_release_signing_key.asc' - add-apt-repository "deb [arch=amd64] http://mirror2.hs-esslingen.de/mariadb/repo/10.4/ubuntu $(lsb_release -cs) main" - apt-get install $APTOPTS mariadb-backup - ;; -"postgresql") - apt-key adv --fetch-keys 'https://www.postgresql.org/media/keys/ACCC4CF8.asc' - add-apt-repository "deb [arch=amd64] http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main" - apt-get install $APTOPTS postgresql-client-12 - ;; -*) - echo "datastore $1 not supported" - exit 1 - ;; -esac + +# +# usage() +# +usage() { + echo "Usage : $(basename $0) [--datastore datastore] [--datastore-version datastore-version]" + echo "" + echo " Command parameters:" + echo " 'datastore' is the datastore. The options are: 'mariadb', 'mysql', 'postgresql'" + echo " 'datastore-version' is the datastore version of the datastore." + echo "" + exit 1 +} + + +# +# parse options +# +OPT_DATASTORE="" +OPT_DATASTORE_VERSION="" + +if [ $# -eq 1 ]; then + # TODO(hiwkby) We should avoid hardcoding of datastore versions but + # for compatibility, we must accept the hardcoded version string. + if [ "$1" = "mysql5.7" ]; then + OPT_DATASTORE="mysql" + OPT_DATASTORE_VERSION="5.7" + elif [ "$1" = "mysql8.0" ]; then + OPT_DATASTORE="mysql" + OPT_DATASTORE_VERSION="8.0" + elif [ "$1" = "mariadb" ]; then + OPT_DATASTORE="mariadb" + OPT_DATASTORE_VERSION="10.4" + elif [ "$1" = "postgresql" ]; then + OPT_DATASTORE="postgresql" + OPT_DATASTORE_VERSION="12" + else + usage + fi +else + while [ $# -ne 0 ]; do + if [ -z "$1" ]; then + break + elif [ "$1" = "--datastore" ]; then + shift + if [ $# -eq 0 ]; then + echo "\"--datastore\" option should have a datastore name" + exit 1 + fi + OPT_DATASTORE="$1" + elif [ "$1" = "--datastore-version" ]; then + shift + if [ $# -eq 0 ]; then + echo "\"--datastore-version\" option should have a database version" + exit 1 + fi + OPT_DATASTORE_VERSION="$1" + elif [ "$1" = "--help" ]; then + usage + fi + shift + done +fi + +if [ "${OPT_DATASTORE}" = "mysql" ]; then + if [ "${OPT_DATASTORE_VERSION}" = "5.7" ]; then + curl -sSL https://repo.percona.com/apt/percona-release_latest.${OS_RELEASE_CODENAME}_all.deb -o percona-release.deb + dpkg -i percona-release.deb + percona-release enable-only tools release + apt-get update + apt-get install ${APTOPTS} percona-xtrabackup-24 + rm -f percona-release.deb + elif [ "${OPT_DATASTORE_VERSION}" = "8.0" ]; then + curl -sSL https://repo.percona.com/apt/percona-release_latest.${OS_RELEASE_CODENAME}_all.deb -o percona-release.deb + dpkg -i percona-release.deb + percona-release enable-only tools release + apt-get update + apt-get install ${APTOPTS} percona-xtrabackup-80 + rm -f percona-release.deb + else + echo "datastore ${OPT_DATASTORE} with ${OPT_DATASTORE_VERSION} not supported" + exit 1 + fi +elif [ "${OPT_DATASTORE}" = "mariadb" ]; then + # See the url below about the supported version. + # https://mariadb.com/docs/xpand/ref/repo/cli/mariadb_repo_setup/mariadb-server-version/ + apt-key adv --fetch-keys 'https://mariadb.org/mariadb_release_signing_key.asc' + if $(curl -LsS -O https://downloads.mariadb.com/MariaDB/mariadb_repo_setup); then + if [ -f "./mariadb_repo_setup" ]; then + chmod u+x "./mariadb_repo_setup" + if $(./mariadb_repo_setup --mariadb-server-version=${OPT_DATASTORE_VERSION}); then + apt-get install ${APTOPTS} mariadb-backup + else + echo "mariadb_repo_setup command failed" + exit 1 + fi + else + echo "no such a script mariadb_repo_setup" + exit 1 + fi + else + echo "curl command failed" + exit 1 + fi +elif [ "${OPT_DATASTORE}" = "postgresql" ]; then + # See here for the supported version + # https://www.postgresql.org/support/versioning/ + apt-key adv --fetch-keys 'https://www.postgresql.org/media/keys/ACCC4CF8.asc' + add-apt-repository "deb [arch=amd64] http://apt.postgresql.org/pub/repos/apt/ ${OS_RELEASE_CODENAME}-pgdg main" + + # postgresql-client-{6,7,8,9}.x or postgresql-client-{10,11,12} + DATASTORE_CLIENT_PKG_VERSION=$(echo ${OPT_DATASTORE_VERSION} | awk -F'.' '{ if ($1 > 9) {print $1} else {print $1 "." $2} }') + if [ -z "${DATASTORE_CLIENT_PKG_VERSION}" ]; then + echo "no postgresql-client version" + exit 1 + fi + apt-get install ${APTOPTS} postgresql-client-${DATASTORE_CLIENT_PKG_VERSION} +fi apt-get clean rm -rf /var/lib/apt/lists/* diff --git a/devstack/plugin.sh b/devstack/plugin.sh index 113ceea235..90bde6fdec 100644 --- a/devstack/plugin.sh +++ b/devstack/plugin.sh @@ -532,10 +532,10 @@ function create_registry_container { done pushd $DEST/trove/backup # build backup images - sudo docker build --network host -t 127.0.0.1:4000/trove-datastores/db-backup-mysql5.7:1.1.0 --build-arg DATASTORE=mysql5.7 . - sudo docker build --network host -t 127.0.0.1:4000/trove-datastores/db-backup-mysql8.0:1.1.0 --build-arg DATASTORE=mysql8.0 . - sudo docker build --network host -t 127.0.0.1:4000/trove-datastores/db-backup-mariadb:1.1.0 --build-arg DATASTORE=mariadb . - sudo docker build --network host -t 127.0.0.1:4000/trove-datastores/db-backup-postgresql:1.1.2 --build-arg DATASTORE=postgresql . + sudo docker build --network host -t 127.0.0.1:4000/trove-datastores/db-backup-mysql5.7:1.1.0 --build-arg DATASTORE=mysql --build-arg DATASTORE_VERSION=5.7 . + sudo docker build --network host -t 127.0.0.1:4000/trove-datastores/db-backup-mysql8.0:1.1.0 --build-arg DATASTORE=mysql --build-arg DATASTORE_VERSION=8.0 . + sudo docker build --network host -t 127.0.0.1:4000/trove-datastores/db-backup-mariadb:1.1.0 --build-arg DATASTORE=mariadb --build-arg DATASTORE_VERSION=10.4 . + sudo docker build --network host -t 127.0.0.1:4000/trove-datastores/db-backup-postgresql:1.1.2 --build-arg DATASTORE=postgresql --build-arg DATASTORE_VERSION=12 . popd # push backup images for backupimg in {"db-backup-mysql5.7:1.1.0","db-backup-mysql8.0:1.1.0","db-backup-mariadb:1.1.0","db-backup-postgresql:1.1.2"}; diff --git a/playbooks/image-build/docker-registry.yaml b/playbooks/image-build/docker-registry.yaml index c16f6f905a..5036e4e592 100644 --- a/playbooks/image-build/docker-registry.yaml +++ b/playbooks/image-build/docker-registry.yaml @@ -28,22 +28,22 @@ - "postgres:12" - name: Build mysql 5.7 backup image become: true - shell: docker build -t 127.0.0.1:5000/trove-datastores/db-backup-mysql5.7:1.1.0 --build-arg DATASTORE=mysql5.7 . + shell: docker build -t 127.0.0.1:5000/trove-datastores/db-backup-mysql5.7:1.1.0 --build-arg DATASTORE=mysql --build-arg DATASTORE_VERSION=5.7 . args: chdir: "{{ ansible_user_dir }}/src/opendev.org/openstack/trove/backup" - name: Build mysql 8.0 backup image become: true - shell: docker build -t 127.0.0.1:5000/trove-datastores/db-backup-mysql8.0:1.1.0 --build-arg DATASTORE=mysql8.0 . + shell: docker build -t 127.0.0.1:5000/trove-datastores/db-backup-mysql8.0:1.1.0 --build-arg DATASTORE=mysql --build-arg DATASTORE_VERSION=8.0 . args: chdir: "{{ ansible_user_dir }}/src/opendev.org/openstack/trove/backup" - name: Build mariadb backup image become: true - shell: docker build -t 127.0.0.1:5000/trove-datastores/db-backup-mariadb:1.1.0 --build-arg DATASTORE=mariadb . + shell: docker build -t 127.0.0.1:5000/trove-datastores/db-backup-mariadb:1.1.0 --build-arg DATASTORE=mariadb --build-arg DATASTORE_VERSION=10.4 . args: chdir: "{{ ansible_user_dir }}/src/opendev.org/openstack/trove/backup" - name: Build postgresql backup image become: true - shell: docker build -t 127.0.0.1:5000/trove-datastores/db-backup-postgresql:1.1.2 --build-arg DATASTORE=postgresql . + shell: docker build -t 127.0.0.1:5000/trove-datastores/db-backup-postgresql:1.1.2 --build-arg DATASTORE=postgresql --build-arg DATASTORE_VERSION=12 . args: chdir: "{{ ansible_user_dir }}/src/opendev.org/openstack/trove/backup" - name: Push the backup images diff --git a/releasenotes/notes/support-new-mariadb-versions-in-backup-docker-image-7e3106446dcb4dd0.yaml b/releasenotes/notes/support-new-mariadb-versions-in-backup-docker-image-7e3106446dcb4dd0.yaml new file mode 100644 index 0000000000..886fe36fd9 --- /dev/null +++ b/releasenotes/notes/support-new-mariadb-versions-in-backup-docker-image-7e3106446dcb4dd0.yaml @@ -0,0 +1,6 @@ +--- +features: + - | + The user can optionally specify the datastore version of backup docker + image in Dockerfile by calling the install.sh script with the --datastore + parameter and the --datastore-version parameter.