From 818063573368f3162e95f70bd63821d1b96ad8df Mon Sep 17 00:00:00 2001 From: Steve Wilkerson Date: Fri, 4 Jan 2019 16:25:57 -0600 Subject: [PATCH] Helm-toolkit: Update job for creating s3 buckets This updates the helm-toolkit manifest template and scipts for creating an S3 bucket and linking it to a user. This moves away from the previous python implementation that used rgwadmin, and instead uses s3cmd for a cleaner approach that can support more recent versions of ceph Change-Id: I305062a5daa063bfe21a12448d7a3957bca00bf4 --- elasticsearch/templates/configmap-bin.yaml | 2 +- elasticsearch/values.yaml | 2 +- .../manifests/_job-s3-bucket.yaml.tpl | 10 +- .../scripts/_create-s3-bucket.py.tpl | 94 ------------------- .../scripts/_create-s3-bucket.sh.tpl | 45 +++++++++ 5 files changed, 52 insertions(+), 101 deletions(-) delete mode 100644 helm-toolkit/templates/scripts/_create-s3-bucket.py.tpl create mode 100644 helm-toolkit/templates/scripts/_create-s3-bucket.sh.tpl diff --git a/elasticsearch/templates/configmap-bin.yaml b/elasticsearch/templates/configmap-bin.yaml index 742bb1745..543e9461f 100644 --- a/elasticsearch/templates/configmap-bin.yaml +++ b/elasticsearch/templates/configmap-bin.yaml @@ -30,7 +30,7 @@ data: {{ tuple "bin/_helm-tests.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} ceph-admin-keyring.sh: | {{ tuple "bin/_ceph-admin-keyring.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} - create-s3-bucket.py: | + create-s3-bucket.sh: | {{- include "helm-toolkit.scripts.create_s3_bucket" . | indent 4 }} create-s3-user.sh: | {{- include "helm-toolkit.scripts.create_s3_user" . | indent 4 }} diff --git a/elasticsearch/values.yaml b/elasticsearch/values.yaml index cfcd75472..d96cc10df 100644 --- a/elasticsearch/values.yaml +++ b/elasticsearch/values.yaml @@ -23,7 +23,7 @@ images: curator: docker.io/bobrik/curator:5.2.0 elasticsearch: docker.io/srwilkers/elasticsearch-s3:v0.1.0 ceph_key_placement: docker.io/port/ceph-config-helper:v1.10.3 - s3_bucket: docker.io/port/ceph-config-helper:v1.10.3 + s3_bucket: docker.io/ceph/daemon:tag-build-master-luminous-ubuntu-16.04 s3_user: docker.io/port/ceph-config-helper:v1.10.3 helm_tests: docker.io/openstackhelm/heat:newton prometheus_elasticsearch_exporter: docker.io/justwatch/elasticsearch_exporter:1.0.1 diff --git a/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl b/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl index e3d3e67e0..70cd03417 100644 --- a/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl +++ b/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl @@ -56,7 +56,7 @@ spec: imagePullPolicy: {{ $envAll.Values.images.pull_policy }} {{ tuple $envAll $envAll.Values.pod.resources.jobs.s3_bucket | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} command: - - /tmp/create-s3-bucket.py + - /tmp/create-s3-bucket.sh env: {{- with $env := dict "s3AdminSecret" $envAll.Values.secrets.rgw.admin }} {{- include "helm-toolkit.snippets.rgw_s3_admin_env_vars" $env | indent 12 }} @@ -69,9 +69,9 @@ spec: - name: RGW_HOST value: {{ tuple "ceph_object_store" "internal" "api" $envAll | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }} volumeMounts: - - name: s3-bucket-py - mountPath: /tmp/create-s3-bucket.py - subPath: create-s3-bucket.py + - name: s3-bucket-sh + mountPath: /tmp/create-s3-bucket.sh + subPath: create-s3-bucket.sh readOnly: true - name: etcceph mountPath: /etc/ceph @@ -86,7 +86,7 @@ spec: readOnly: true {{ end }} volumes: - - name: s3-bucket-py + - name: s3-bucket-sh configMap: name: {{ $configMapBin | quote }} defaultMode: 0555 diff --git a/helm-toolkit/templates/scripts/_create-s3-bucket.py.tpl b/helm-toolkit/templates/scripts/_create-s3-bucket.py.tpl deleted file mode 100644 index 643fe9160..000000000 --- a/helm-toolkit/templates/scripts/_create-s3-bucket.py.tpl +++ /dev/null @@ -1,94 +0,0 @@ -{{/* -Copyright 2017 The Openstack-Helm Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/}} - -{{- define "helm-toolkit.scripts.create_s3_bucket" }} -#!/usr/bin/env python - -import os -import sys -import logging -import rgwadmin -import rgwadmin.exceptions - -# Create logger, console handler and formatter -logger = logging.getLogger('OpenStack-Helm S3 Bucket') -logger.setLevel(logging.DEBUG) -ch = logging.StreamHandler() -ch.setLevel(logging.DEBUG) -formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') - -# Set the formatter and add the handler -ch.setFormatter(formatter) -logger.addHandler(ch) - -# Get S3 admin user's access key -if "S3_ADMIN_ACCESS_KEY" in os.environ: - access_key = os.environ['S3_ADMIN_ACCESS_KEY'] - logger.info('Found S3 admin access key') -else: - logger.critical('S3 admin access key environment variable not set') - sys.exit(1) - -# Get S3 admin user's secret key -if "S3_ADMIN_SECRET_KEY" in os.environ: - secret_key = os.environ['S3_ADMIN_SECRET_KEY'] - logger.info('Found S3 admin secret key') -else: - logger.critical('S3 admin secret key environment variable not set') - sys.exit(1) - -# Get RGW S3 host endpoint -if "RGW_HOST" in os.environ: - server = os.environ['RGW_HOST'] - logger.info('Found RGW S3 host endpoint') -else: - logger.critical('RGW S3 host endpoint environment variable not set') - sys.exit(1) - -# Get name of S3 user to link to bucket -if "S3_USERNAME" in os.environ: - s3_user = os.environ['S3_USERNAME'] - logger.info('Found S3 user name') -else: - logger.critical('S3 user name environment variable not set') - sys.exit(1) - -# Get name of bucket to create for user link -if "S3_BUCKET" in os.environ: - s3_bucket = os.environ['S3_BUCKET'] - logger.info('Found S3 bucket name') -else: - logger.critical('S3 bucket name environment variable not set') - sys.exit(1) - -try: - rgw_admin = rgwadmin.RGWAdmin(access_key, secret_key, server, secure=False) - try: - rgw_admin.get_bucket(bucket=s3_bucket,uid=s3_user) - except (rgwadmin.exceptions.NoSuchBucket, rgwadmin.exceptions.NoSuchKey), e: - rgw_admin.create_bucket(bucket=s3_bucket) - bucket = rgw_admin.get_bucket(bucket=s3_bucket) - bucket_id = bucket['id'] - rgw_admin.link_bucket(bucket=s3_bucket, bucket_id=bucket_id, uid=s3_user) - logger.info("Created bucket {} and linked it to user {}".format(s3_bucket, s3_user)) - sys.exit(0) - else: - logger.info("The bucket {} exists for user {}! Exiting without creating a new bucket!".format(s3_bucket, s3_user)) -except rgwadmin.exceptions.InvalidArgument: - logger.critical("Invalid arguments supplied for rgwadmin connection. Please check your s3 keys and endpoint") - sys.exit(1) - -{{- end }} diff --git a/helm-toolkit/templates/scripts/_create-s3-bucket.sh.tpl b/helm-toolkit/templates/scripts/_create-s3-bucket.sh.tpl new file mode 100644 index 000000000..365b4a5ec --- /dev/null +++ b/helm-toolkit/templates/scripts/_create-s3-bucket.sh.tpl @@ -0,0 +1,45 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- define "helm-toolkit.scripts.create_s3_bucket" }} +#!/bin/bash + +function create_rgw_s3_bucket () +{ + create_bucket=$(s3cmd mb s3://$S3_BUCKET --host=$RGW_HOST --access_key=$S3_ADMIN_ACCESS_KEY --secret_key=$S3_ADMIN_SECRET_KEY --no-encrypt --no-check-certificate) + if [ $? -eq 0 ]; then + echo "Bucket $S3_BUCKET created" + else + echo "Error trying to create bucket $S3_BUCKET" + exit 1 + fi +} + +function modify_bucket_acl () +{ + modify_acl=$(s3cmd setacl s3://$S3_BUCKET --host=$RGW_HOST --access_key=$S3_ADMIN_ACCESS_KEY --secret_key=$S3_ADMIN_SECRET_KEY --no-encrypt --no-check-certificate --acl-grant=read:$S3_USERNAME --acl-grant=write:$S3_USERNAME) + if [ $? -eq 0 ]; then + echo "Bucket $S3_BUCKET ACL updated" + else + echo "Error trying to update bucket $S3_BUCKET ACL" + exit 1 + fi +} + +create_rgw_s3_bucket +modify_bucket_acl + +{{- end }}