Helm-toolkit: Update job for creating s3 buckets

This updates the helm-toolkit manifest template and scipts for
creating an S3 bucket and linking it to a user. This moves away
from the previous python implementation that used rgwadmin, and
instead uses s3cmd for a cleaner approach that can support more
recent versions of ceph

Change-Id: I305062a5daa063bfe21a12448d7a3957bca00bf4

elasticsearch/templates/configmap-bin.yaml

@@ -30,7 +30,7 @@ data:
 {{ tuple "bin/_helm-tests.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
   ceph-admin-keyring.sh: |
 {{ tuple "bin/_ceph-admin-keyring.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
-  create-s3-bucket.py: |
+  create-s3-bucket.sh: |
 {{- include "helm-toolkit.scripts.create_s3_bucket" . | indent 4 }}
   create-s3-user.sh: |
 {{- include "helm-toolkit.scripts.create_s3_user" . | indent 4 }}

elasticsearch/values.yaml

@@ -23,7 +23,7 @@ images:
     curator:  docker.io/bobrik/curator:5.2.0
     elasticsearch: docker.io/srwilkers/elasticsearch-s3:v0.1.0
     ceph_key_placement: docker.io/port/ceph-config-helper:v1.10.3
-    s3_bucket: docker.io/port/ceph-config-helper:v1.10.3
+    s3_bucket: docker.io/ceph/daemon:tag-build-master-luminous-ubuntu-16.04
     s3_user: docker.io/port/ceph-config-helper:v1.10.3
     helm_tests: docker.io/openstackhelm/heat:newton
     prometheus_elasticsearch_exporter: docker.io/justwatch/elasticsearch_exporter:1.0.1

helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl

@@ -56,7 +56,7 @@ spec:
           imagePullPolicy: {{ $envAll.Values.images.pull_policy }}
 {{ tuple $envAll $envAll.Values.pod.resources.jobs.s3_bucket | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
           command:
-            - /tmp/create-s3-bucket.py
+            - /tmp/create-s3-bucket.sh
           env:
 {{- with $env := dict "s3AdminSecret" $envAll.Values.secrets.rgw.admin }}
 {{- include "helm-toolkit.snippets.rgw_s3_admin_env_vars" $env | indent 12 }}
@@ -69,9 +69,9 @@ spec:
             - name: RGW_HOST
               value: {{ tuple "ceph_object_store" "internal" "api" $envAll | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}
           volumeMounts:
-            - name: s3-bucket-py
+            - name: s3-bucket-sh
-              mountPath: /tmp/create-s3-bucket.py
+              mountPath: /tmp/create-s3-bucket.sh
-              subPath: create-s3-bucket.py
+              subPath: create-s3-bucket.sh
               readOnly: true
             - name: etcceph
               mountPath: /etc/ceph
@@ -86,7 +86,7 @@ spec:
               readOnly: true
             {{ end }}
       volumes:
-        - name: s3-bucket-py
+        - name: s3-bucket-sh
           configMap:
             name: {{ $configMapBin | quote }}
             defaultMode: 0555

helm-toolkit/templates/scripts/_create-s3-bucket.py.tpl

@@ -1,94 +0,0 @@
-{{/*
-Copyright 2017 The Openstack-Helm Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-   http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/}}
-
-{{- define "helm-toolkit.scripts.create_s3_bucket" }}
-#!/usr/bin/env python
-
-import os
-import sys
-import logging
-import rgwadmin
-import rgwadmin.exceptions
-
-# Create logger, console handler and formatter
-logger = logging.getLogger('OpenStack-Helm S3 Bucket')
-logger.setLevel(logging.DEBUG)
-ch = logging.StreamHandler()
-ch.setLevel(logging.DEBUG)
-formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
-
-# Set the formatter and add the handler
-ch.setFormatter(formatter)
-logger.addHandler(ch)
-
-# Get S3 admin user's access key
-if "S3_ADMIN_ACCESS_KEY" in os.environ:
-  access_key = os.environ['S3_ADMIN_ACCESS_KEY']
-  logger.info('Found S3 admin access key')
-else:
-  logger.critical('S3 admin access key environment variable not set')
-  sys.exit(1)
-
-# Get S3 admin user's secret key
-if "S3_ADMIN_SECRET_KEY" in os.environ:
-  secret_key = os.environ['S3_ADMIN_SECRET_KEY']
-  logger.info('Found S3 admin secret key')
-else:
-  logger.critical('S3 admin secret key environment variable not set')
-  sys.exit(1)
-
-# Get RGW S3 host endpoint
-if "RGW_HOST" in os.environ:
-  server = os.environ['RGW_HOST']
-  logger.info('Found RGW S3 host endpoint')
-else:
-  logger.critical('RGW S3 host endpoint environment variable not set')
-  sys.exit(1)
-
-# Get name of S3 user to link to bucket
-if "S3_USERNAME" in os.environ:
-  s3_user = os.environ['S3_USERNAME']
-  logger.info('Found S3 user name')
-else:
-  logger.critical('S3 user name environment variable not set')
-  sys.exit(1)
-
-# Get name of bucket to create for user link
-if "S3_BUCKET" in os.environ:
-  s3_bucket = os.environ['S3_BUCKET']
-  logger.info('Found S3 bucket name')
-else:
-  logger.critical('S3 bucket name environment variable not set')
-  sys.exit(1)
-
-try:
-  rgw_admin = rgwadmin.RGWAdmin(access_key, secret_key, server, secure=False)
-  try:
-    rgw_admin.get_bucket(bucket=s3_bucket,uid=s3_user)
-  except (rgwadmin.exceptions.NoSuchBucket, rgwadmin.exceptions.NoSuchKey), e:
-    rgw_admin.create_bucket(bucket=s3_bucket)
-    bucket = rgw_admin.get_bucket(bucket=s3_bucket)
-    bucket_id = bucket['id']
-    rgw_admin.link_bucket(bucket=s3_bucket, bucket_id=bucket_id, uid=s3_user)
-    logger.info("Created bucket {} and linked it to user {}".format(s3_bucket, s3_user))
-    sys.exit(0)
-  else:
-    logger.info("The bucket {} exists for user {}! Exiting without creating a new bucket!".format(s3_bucket, s3_user))
-except rgwadmin.exceptions.InvalidArgument:
-  logger.critical("Invalid arguments supplied for rgwadmin connection. Please check your s3 keys and endpoint")
-  sys.exit(1)
-
-{{- end }}

helm-toolkit/templates/scripts/_create-s3-bucket.sh.tpl

@@ -0,0 +1,45 @@
+{{/*
+Copyright 2017 The Openstack-Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- define "helm-toolkit.scripts.create_s3_bucket" }}
+#!/bin/bash
+
+function create_rgw_s3_bucket ()
+{
+  create_bucket=$(s3cmd mb s3://$S3_BUCKET --host=$RGW_HOST --access_key=$S3_ADMIN_ACCESS_KEY --secret_key=$S3_ADMIN_SECRET_KEY --no-encrypt --no-check-certificate)
+  if [ $? -eq 0 ]; then
+    echo "Bucket $S3_BUCKET created"
+  else
+    echo "Error trying to create bucket $S3_BUCKET"
+    exit 1
+  fi
+}
+
+function modify_bucket_acl ()
+{
+  modify_acl=$(s3cmd setacl s3://$S3_BUCKET --host=$RGW_HOST --access_key=$S3_ADMIN_ACCESS_KEY --secret_key=$S3_ADMIN_SECRET_KEY --no-encrypt --no-check-certificate --acl-grant=read:$S3_USERNAME --acl-grant=write:$S3_USERNAME)
+  if [ $? -eq 0 ]; then
+    echo "Bucket $S3_BUCKET ACL updated"
+  else
+    echo "Error trying to update bucket $S3_BUCKET ACL"
+    exit 1
+  fi
+}
+
+create_rgw_s3_bucket
+modify_bucket_acl
+
+{{- end }}