openstack/openstack-helm
Code Issues Proposed changes

Refactor Ceph secret generation

Browse Source 
This PS refactors the ceph chart and secret generation process.
The updated chart replaces the existing "bootstrap" chart.
Additionally, Ceph manifests and deployment guides were modified
accordingly.

Change-Id: I6f5bb88fc0f40cfee8865d9dab83859d765e7537
Co-Authored-By: Larry Rensing <lr699s@att.com>
This commit is contained in:
Pete Birley 2017-06-14 13:57:21 -05:00
parent efa8293e54
commit 8ef5d94674
81 changed files with 1518 additions and 1061 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Makefile
View File

@ -15,8 +15,8 @@
HELM = helm
TASK = build
CHARTS = helm-toolkit bootstrap ceph mariadb etcd rabbitmq memcached
CHARTS += keystone glance cinder horizon neutron nova heat
CHARTS = helm-toolkit ceph mariadb etcd rabbitmq
CHARTS += memcached keystone glance cinder horizon neutron nova heat
CHARTS += barbican mistral senlin magnum ingress
all: $(CHARTS)

3
bootstrap/.gitignore vendored
View File

@ -1,3 +0,0 @@
secrets/*
!secrets/.gitkeep
templates/_secrets.tpl

27
bootstrap/.helmignore
View File

@ -1,27 +0,0 @@
# Patterns to ignore when building packages.
# This supports shell glob matching, relative path matching, and
# negation (prefixed with !). Only one pattern per line.
.DS_Store
# Common VCS dirs
.git/
.gitignore
.bzr/
.bzrignore
.hg/
.hgignore
.svn/
# Common backup files
*.swp
*.bak
*.tmp
*~
# Various IDEs
.project
.idea/
*.tmproj
bin/
etc/
patches/
*.py
Makefile

18
bootstrap/Chart.yaml
View File

@ -1,18 +0,0 @@
# Copyright 2017 The Openstack-Helm Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
apiVersion: v1
description: OpenStack-Helm namespace bootstrap
name: bootstrap
version: 0.1.0

18
bootstrap/requirements.yaml
View File

@ -1,18 +0,0 @@
# Copyright 2017 The Openstack-Helm Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
dependencies:
- name: helm-toolkit
repository: http://localhost:8879/charts
version: 0.1.0

18
bootstrap/values.yaml
View File

@ -1,18 +0,0 @@
# Copyright 2017 The Openstack-Helm Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Default values for bootstrap.
# This is a YAML-formatted file.
# Declare name/value pairs to be passed into your templates.
# name: value

14
ceph/templates/bin/_ceph-key.py.tpl Normal file
View File

@ -0,0 +1,14 @@
#!/bin/python
import os
import struct
import time
import base64
key = os.urandom(16)
header = struct.pack(
'<hiih',
1, # le16 type: CEPH_CRYPTO_AES
int(time.time()), # le32 created: seconds
0, # le32 created: nanoseconds,
len(key), # le16: len(key)
)
print(base64.b64encode(header + key).decode('ascii'))

37
ceph/templates/bin/_ceph-key.sh.tpl Normal file
View File

@ -0,0 +1,37 @@
#!/bin/bash
set -ex
function ceph_gen_key () {
python ${CEPH_GEN_DIR}/ceph-key.py
}
function kube_ceph_keyring_gen () {
CEPH_KEY=$1
CEPH_KEY_TEMPLATE=$2
sed "s|{{"{{"}} key {{"}}"}}|${CEPH_KEY}|" ${CEPH_TEMPLATES_DIR}/${CEPH_KEY_TEMPLATE} | base64 | tr -d '\n'
}
function create_kube_key () {
CEPH_KEYRING=$1
CEPH_KEYRING_NAME=$2
CEPH_KEYRING_TEMPLATE=$3
KUBE_SECRET_NAME=$4
if ! kubectl get --namespace ${DEPLOYMENT_NAMESPACE} secrets ${KUBE_SECRET_NAME}; then
{
cat <<EOF
---
apiVersion: v1
kind: Secret
metadata:
name: ${KUBE_SECRET_NAME}
type: Opaque
data:
${CEPH_KEYRING_NAME}: |
$( kube_ceph_keyring_gen ${CEPH_KEYRING} ${CEPH_KEYRING_TEMPLATE} )
EOF
} | kubectl create --namespace ${DEPLOYMENT_NAMESPACE} -f -
fi
}
#create_kube_key <ceph_key> <ceph_keyring_name> <ceph_keyring_template> <kube_secret_name>
create_kube_key $(ceph_gen_key) ${CEPH_KEYRING_NAME} ${CEPH_KEYRING_TEMPLATE} ${KUBE_SECRET_NAME}

22
ceph/templates/bin/_ceph-namespace-client-key.sh.tpl Normal file
View File

@ -0,0 +1,22 @@
#!/bin/bash
set -ex
ceph_activate_namespace() {
kube_namespace=$1
{
cat <<EOF
apiVersion: v1
kind: Secret
metadata:
name: "${PVC_CEPH_STORAGECLASS_USER_SECRET_NAME}"
type: kubernetes.io/rbd
data:
key: |
$(kubectl get secret ${PVC_CEPH_STORAGECLASS_ADMIN_SECRET_NAME} \
--namespace=${PVC_CEPH_STORAGECLASS_DEPLOYED_NAMESPACE} \
-o json | jq -r '.data | .[]')
EOF
} | kubectl create --namespace ${kube_namespace} -f -
}
ceph_activate_namespace ${DEPLOYMENT_NAMESPACE}

62
ceph/templates/bin/_ceph-storage-key.sh.tpl Normal file
View File

@ -0,0 +1,62 @@
#!/bin/bash
set -ex
function ceph_gen_key () {
python ${CEPH_GEN_DIR}/ceph-key.py
}
function kube_ceph_keyring_gen () {
CEPH_KEY=$1
CEPH_KEY_TEMPLATE=$2
sed "s|{{"{{"}} key {{"}}"}}|${CEPH_KEY}|" ${CEPH_TEMPLATES_DIR}/${CEPH_KEY_TEMPLATE} | base64 | tr -d '\n'
}
CEPH_CLIENT_KEY=$(ceph_gen_key)
function create_kube_key () {
CEPH_KEYRING=$1
CEPH_KEYRING_NAME=$2
CEPH_KEYRING_TEMPLATE=$3
KUBE_SECRET_NAME=$4
if ! kubectl get --namespace ${DEPLOYMENT_NAMESPACE} secrets ${KUBE_SECRET_NAME}; then
{
cat <<EOF
---
apiVersion: v1
kind: Secret
metadata:
name: ${KUBE_SECRET_NAME}
type: Opaque
data:
${CEPH_KEYRING_NAME}: |
$( kube_ceph_keyring_gen ${CEPH_KEYRING} ${CEPH_KEYRING_TEMPLATE} )
EOF
} | kubectl create --namespace ${DEPLOYMENT_NAMESPACE} -f -
fi
}
#create_kube_key <ceph_key> <ceph_keyring_name> <ceph_keyring_template> <kube_secret_name>
create_kube_key ${CEPH_CLIENT_KEY} ${CEPH_KEYRING_NAME} ${CEPH_KEYRING_TEMPLATE} ${CEPH_KEYRING_ADMIN_NAME}
function create_kube_storage_key () {
CEPH_KEYRING=$1
KUBE_SECRET_NAME=$2
if ! kubectl get --namespace ${DEPLOYMENT_NAMESPACE} secrets ${KUBE_SECRET_NAME}; then
{
cat <<EOF
---
apiVersion: v1
kind: Secret
metadata:
name: ${KUBE_SECRET_NAME}
type: kubernetes.io/rbd
data:
key: |
$( echo ${CEPH_KEYRING} | base64 | tr -d '\n' )
EOF
} | kubectl create --namespace ${DEPLOYMENT_NAMESPACE} -f -
fi
}
#create_kube_storage_key <ceph_key> <kube_secret_name>
create_kube_storage_key ${CEPH_CLIENT_KEY} ${CEPH_STORAGECLASS_ADMIN_SECRET_NAME}

31
ceph/templates/configmap-bin.yaml Normal file
View File

@ -0,0 +1,31 @@
# Copyright 2017 The Openstack-Helm Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
---
apiVersion: v1
kind: ConfigMap
metadata:
name: ceph-bin
data:
{{- if .Values.manifests_enabled.storage_secrets }}
ceph-key.py: |+
{{ tuple "bin/_ceph-key.py.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
ceph-key.sh: |+
{{ tuple "bin/_ceph-key.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
ceph-storage-key.sh: |+
{{ tuple "bin/_ceph-storage-key.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
{{- end }}
{{- if .Values.manifests_enabled.client_secrets }}
ceph-namespace-client-key.sh: |+
{{ tuple "bin/_ceph-namespace-client-key.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
{{- end }}

25
bootstrap/templates/secrets.yaml → ceph/templates/configmap-etc.yaml
View File

@ -12,21 +12,18 @@
# See the License for the specific language governing permissions and
# limitations under the License.
{{- if empty .Values.conf.ceph.config.global.mon_host -}}
{{- $monHost := tuple "ceph_mon" "internal" . | include "helm-toolkit.endpoints.hostname_endpoint_lookup" }}
{{- $monHostDomain := default .Release.Namespace .Values.ceph.namespace }}
{{- $monHostURI := cat $monHost "." $monHostDomain | nospace -}}
{{- $monHostURI | set .Values.conf.ceph.config.global "mon_host" | quote | trunc 0 -}}
{{- end -}}
---
apiVersion: v1
kind: Secret
kind: ConfigMap
metadata:
name: "pvc-ceph-conf-combined-storageclass"
type: kubernetes.io/rbd
name: ceph-etc
data:
key: |
{{ include "secrets/ceph-client-key" . | b64enc | indent 4 }}
---
apiVersion: v1
kind: Secret
metadata:
name: "pvc-ceph-client-key"
type: kubernetes.io/rbd
data:
key: |
{{ include "secrets/ceph-client-key" . | b64enc | indent 4 }}
ceph.conf: |+
{{ tuple "etc/_ceph.conf.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}

31
ceph/templates/configmap-templates.yaml Normal file
View File

@ -0,0 +1,31 @@
# Copyright 2017 The Openstack-Helm Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
{{- if .Values.manifests_enabled.storage_secrets }}
---
apiVersion: v1
kind: ConfigMap
metadata:
name: ceph-templates
data:
admin.keyring: |+
{{ tuple "templates/_admin.keyring.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
bootstrap.keyring.mds: |+
{{ tuple "templates/_bootstrap.keyring.mds.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
bootstrap.keyring.osd: |+
{{ tuple "templates/_bootstrap.keyring.osd.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
bootstrap.keyring.rgw: |+
{{ tuple "templates/_bootstrap.keyring.rgw.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
mon.keyring: |+
{{ tuple "templates/_mon.keyring.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
{{- end }}

124
ceph/templates/daemonset-osd.yaml
View File

@ -12,6 +12,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
{{- if .Values.manifests_enabled.deployment }}
{{- $envAll := . }}
{{- $dependencies := .Values.dependencies.osd }}
---
kind: DaemonSet
apiVersion: extensions/v1beta1
@ -29,47 +32,21 @@ spec:
spec:
nodeSelector:
{{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }}
volumes:
- name: devices
hostPath:
path: /dev
- name: ceph
hostPath:
path: {{ .Values.storage.var_directory }}
- name: ceph-conf
secret:
secretName: ceph-conf-combined
- name: ceph-bootstrap-osd-keyring
secret:
secretName: ceph-bootstrap-osd-keyring
- name: ceph-bootstrap-mds-keyring
secret:
secretName: ceph-bootstrap-mds-keyring
- name: ceph-bootstrap-rgw-keyring
secret:
secretName: ceph-bootstrap-rgw-keyring
- name: osd-directory
hostPath:
path: {{ .Values.storage.osd_directory }}
initContainers:
{{ tuple $envAll $dependencies "" | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
containers:
- name: osd-pod
image: {{ .Values.images.daemon }}
imagePullPolicy: {{ .Values.images.pull_policy }}
volumeMounts:
- name: devices
mountPath: /dev
- name: ceph
mountPath: /var/lib/ceph
- name: ceph-conf
mountPath: /etc/ceph
- name: ceph-bootstrap-osd-keyring
mountPath: /var/lib/ceph/bootstrap-osd
- name: ceph-bootstrap-mds-keyring
mountPath: /var/lib/ceph/bootstrap-mds
- name: ceph-bootstrap-rgw-keyring
mountPath: /var/lib/ceph/bootstrap-rgw
- name: osd-directory
mountPath: /var/lib/ceph/osd
{{- if .Values.resources.enabled }}
resources:
requests:
memory: {{ .Values.resources.osd.requests.memory | quote }}
cpu: {{ .Values.resources.osd.requests.cpu | quote }}
limits:
memory: {{ .Values.resources.osd.limits.memory | quote }}
cpu: {{ .Values.resources.osd.limits.cpu | quote }}
{{- end }}
securityContext:
privileged: true
env:
@ -81,6 +58,10 @@ spec:
value: ceph
- name: CEPH_GET_ADMIN_KEY
value: "1"
command:
- /entrypoint.sh
ports:
- containerPort: 6800
livenessProbe:
tcpSocket:
port: 6800
@ -90,10 +71,65 @@ spec:
tcpSocket:
port: 6800
timeoutSeconds: 5
resources:
requests:
memory: {{ .Values.resources.osd.requests.memory | quote }}
cpu: {{ .Values.resources.osd.requests.cpu | quote }}
limits:
memory: {{ .Values.resources.osd.limits.memory | quote }}
cpu: {{ .Values.resources.osd.limits.cpu | quote }}
volumeMounts:
- name: devices
mountPath: /dev
readOnly: false
- name: ceph
mountPath: /var/lib/ceph
readOnly: false
- name: ceph-etc
mountPath: /etc/ceph/ceph.conf
subPath: ceph.conf
readOnly: true
- name: ceph-client-admin-keyring
mountPath: /etc/ceph/ceph.client.admin.keyring
subPath: ceph.client.admin.keyring
readOnly: false
- name: ceph-mon-keyring
mountPath: /etc/ceph/ceph.mon.keyring
subPath: ceph.mon.keyring
readOnly: false
- name: ceph-bootstrap-osd-keyring
mountPath: /var/lib/ceph/bootstrap-osd/ceph.keyring
subPath: ceph.keyring
readOnly: false
- name: ceph-bootstrap-mds-keyring
mountPath: /var/lib/ceph/bootstrap-mds/ceph.keyring
subPath: ceph.keyring
readOnly: false
- name: ceph-bootstrap-rgw-keyring
mountPath: /var/lib/ceph/bootstrap-rgw/ceph.keyring
subPath: ceph.keyring
readOnly: false
- name: osd-directory
mountPath: /var/lib/ceph/osd
volumes:
- name: devices
hostPath:
path: /dev
- name: ceph
hostPath:
path: {{ .Values.ceph.storage.var_directory }}
- name: ceph-etc
configMap:
name: ceph-etc
- name: ceph-client-admin-keyring
secret:
secretName: {{ .Values.secrets.keyrings.admin }}
- name: ceph-mon-keyring
secret:
secretName: {{ .Values.secrets.keyrings.mon }}
- name: ceph-bootstrap-osd-keyring
secret:
secretName: {{ .Values.secrets.keyrings.osd }}
- name: ceph-bootstrap-mds-keyring
secret:
secretName: {{ .Values.secrets.keyrings.mds }}
- name: ceph-bootstrap-rgw-keyring
secret:
secretName: {{ .Values.secrets.keyrings.rgw }}
- name: osd-directory
hostPath:
path: {{ .Values.ceph.storage.osd_directory }}
{{- end }}

85
ceph/templates/deployment-mds.yaml
View File

@ -12,7 +12,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
---
{{- if .Values.manifests_enabled.deployment }}
{{- if .Values.ceph.enabled.mds }}
{{- $envAll := . }}
{{- $dependencies := .Values.dependencies.mds }}
kind: Deployment
apiVersion: apps/v1beta1
metadata:
@ -31,24 +34,22 @@ spec:
spec:
nodeSelector:
{{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }}
initContainers:
{{ tuple $envAll $dependencies "" | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
serviceAccount: default
volumes:
- name: ceph-conf
secret:
secretName: ceph-conf-combined
- name: ceph-bootstrap-osd-keyring
secret:
secretName: ceph-bootstrap-osd-keyring
- name: ceph-bootstrap-mds-keyring
secret:
secretName: ceph-bootstrap-mds-keyring
- name: ceph-bootstrap-rgw-keyring
secret:
secretName: ceph-bootstrap-rgw-keyring
containers:
- name: ceph-mds
image: {{ .Values.images.daemon }}
imagePullPolicy: {{ .Values.images.pull_policy }}
{{- if .Values.resources.enabled }}
resources:
requests:
memory: {{ .Values.resources.mds.requests.memory | quote }}
cpu: {{ .Values.resources.mds.requests.cpu | quote }}
limits:
memory: {{ .Values.resources.mds.limits.memory | quote }}
cpu: {{ .Values.resources.mds.limits.cpu | quote }}
{{- end }}
ports:
- containerPort: 6800
env:
@ -60,15 +61,33 @@ spec:
value: k8s
- name: CLUSTER
value: ceph
command:
- /entrypoint.sh
volumeMounts:
- name: ceph-conf
mountPath: /etc/ceph
- name: ceph-etc
mountPath: /etc/ceph/ceph.conf
subPath: ceph.conf
readOnly: true
- name: ceph-client-admin-keyring
mountPath: /etc/ceph/ceph.client.admin.keyring
subPath: ceph.client.admin.keyring
readOnly: true
- name: ceph-mon-keyring
mountPath: /etc/ceph/ceph.mon.keyring
subPath: ceph.mon.keyring
readOnly: true
- name: ceph-bootstrap-osd-keyring
mountPath: /var/lib/ceph/bootstrap-osd
mountPath: /var/lib/ceph/bootstrap-osd/ceph.keyring
subPath: ceph.keyring
readOnly: false
- name: ceph-bootstrap-mds-keyring
mountPath: /var/lib/ceph/bootstrap-mds
mountPath: /var/lib/ceph/bootstrap-mds/ceph.keyring
subPath: ceph.keyring
readOnly: false
- name: ceph-bootstrap-rgw-keyring
mountPath: /var/lib/ceph/bootstrap-rgw
mountPath: /var/lib/ceph/bootstrap-rgw/ceph.keyring
subPath: ceph.keyring
readOnly: false
livenessProbe:
tcpSocket:
port: 6800
@ -78,10 +97,24 @@ spec:
tcpSocket:
port: 6800
timeoutSeconds: 5
resources:
requests:
memory: {{ .Values.resources.mds.requests.memory | quote }}
cpu: {{ .Values.resources.mds.requests.cpu | quote }}
limits:
memory: {{ .Values.resources.mds.limits.memory | quote }}
cpu: {{ .Values.resources.mds.limits.cpu | quote }}
volumes:
- name: ceph-etc
configMap:
name: ceph-etc
- name: ceph-client-admin-keyring
secret:
secretName: {{ .Values.secrets.keyrings.admin }}
- name: ceph-mon-keyring
secret:
secretName: {{ .Values.secrets.keyrings.mon }}
- name: ceph-bootstrap-osd-keyring
secret:
secretName: {{ .Values.secrets.keyrings.osd }}
- name: ceph-bootstrap-mds-keyring
secret:
secretName: {{ .Values.secrets.keyrings.mds }}
- name: ceph-bootstrap-rgw-keyring
secret:
secretName: {{ .Values.secrets.keyrings.rgw }}
{{- end }}
{{- end }}

82
ceph/templates/deployment-moncheck.yaml
View File

@ -12,6 +12,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
{{- if .Values.manifests_enabled.deployment }}
{{- $envAll := . }}
{{- $dependencies := .Values.dependencies.moncheck }}
---
kind: Deployment
apiVersion: apps/v1beta1
@ -31,24 +34,22 @@ spec:
spec:
nodeSelector:
{{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }}
initContainers:
{{ tuple $envAll $dependencies "" | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
serviceAccount: default
volumes:
- name: ceph-conf
secret:
secretName: ceph-conf-combined
- name: ceph-bootstrap-osd-keyring
secret:
secretName: ceph-bootstrap-osd-keyring
- name: ceph-bootstrap-mds-keyring
secret:
secretName: ceph-bootstrap-mds-keyring
- name: ceph-bootstrap-rgw-keyring
secret:
secretName: ceph-bootstrap-rgw-keyring
containers:
- name: ceph-mon
image: {{ .Values.images.daemon }}
imagePullPolicy: {{ .Values.images.pull_policy }}
{{- if .Values.resources.enabled }}
resources:
requests:
memory: {{ .Values.resources.mon_check.requests.memory | quote }}
cpu: {{ .Values.resources.mon_check.requests.cpu | quote }}
limits:
memory: {{ .Values.resources.mon_check.limits.memory | quote }}
cpu: {{ .Values.resources.mon_check.limits.cpu | quote }}
{{- end }}
ports:
- containerPort: 6789
env:
@ -60,19 +61,50 @@ spec:
value: "1"
- name: CLUSTER
value: ceph
command:
- /entrypoint.sh
volumeMounts:
- name: ceph-conf
mountPath: /etc/ceph
- name: ceph-etc
mountPath: /etc/ceph/ceph.conf
subPath: ceph.conf
readOnly: true
- name: ceph-client-admin-keyring
mountPath: /etc/ceph/ceph.client.admin.keyring
subPath: ceph.client.admin.keyring
readOnly: true
- name: ceph-mon-keyring
mountPath: /etc/ceph/ceph.mon.keyring
subPath: ceph.mon.keyring
readOnly: true
- name: ceph-bootstrap-osd-keyring
mountPath: /var/lib/ceph/bootstrap-osd
mountPath: /var/lib/ceph/bootstrap-osd/ceph.keyring
subPath: ceph.keyring
readOnly: false
- name: ceph-bootstrap-mds-keyring
mountPath: /var/lib/ceph/bootstrap-mds
mountPath: /var/lib/ceph/bootstrap-mds/ceph.keyring
subPath: ceph.keyring
readOnly: false
- name: ceph-bootstrap-rgw-keyring
mountPath: /var/lib/ceph/bootstrap-rgw
resources:
requests:
memory: {{ .Values.resources.mon_check.requests.memory | quote }}
cpu: {{ .Values.resources.mon_check.requests.cpu | quote }}
limits:
memory: {{ .Values.resources.mon_check.limits.memory | quote }}
cpu: {{ .Values.resources.mon_check.limits.cpu | quote }}
mountPath: /var/lib/ceph/bootstrap-rgw/ceph.keyring
subPath: ceph.keyring
readOnly: false
volumes:
- name: ceph-etc
configMap:
name: ceph-etc
- name: ceph-client-admin-keyring
secret:
secretName: {{ .Values.secrets.keyrings.admin }}
- name: ceph-mon-keyring
secret:
secretName: {{ .Values.secrets.keyrings.mon }}
- name: ceph-bootstrap-osd-keyring
secret:
secretName: {{ .Values.secrets.keyrings.osd }}
- name: ceph-bootstrap-mds-keyring
secret:
secretName: {{ .Values.secrets.keyrings.mds }}
- name: ceph-bootstrap-rgw-keyring
secret:
secretName: {{ .Values.secrets.keyrings.rgw }}
{{- end }}

84
ceph/templates/deployment-rgw.yaml
View File

@ -12,7 +12,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
{{- if .Values.rgw.enabled }}
{{- if .Values.manifests_enabled.deployment }}
{{- if .Values.ceph.enabled.rgw }}
{{- $envAll := . }}
{{- $dependencies := .Values.dependencies.rgw }}
---
kind: Deployment
apiVersion: apps/v1beta1
@ -32,24 +35,22 @@ spec:
spec:
nodeSelector:
{{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }}
initContainers:
{{ tuple $envAll $dependencies "" | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
serviceAccount: default
volumes:
- name: ceph-conf
secret:
secretName: ceph-conf-combined
- name: ceph-bootstrap-osd-keyring
secret:
secretName: ceph-bootstrap-osd-keyring
- name: ceph-bootstrap-mds-keyring
secret:
secretName: ceph-bootstrap-mds-keyring
- name: ceph-bootstrap-rgw-keyring
secret:
secretName: ceph-bootstrap-rgw-keyring
containers:
- name: ceph-rgw
image: {{ .Values.images.daemon }}
imagePullPolicy: {{ .Values.images.pull_policy }}
{{- if .Values.resources.enabled }}
resources:
requests:
memory: {{ .Values.resources.rgw.requests.memory | quote }}
cpu: {{ .Values.resources.rgw.requests.cpu | quote }}
limits:
memory: {{ .Values.resources.rgw.limits.memory | quote }}
cpu: {{ .Values.resources.rgw.limits.cpu | quote }}
{{- end }}
ports:
- containerPort: {{ .Values.network.port.rgw_target }}
env:
@ -61,15 +62,33 @@ spec:
value: k8s
- name: CLUSTER
value: ceph
command:
- /entrypoint.sh
volumeMounts:
- name: ceph-conf
mountPath: /etc/ceph
- name: ceph-etc
mountPath: /etc/ceph/ceph.conf
subPath: ceph.conf
readOnly: true
- name: ceph-client-admin-keyring
mountPath: /etc/ceph/ceph.client.admin.keyring
subPath: ceph.client.admin.keyring
readOnly: true
- name: ceph-mon-keyring
mountPath: /etc/ceph/ceph.mon.keyring
subPath: ceph.mon.keyring
readOnly: true
- name: ceph-bootstrap-osd-keyring
mountPath: /var/lib/ceph/bootstrap-osd
mountPath: /var/lib/ceph/bootstrap-osd/ceph.keyring
subPath: ceph.keyring
readOnly: false
- name: ceph-bootstrap-mds-keyring
mountPath: /var/lib/ceph/bootstrap-mds
mountPath: /var/lib/ceph/bootstrap-mds/ceph.keyring
subPath: ceph.keyring
readOnly: false
- name: ceph-bootstrap-rgw-keyring
mountPath: /var/lib/ceph/bootstrap-rgw
mountPath: /var/lib/ceph/bootstrap-rgw/ceph.keyring
subPath: ceph.keyring
readOnly: false
livenessProbe:
httpGet:
path: /
@ -81,11 +100,24 @@ spec:
path: /
port: {{ .Values.network.port.rgw_target }}
timeoutSeconds: 5
resources:
requests:
memory: {{ .Values.resources.rgw.requests.memory | quote }}
cpu: {{ .Values.resources.rgw.requests.cpu | quote }}
limits:
memory: {{ .Values.resources.rgw.limits.memory | quote }}
cpu: {{ .Values.resources.rgw.limits.cpu | quote }}
volumes:
- name: ceph-etc
configMap:
name: ceph-etc
- name: ceph-client-admin-keyring
secret:
secretName: {{ .Values.secrets.keyrings.admin }}
- name: ceph-mon-keyring
secret:
secretName: {{ .Values.secrets.keyrings.mon }}
- name: ceph-bootstrap-osd-keyring
secret:
secretName: {{ .Values.secrets.keyrings.osd }}
- name: ceph-bootstrap-mds-keyring
secret:
secretName: {{ .Values.secrets.keyrings.mds }}
- name: ceph-bootstrap-rgw-keyring
secret:
secretName: {{ .Values.secrets.keyrings.rgw }}
{{- end }}
{{- end }}

77
ceph/templates/etc/_ceph.conf.tpl Normal file
View File

@ -0,0 +1,77 @@
[global]
fsid = {{ uuidv4 | default .Values.conf.ceph.config.global.uuid | quote }}
cephx = {{ .Values.conf.ceph.config.global.cephx | default "true" | quote }}
cephx_require_signatures = {{ .Values.conf.ceph.config.global.cephx_require_signatures | default "false" | quote }}
cephx_cluster_require_signatures = {{ .Values.conf.ceph.config.global.cephx_cluster_require_signatures | default "true" | quote }}
cephx_service_require_signatures = {{ .Values.conf.ceph.config.global.cephx_service_require_signatures | default "false" | quote }}
# auth
max_open_files = {{ .Values.conf.ceph.config.global.max_open_files | default "131072" | quote }}
osd_pool_default_pg_num = {{ .Values.conf.ceph.config.global.osd_pool_default_pg_num | default "128" | quote }}
osd_pool_default_pgp_num = {{ .Values.conf.ceph.config.global.osd_pool_default_pgp_num | default "128" | quote }}
osd_pool_default_size = {{ .Values.conf.ceph.config.global.osd_pool_default_size | default "3" | quote }}
osd_pool_default_min_size = {{ .Values.conf.ceph.config.global.osd_pool_default_min_size | default "1" | quote }}
mon_osd_full_ratio = {{ .Values.conf.ceph.config.global.mon_osd_full_ratio | default ".95" | quote }}
mon_osd_nearfull_ratio = {{ .Values.conf.ceph.config.global.mon_osd_nearfull_ratio | default ".85" | quote }}
mon_host = {{ .Values.conf.ceph.config.global.mon_host | quote }}
rgw_thread_pool_size = {{ .Values.conf.ceph.config.global.rgw_thread_pool_size | default "1024" | quote }}
rgw_num_rados_handles = {{ .Values.conf.ceph.config.global.rgw_num_rados_handles | default "100" | quote }}
[mon]
mon_osd_down_out_interval = {{ .Values.conf.ceph.config.mon.mon_osd_down_out_interval | default "600" | quote }}
mon_osd_min_down_reporters = {{ .Values.conf.ceph.config.mon.mon_osd_min_down_reporters | default "4" | quote }}
mon_clock_drift_allowed = {{ .Values.conf.ceph.config.mon.mon_clock_drift_allowed | default "0.15" | quote }}
mon_clock_drift_warn_backoff = {{ .Values.conf.ceph.config.mon.mon_clock_drift_warn_backoff | default "30" | quote }}
mon_osd_report_timeout = {{ .Values.conf.ceph.config.mon.mon_osd_report_timeout | default "300" | quote }}
[osd]
# network
cluster_network = {{ .Values.network.cluster | default "192.168.0.0/16" | quote }}
public_network = {{ .Values.network.public | default "192.168.0.0/16" | quote }}
osd_mon_heartbeat_interval = {{ .Values.conf.ceph.config.osd.osd_mon_heartbeat_interval | default "30" | quote }}
# ports
ms_bind_port_min = {{ .Values.conf.ceph.config.osd.ms_bind_port_min | default "6800" | quote }}
ms_bind_port_max = {{ .Values.conf.ceph.config.osd.ms_bind_port_max | default "7100" | quote }}
# journal
journal_size = {{ .Values.conf.ceph.config.osd.journal_size | default "100" | quote }}
# filesystem
osd_mkfs_type = {{ .Values.conf.ceph.config.osd.osd_mkfs_type | default "xfs" | quote }}
osd_mkfs_options_xfs = {{ .Values.conf.ceph.config.osd.osd_mkfs_options_xfs | default "-f -i size=2048" | quote }}
osd_max_object_name_len = {{ .Values.conf.ceph.config.osd.osd_max_object_name_len | default "256" | quote }}
# crush
osd_pool_default_crush_rule = {{ .Values.conf.ceph.config.osd.osd_pool_default_crush_rule | default "0" | quote }}
osd_crush_update_on_start = {{ .Values.conf.ceph.config.osd.osd_crush_update_on_start | default "true" | quote }}
# backend
osd_objectstore = {{ .Values.conf.ceph.config.osd.osd_objectstore | default "filestore" | quote }}
# performance tuning
filestore_merge_threshold = {{ .Values.conf.ceph.config.osd.filestore_merge_threshold | default "40" | quote }}
filestore_split_multiple = {{ .Values.conf.ceph.config.osd.filestore_split_multiple | default "8" | quote }}
osd_op_threads = {{ .Values.conf.ceph.config.osd.osd_op_threads | default "8" | quote }}
filestore_op_threads = {{ .Values.conf.ceph.config.osd.filestore_op_threads | default "8" | quote }}
filestore_max_sync_interval = {{ .Values.conf.ceph.config.osd.filestore_max_sync_interval | default "5" | quote }}
osd_max_scrubs = {{ .Values.conf.ceph.config.osd.osd_max_scrubs | default "1" | quote }}
# recovery tuning
osd_recovery_max_active = {{ .Values.conf.ceph.config.osd.osd_recovery_max_active | default "5" | quote }}
osd_max_backfills = {{ .Values.conf.ceph.config.osd.osd_max_backfills | default "2" | quote }}
osd_recovery_op_priority = {{ .Values.conf.ceph.config.osd.osd_recovery_op_priority | default "2" | quote }}
osd_client_op_priority = {{ .Values.conf.ceph.config.osd.osd_client_op_priority | default "63" | quote }}
osd_recovery_max_chunk = {{ .Values.conf.ceph.config.osd.osd_client_op_priority | default "osd_recovery_max_chunk" | quote }}
osd_recovery_threads = {{ .Values.conf.ceph.config.osd.osd_recovery_threads | default "1" | quote }}
[client]
rbd_cache_enabled = {{ .Values.conf.ceph.config.client.rbd_cache_enabled | default "true" | quote }}
rbd_cache_writethrough_until_flush = {{ .Values.conf.ceph.config.client.rbd_cache_writethrough_until_flush | default "true" | quote }}
rbd_default_features = {{ .Values.conf.ceph.config.client.rbd_default_features | default "1" | quote }}
[mds]
mds_cache_size = {{ .Values.conf.ceph.config.client.mds_mds_cache_size | default "100000" | quote }}

85
ceph/templates/job-keyring.yaml Normal file
View File

@ -0,0 +1,85 @@
# Copyright 2017 The Openstack-Helm Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
{{- if .Values.manifests_enabled.storage_secrets }}
{{- $envAll := . }}
{{- range $key1, $cephBootstrapKey := tuple "mds" "osd" "rgw" "mon" }}
---
apiVersion: batch/v1
kind: Job
metadata:
name: ceph-{{ $cephBootstrapKey }}-keyring-generator
spec:
template:
spec:
restartPolicy: OnFailure
containers:
- name: ceph-secret-generator
image: {{ $envAll.Values.images.ceph_config_helper }}
imagePullPolicy: {{ $envAll.Values.images.pull_policy }}
{{- if $envAll.Values.resources.enabled }}
resources:
requests:
memory: {{ .Values.resources.jobs.secret_provisioning.requests.memory | quote }}
cpu: {{ .Values.resources.jobs.secret_provisioning.requests.cpu | quote }}
limits:
memory: {{ .Values.resources.jobs.secret_provisioning.limits.memory | quote }}
cpu: {{ .Values.resources.jobs.secret_provisioning.limits.cpu | quote }}
{{- end }}
env:
- name: DEPLOYMENT_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: CEPH_GEN_DIR
value: /opt/ceph
- name: CEPH_TEMPLATES_DIR
value: /opt/ceph/templates
{{- if eq $cephBootstrapKey "mon"}}
- name: CEPH_KEYRING_NAME
value: ceph.mon.keyring
- name: CEPH_KEYRING_TEMPLATE
value: mon.keyring
{{- else }}
- name: CEPH_KEYRING_NAME
value: ceph.keyring
- name: CEPH_KEYRING_TEMPLATE
value: bootstrap.keyring.{{ $cephBootstrapKey }}
{{- end }}
- name: KUBE_SECRET_NAME
value: {{ index $envAll.Values.secrets.keyrings $cephBootstrapKey }}
command:
- /opt/ceph/ceph-key.sh
volumeMounts:
- name: ceph-bin
mountPath: /opt/ceph/ceph-key.sh
subPath: ceph-key.sh
readOnly: true
- name: ceph-bin
mountPath: /opt/ceph/ceph-key.py
subPath: ceph-key.py
readOnly: true
- name: ceph-templates
mountPath: /opt/ceph/templates
readOnly: true
volumes:
- name: ceph-bin
configMap:
name: ceph-bin
defaultMode: 0555
- name: ceph-templates
configMap:
name: ceph-templates
{{ end }}
{{ end }}

61
ceph/templates/job-namespace-client-key.yaml Normal file
View File

@ -0,0 +1,61 @@
# Copyright 2017 The Openstack-Helm Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
{{- if .Values.manifests_enabled.client_secrets }}
---
apiVersion: batch/v1
kind: Job
metadata:
name: ceph-namespace-client-key-generator
spec:
template:
spec:
restartPolicy: OnFailure
containers:
- name: ceph-storage-keys-generator
image: {{ .Values.images.ceph_config_helper }}
imagePullPolicy: {{ .Values.images.pull_policy }}
{{- if .Values.resources.enabled }}
resources:
requests:
memory: {{ .Values.resources.jobs.secret_provisioning.requests.memory | quote }}
cpu: {{ .Values.resources.jobs.secret_provisioning.requests.cpu | quote }}
limits:
memory: {{ .Values.resources.jobs.secret_provisioning.limits.memory | quote }}
cpu: {{ .Values.resources.jobs.secret_provisioning.limits.cpu | quote }}
{{- end }}
env:
- name: DEPLOYMENT_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: PVC_CEPH_STORAGECLASS_USER_SECRET_NAME
value: {{ .Values.storageclass.user_secret_name }}
- name: PVC_CEPH_STORAGECLASS_ADMIN_SECRET_NAME
value: {{ .Values.storageclass.admin_secret_name }}
- name: PVC_CEPH_STORAGECLASS_DEPLOYED_NAMESPACE
value: {{ .Values.storageclass.admin_secret_namespace }}
command:
- /opt/ceph/ceph-namespace-client-key.sh
volumeMounts:
- name: ceph-bin
mountPath: /opt/ceph/ceph-namespace-client-key.sh
subPath: ceph-namespace-client-key.sh
readOnly: true
volumes:
- name: ceph-bin
configMap:
name: ceph-bin
defaultMode: 0555
{{- end }}

77
ceph/templates/job-storage-admin-keys.yaml Normal file
View File

@ -0,0 +1,77 @@
# Copyright 2017 The Openstack-Helm Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
{{- if .Values.manifests_enabled.storage_secrets }}
---
apiVersion: batch/v1
kind: Job
metadata:
name: ceph-storage-keys-generator
spec:
template:
spec:
restartPolicy: OnFailure
containers:
- name: ceph-storage-keys-generator
image: {{ .Values.images.ceph_config_helper }}
imagePullPolicy: {{ .Values.images.pull_policy }}
{{- if .Values.resources.enabled }}
resources:
requests:
memory: {{ .Values.resources.jobs.secret_provisioning.requests.memory | quote }}
cpu: {{ .Values.resources.jobs.secret_provisioning.requests.cpu | quote }}
limits:
memory: {{ .Values.resources.jobs.secret_provisioning.limits.memory | quote }}
cpu: {{ .Values.resources.jobs.secret_provisioning.limits.cpu | quote }}
{{- end }}
env:
- name: DEPLOYMENT_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: CEPH_GEN_DIR
value: /opt/ceph
- name: CEPH_TEMPLATES_DIR
value: /opt/ceph/templates
- name: CEPH_KEYRING_NAME
value: ceph.client.admin.keyring
- name: CEPH_KEYRING_TEMPLATE
value: admin.keyring
- name: CEPH_KEYRING_ADMIN_NAME
value: {{ .Values.secrets.keyrings.admin }}
- name: CEPH_STORAGECLASS_ADMIN_SECRET_NAME
value: {{ .Values.storageclass.admin_secret_name }}
command:
- /opt/ceph/ceph-storage-key.sh
volumeMounts:
- name: ceph-bin
mountPath: /opt/ceph/ceph-storage-key.sh
subPath: ceph-storage-key.sh
readOnly: true
- name: ceph-bin
mountPath: /opt/ceph/ceph-key.py
subPath: ceph-key.py
readOnly: true
- name: ceph-templates
mountPath: /opt/ceph/templates
readOnly: true
volumes:
- name: ceph-bin
configMap:
name: ceph-bin
defaultMode: 0555
- name: ceph-templates
configMap:
name: ceph-templates
{{- end }}

2
ceph/templates/pdb-mon.yaml
View File

@ -1,3 +1,4 @@
{{- if .Values.manifests_enabled.deployment }}
apiVersion: policy/v1beta1
kind: PodDisruptionBudget
metadata:
@ -8,3 +9,4 @@ spec:
matchLabels:
app: ceph
daemon: mon
{{- end }}

65
ceph/templates/secrets.yaml
View File

@ -1,65 +0,0 @@
# Copyright 2017 The Openstack-Helm Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
{{- if .Values.secrets.use_common_secrets -}}
---
apiVersion: v1
kind: Secret
metadata:
name: "ceph-conf-combined"
type: Opaque
data:
ceph.conf: |
{{ include "secrets/ceph.conf" . | b64enc | indent 4 }}
ceph.client.admin.keyring: |
{{ include "secrets/ceph.client.admin.keyring" . | b64enc | indent 4 }}
ceph.mon.keyring: |
{{ include "secrets/ceph.mon.keyring" . | b64enc | indent 4 }}
---
apiVersion: v1
kind: Secret
metadata:
name: "ceph-bootstrap-rgw-keyring"
type: Opaque
data:
ceph.keyring: |
{{ include "secrets/ceph.rgw.keyring" . | b64enc | indent 4 }}
---
apiVersion: v1
kind: Secret
metadata:
name: "ceph-bootstrap-mds-keyring"
type: Opaque
data:
ceph.keyring: |
{{ include "secrets/ceph.mds.keyring" . | b64enc | indent 4 }}
---
apiVersion: v1
kind: Secret
metadata:
name: "ceph-bootstrap-osd-keyring"
type: Opaque
data:
ceph.keyring: |
{{ include "secrets/ceph.osd.keyring" . | b64enc | indent 4 }}
---
apiVersion: v1
kind: Secret
metadata:
name: "ceph-client-key"
type: Opaque
data:
ceph-client-key: |
{{ include "secrets/ceph-client-key" . | b64enc | indent 4 }}
{{- end -}}

4
ceph/templates/service-rgw.yaml
View File

@ -12,7 +12,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
{{- if .Values.rgw.enabled }}
{{- if .Values.manifests_enabled.deployment }}
{{- if .Values.ceph.enabled.rgw }}
---
apiVersion: v1
kind: Service
@ -30,3 +31,4 @@ spec:
app: ceph
daemon: rgw
{{- end }}
{{- end }}

4
ceph/templates/service.yaml
View File

@ -12,11 +12,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.
{{- if .Values.manifests_enabled.deployment }}
---
kind: Service
apiVersion: v1
metadata:
name: ceph-mon
name: {{ .Values.endpoints.ceph_mon.hosts.default }}
labels:
app: ceph
daemon: mon
@ -37,3 +38,4 @@ spec:
app: ceph
daemon: mon
clusterIP: None
{{- end }}

101
ceph/templates/statefulset-mon.yaml
View File

@ -12,6 +12,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
{{- if .Values.manifests_enabled.deployment }}
{{- $envAll := . }}
{{- $dependencies := .Values.dependencies.mon }}
---
apiVersion: apps/v1beta1
kind: StatefulSet
@ -21,7 +24,7 @@ metadata:
daemon: mon
name: ceph-mon
spec:
serviceName: {{ .Values.service.mon.name | quote }}
serviceName: {{ tuple "ceph_mon" "internal" . | include "helm-toolkit.endpoints.hostname_endpoint_lookup" }}
replicas: {{ .Values.replicas.mon }}
template:
metadata:
@ -46,33 +49,22 @@ spec:
weight: 10
nodeSelector:
{{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }}
initContainers:
{{ tuple $envAll $dependencies "" | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
serviceAccount: default
volumes:
- name: ceph-conf
secret:
secretName: ceph-conf-combined
- name: ceph-bootstrap-osd-keyring
secret:
secretName: ceph-bootstrap-osd-keyring
- name: ceph-bootstrap-mds-keyring
secret:
secretName: ceph-bootstrap-mds-keyring
- name: ceph-bootstrap-rgw-keyring
secret:
secretName: ceph-bootstrap-rgw-keyring
- name: ceph-monfs
hostPath:
path: {{ .Values.storage.mon_directory }}
containers:
- name: ceph-mon
image: {{ .Values.images.daemon }}
imagePullPolicy: {{ .Values.images.pull_policy }}
lifecycle:
preStop:
exec:
# remove the mon on Pod stop.
command:
- "/remove-mon.sh"
{{- if .Values.resources.enabled }}
resources:
requests:
memory: {{ .Values.resources.mon.requests.memory | quote }}
cpu: {{ .Values.resources.mon.requests.cpu | quote }}
limits:
memory: {{ .Values.resources.mon.limits.memory | quote }}
cpu: {{ .Values.resources.mon.limits.cpu | quote }}
{{- end }}
ports:
- containerPort: 6789
env:
@ -90,17 +82,39 @@ spec:
valueFrom:
fieldRef:
fieldPath: status.podIP
command:
- /entrypoint.sh
lifecycle:
preStop:
exec:
# remove the mon on Pod stop.
command:
- "/remove-mon.sh"
volumeMounts:
- name: ceph-conf
mountPath: /etc/ceph
- name: ceph-etc
mountPath: /etc/ceph/ceph.conf
subPath: ceph.conf
readOnly: true
- name: ceph-client-admin-keyring
mountPath: /etc/ceph/ceph.client.admin.keyring
subPath: ceph.client.admin.keyring
readOnly: true
- name: ceph-mon-keyring
mountPath: /etc/ceph/ceph.mon.keyring
subPath: ceph.mon.keyring
readOnly: false
- name: ceph-bootstrap-osd-keyring
mountPath: /var/lib/ceph/bootstrap-osd
mountPath: /var/lib/ceph/bootstrap-osd/ceph.keyring
subPath: ceph.keyring
readOnly: false
- name: ceph-bootstrap-mds-keyring
mountPath: /var/lib/ceph/bootstrap-mds
mountPath: /var/lib/ceph/bootstrap-mds/ceph.keyring
subPath: ceph.keyring
readOnly: false
- name: ceph-bootstrap-rgw-keyring
mountPath: /var/lib/ceph/bootstrap-rgw
- name: ceph-monfs
mountPath: /var/lib/ceph/mon
mountPath: /var/lib/ceph/bootstrap-rgw/ceph.keyring
subPath: ceph.keyring
readOnly: false
livenessProbe:
tcpSocket:
port: 6789
@ -110,10 +124,23 @@ spec:
tcpSocket:
port: 6789
timeoutSeconds: 5
resources:
requests:
memory: {{ .Values.resources.mon.requests.memory | quote }}
cpu: {{ .Values.resources.mon.requests.cpu | quote }}
limits:
memory: {{ .Values.resources.mon.limits.memory | quote }}
cpu: {{ .Values.resources.mon.limits.cpu | quote }}
volumes:
- name: ceph-etc
configMap:
name: ceph-etc
- name: ceph-client-admin-keyring
secret:
secretName: {{ .Values.secrets.keyrings.admin }}
- name: ceph-mon-keyring
secret:
secretName: {{ .Values.secrets.keyrings.mon }}
- name: ceph-bootstrap-osd-keyring
secret:
secretName: {{ .Values.secrets.keyrings.osd }}
- name: ceph-bootstrap-mds-keyring
secret:
secretName: {{ .Values.secrets.keyrings.mds }}
- name: ceph-bootstrap-rgw-keyring
secret:
secretName: {{ .Values.secrets.keyrings.rgw }}
{{- end }}

10
ceph/templates/storage.yaml
View File

@ -12,8 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# note that these secrets are handled by the common chart, not the ceph
# chart, as we likely want them "everywhere"
{{- if .Values.manifests_enabled.deployment }}
---
apiVersion: storage.k8s.io/v1
kind: StorageClass
@ -21,12 +20,11 @@ metadata:
name: {{ .Values.storageclass.name }}
provisioner: kubernetes.io/rbd
parameters:
monitors: {{ .Values.storageclass.monitors | default "ceph-mon.ceph:6789" }}
monitors: {{ tuple "ceph_mon" "internal" "mon" . | include "helm-toolkit.endpoints.hostname_endpoint_uri_lookup" }}
adminId: {{ .Values.storageclass.admin_id }}
adminSecretName: {{ .Values.storageclass.admin_secret_name }}
# forcing namespace due to issue with default pipeline of "{{ .Release.Namespace }}" }}
# during helm lint
adminSecretNamespace: {{ .Values.storageclass.admin_secret_namespace | default "ceph" }}
adminSecretNamespace: {{ .Values.storageclass.admin_secret_namespace }}
pool: {{ .Values.storageclass.pool }}
userId: {{ .Values.storageclass.user_id }}
userSecretName: {{ .Values.storageclass.user_secret_name }}
{{- end }}

6
ceph/templates/templates/_admin.keyring.tpl Normal file
View File

@ -0,0 +1,6 @@
[client.admin]
key = {{"{{"}} key {{"}}"}}
auid = 0
caps mds = "allow"
caps mon = "allow *"
caps osd = "allow *"

3
ceph/templates/templates/_bootstrap.keyring.mds.tpl Normal file
View File

@ -0,0 +1,3 @@
[client.bootstrap-mds]
key = {{"{{"}} key {{"}}"}}
caps mon = "allow profile bootstrap-mds"

3
ceph/templates/templates/_bootstrap.keyring.osd.tpl Normal file
View File

@ -0,0 +1,3 @@
[client.bootstrap-osd]
key = {{"{{"}} key {{"}}"}}
caps mon = "allow profile bootstrap-osd"

3
ceph/templates/templates/_bootstrap.keyring.rgw.tpl Normal file
View File

@ -0,0 +1,3 @@
[client.bootstrap-rgw]
key = {{"{{"}} key {{"}}"}}
caps mon = "allow profile bootstrap-rgw"

3
ceph/templates/templates/_mon.keyring.tpl Normal file
View File

@ -0,0 +1,3 @@
[mon.]
key = {{"{{"}} key {{"}}"}}
caps mon = "allow *"

162
ceph/values.yaml
View File

@ -12,6 +12,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
manifests_enabled:
storage_secrets: true
client_secrets: true
deployment: true
replicas:
mon: 3
rgw: 3
@ -22,8 +27,10 @@ service:
name: ceph-mon
images:
daemon: docker.io/library/ceph/daemon:tag-build-master-jewel-ubuntu-16.04
pull_policy: IfNotPresent
dep_check: docker.io/kolla/ubuntu-source-kubernetes-entrypoint:4.0.0
daemon: quay.io/attcomdev/ceph-daemon:tag-build-master-jewel-ubuntu-16.04
ceph_config_helper: docker.io/port/ceph-config-helper:v1.6.5
pull_policy: Always
labels:
node_selector_key: ceph-storage
@ -33,13 +40,114 @@ pod_disruption_budget:
mon:
min_available: 0
secrets:
keyrings:
mon: ceph-mon-keyring
mds: ceph-bootstrap-mds-keyring
osd: ceph-bootstrap-osd-keyring
rgw: ceph-bootstrap-rgw-keyring
admin: ceph-client-admin-keyring
network:
public: "10.25.0.0/16"
public: "192.168.0.0/16"
cluster: "192.168.0.0/16"
port:
mon: 6789
rgw_ingress: 80
rgw_target: 8088
conf:
ceph:
override:
append:
config:
global:
# auth
cephx: true
cephx_require_signatures: false
cephx_cluster_require_signatures: true
cephx_service_require_signatures: false
max_open_files: 131072
osd_pool_default_pg_num: 128
osd_pool_default_pgp_num: 128
osd_pool_default_size: 3
osd_pool_default_min_size: 1
mon_osd_full_ratio: .95
mon_osd_nearfull_ratio: .85
mon_host: null
mon:
mon_osd_down_out_interval: 600
mon_osd_min_down_reporters: 4
mon_clock_drift_allowed: .15
mon_clock_drift_warn_backoff: 30
mon_osd_report_timeout: 300
osd:
journal_size: 100
osd_mkfs_type: xfs
osd_mkfs_options_xfs: -f -i size=2048
osd_mon_heartbeat_interval: 30
osd_max_object_name_len: 256
#crush
osd_pool_default_crush_rule: 0
osd_crush_update_on_start: true
#backend
osd_objectstore: filestore
#performance tuning
filestore_merge_threshold: 40
filestore_split_multiple: 8
osd_op_threads: 8
filestore_op_threads: 8
filestore_max_sync_interval: 5
osd_max_scrubs: 1
#recovery tuning
osd_recovery_max_active: 5
osd_max_backfills: 2
osd_recovery_op_priority: 2
osd_client_op_priority: 63
osd_recovery_max_chunk: 1048576
osd_recovery_threads: 1
#ports
ms_bind_port_min: 6800
ms_bind_port_max: 7100
client:
rbd_cache_enabled: true
rbd_cache_writethrough_until_flush: true
rbd_default_features: "1"
mds:
mds_cache_size: 100000
dependencies:
mon:
jobs:
service:
osd:
jobs:
services:
- service: ceph_mon
endpoint: internal
moncheck:
jobs:
services:
- service: ceph_mon
endpoint: internal
rgw:
jobs:
services:
- service: ceph_mon
endpoint: internal
mds:
jobs:
services:
- service: ceph_mon
endpoint: internal
ceph:
enabled:
mds: true
rgw: false
storage:
osd_directory: /var/lib/openstack-helm/ceph/osd
var_directory: /var/lib/openstack-helm/ceph/ceph
@ -50,6 +158,7 @@ rgw:
enabled: false
resources:
enabled: false
osd:
requests:
memory: "512Mi"
@ -85,36 +194,14 @@ resources:
limits:
memory: "50Mi"
cpu: "500m"
# Setting this to false will assume you will
# setup and orchestrate your own secrets and
# configmaps outside of this helm chart
#
# The list below is in the format of
#
# configMapName:
# elementKeyName
#
# ceph.conf:
# ceph.conf
# ceph.client.admin.keyring
# ceph.client.admin.keyring
# ceph.mon.keyring:
# ceph.mon.keyring
# ceph-bootstrap-rgw-keyring:
# ceph.keyring
# ceph.rgw.keyring
# ceph-bootstrap-mds-keyring:
# ceph.keyring
# ceph.mds.keyring
# ceph-bootstrap-osd-keyring:
# ceph.keyring
# ceph.osd.keyring
# ceph-client-key:
# ceph-client-key
secrets:
use_common_secrets: true
jobs:
secret_provisioning:
limits:
memory: "1024Mi"
cpu: "2000m"
requests:
memory: "128Mi"
cpu: "500m"
# if you change provision_storage_class to false
# it is presumed you manage your own storage
# class definition externally
@ -125,6 +212,13 @@ storageclass:
pool: rbd
admin_id: admin
admin_secret_name: pvc-ceph-conf-combined-storageclass
admin_secret_namespace: null
admin_secret_namespace: ceph
user_id: admin
user_secret_name: pvc-ceph-client-key
endpoints:
ceph_mon:
hosts:
default: ceph-mon
port:
mon: 6789

9
cinder/templates/_funcs.tpl
View File

@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
{{- define "cinder.is_ceph_configured" -}}
{{- define "cinder.is_ceph_volume_configured" -}}
{{- range $section, $values := .Values.conf.backends -}}
{{- if kindIs "map" $values -}}
{{- if eq $values.volume_driver "cinder.volume.drivers.rbd.RBDDriver" -}}
@ -21,3 +21,10 @@ true
{{- end -}}
{{- end -}}
{{- end -}}
{{- define "cinder.is_ceph_backup_configured" -}}
{{- $values := .Values.conf.cinder.default.cinder -}}
{{- if eq $values.backup_driver "cinder.backup.drivers.ceph" -}}
true
{{- end -}}
{{- end -}}

11
cinder/templates/etc/_ceph-cinder.keyring.tpl → cinder/templates/bin/_ceph-keyring.sh.tpl
View File

@ -1,3 +1,5 @@
#!/bin/bash
# Copyright 2017 The Openstack-Helm Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
@ -12,9 +14,16 @@
# See the License for the specific language governing permissions and
# limitations under the License.
set -ex
export HOME=/tmp
cat <<EOF > /etc/ceph/ceph.client.{{ .Values.conf.backends.rbd1.rbd_user }}.keyring
[client.{{ .Values.conf.backends.rbd1.rbd_user }}]
{{- if .Values.conf.ceph.cinder_keyring }}
key = {{ .Values.conf.ceph.cinder_keyring }}
{{- else }}
key = {{- include "secrets/ceph-client-key" . -}}
key = $(cat /tmp/client-keyring)
{{- end }}
EOF
exit 0

2
cinder/templates/configmap-bin.yaml
View File

@ -33,6 +33,8 @@ data:
{{ tuple "bin/_cinder-scheduler.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
cinder-volume.sh: |
{{ tuple "bin/_cinder-volume.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
ceph-keyring.sh: |+
{{ tuple "bin/_ceph-keyring.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
{{- if .Values.bootstrap.enabled }}
bootstrap.sh: |+
{{ tuple "bin/_bootstrap.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}

13
cinder/templates/configmap-etc.yaml
View File

@ -97,17 +97,4 @@ data:
{{ .Values.conf.policy.override | indent 4 }}
{{- else -}}
{{ tuple "etc/_policy.json.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
{{- end }}
ceph.conf: |+
{{- if or (include "cinder.is_ceph_configured" .) (eq .Values.conf.cinder.default.cinder.backup_driver "cinder.backup.drivers.ceph") }}
{{ if .Values.conf.ceph.override -}}
{{ .Values.conf.ceph.override | indent 4 }}
{{- else -}}
{{ tuple "etc/_ceph.conf.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
{{- end }}
{{- if .Values.conf.ceph.append -}}
{{ .Values.conf.ceph.append | indent 4 }}
{{- end }}
ceph.client.{{ .Values.conf.backends.rbd1.rbd_user }}.keyring: |+
{{ tuple "etc/_ceph-cinder.keyring.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
{{- end }}

41
cinder/templates/deployment-backup.yaml
View File

@ -42,6 +42,23 @@ spec:
{{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }}
initContainers:
{{ tuple $envAll $dependencies $mounts_cinder_backup_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
{{- if include "cinder.is_ceph_backup_configured" . }}
- name: ceph-keyring-placement
image: {{ .Values.images.backup }}
imagePullPolicy: {{ .Values.images.pull_policy }}
command:
- /tmp/ceph-keyring.sh
volumeMounts:
- name: etcceph
mountPath: /etc/ceph
- name: cinder-bin
mountPath: /tmp/ceph-keyring.sh
subPath: ceph-keyring.sh
- name: ceph-keyring
mountPath: /tmp/client-keyring
subPath: key
readOnly: true
{{ end }}
containers:
- name: cinder-backup
image: {{ .Values.images.backup }}
@ -67,13 +84,15 @@ spec:
subPath: cinder.conf
readOnly: true
{{- if eq .Values.conf.cinder.default.cinder.backup_driver "cinder.backup.drivers.ceph" }}
- name: cinder-etc
- name: etcceph
mountPath: /etc/ceph
- name: ceph-etc
mountPath: /etc/ceph/ceph.conf
subPath: ceph.conf
readOnly: true
- name: cinder-etc
mountPath: /etc/ceph/ceph.client.{{ .Values.conf.backends.rbd1.rbd_user }}.keyring
subPath: ceph.client.{{ .Values.conf.backends.rbd1.rbd_user }}.keyring
- name: ceph-keyring
mountPath: /tmp/client-keyring
subPath: key
readOnly: true
{{- end -}}
{{ if $mounts_cinder_backup.volumeMounts }}{{ toYaml $mounts_cinder_backup.volumeMounts | indent 12 }}{{ end }}
@ -83,4 +102,18 @@ spec:
- name: cinder-etc
configMap:
name: cinder-etc
- name: cinder-bin
configMap:
name: cinder-bin
defaultMode: 0555
{{- if include "cinder.is_ceph_backup_configured" . }}
- name: etcceph
emptyDir: {}
- name: ceph-etc
configMap:
name: ceph-etc
- name: ceph-keyring
secret:
secretName: pvc-ceph-client-key
{{ end }}
{{ if $mounts_cinder_backup.volumes }}{{ toYaml $mounts_cinder_backup.volumes | indent 8 }}{{ end }}

39
cinder/templates/deployment-volume.yaml
View File

@ -42,6 +42,23 @@ spec:
{{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }}
initContainers:
{{ tuple $envAll $dependencies $mounts_cinder_volume_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
{{- if include "cinder.is_ceph_volume_configured" . }}
- name: ceph-keyring-placement
image: {{ .Values.images.volume }}
imagePullPolicy: {{ .Values.images.pull_policy }}
command:
- /tmp/ceph-keyring.sh
volumeMounts:
- name: etcceph
mountPath: /etc/ceph
- name: cinder-bin
mountPath: /tmp/ceph-keyring.sh
subPath: ceph-keyring.sh
- name: ceph-keyring
mountPath: /tmp/client-keyring
subPath: key
readOnly: true
{{ end }}
containers:
- name: cinder-volume
image: {{ .Values.images.volume }}
@ -72,14 +89,16 @@ spec:
mountPath: /etc/cinder/conf/backends.conf
subPath: backends.conf
readOnly: true
{{- if include "cinder.is_ceph_configured" . }}
- name: cinder-etc
{{- if include "cinder.is_ceph_volume_configured" . }}
- name: etcceph
mountPath: /etc/ceph
- name: ceph-etc
mountPath: /etc/ceph/ceph.conf
subPath: ceph.conf
readOnly: true
- name: cinder-etc
mountPath: /etc/ceph/ceph.client.{{ .Values.conf.backends.rbd1.rbd_user }}.keyring
subPath: ceph.client.{{ .Values.conf.backends.rbd1.rbd_user }}.keyring
- name: ceph-keyring
mountPath: /tmp/client-keyring
subPath: key
readOnly: true
{{- end }}
{{ if $mounts_cinder_volume.volumeMounts }}{{ toYaml $mounts_cinder_volume.volumeMounts | indent 12 }}{{ end }}
@ -93,4 +112,14 @@ spec:
- name: cinder-etc
configMap:
name: cinder-etc
{{- if include "cinder.is_ceph_volume_configured" . }}
- name: etcceph
emptyDir: {}
- name: ceph-etc
configMap:
name: ceph-etc
- name: ceph-keyring
secret:
secretName: pvc-ceph-client-key
{{ end }}
{{ if $mounts_cinder_volume.volumes }}{{ toYaml $mounts_cinder_volume.volumes | indent 8 }}{{ end }}

30
cinder/templates/etc/_ceph.conf.tpl
View File

@ -1,30 +0,0 @@
# Copyright 2017 The Openstack-Helm Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
[global]
rgw_thread_pool_size = 1024
rgw_num_rados_handles = 100
{{- if .Values.conf.ceph.monitors }}
[mon]
{{ range .Values.conf.ceph.monitors }}
[mon.{{ . }}]
host = {{ . }}
mon_addr = {{ . }}
{{ end }}
{{- else }}
mon_host = ceph-mon.ceph
{{- end }}
[client]
rbd_cache_enabled = true
rbd_cache_writethrough_until_flush = true

2
cinder/values.yaml
View File

@ -23,6 +23,8 @@ replicas:
scheduler: 1
backup: 1
storage: ceph
labels:
node_selector_key: openstack-control-plane
node_selector_value: enabled

217
doc/source/install/multinode.rst
View File

@ -141,32 +141,13 @@ completed.
Installing Ceph Host Requirements
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
At some future point, we want to ensure that our solution is
cloud-native, allowing installation on any host system without a package
manager and only a container runtime (i.e. CoreOS). Until this happens,
we will need to ensure that ``ceph-common`` is installed on each of our
hosts. Using our Ubuntu example:
You need to ensure that ``ceph-common`` or equivalent is
installed on each of our hosts. Using our Ubuntu example:
::
sudo apt-get install ceph-common -y
We will always attempt to keep host-specific requirements to a minimum,
and we are working with the Ceph team (Sébastien Han) to quickly address
this Ceph requirement.
Ceph Secrets Generation
~~~~~~~~~~~~~~~~~~~~~~~
Another thing of interest is that our deployment assumes that you can
generate secrets at the time of the container deployment. We require the
`sigil <https://github.com/gliderlabs/sigil/releases/download/v0.4.0/sigil_0.4.0_Linux_x86_64.tgz>`__
binary on your deployment host in order to perform this action.
::
curl -L https://github.com/gliderlabs/sigil/releases/download/v0.4.0/sigil_0.4.0_Linux_x86_64.tgz | sudo tar -zxC /usr/local/bin
Kubernetes Controller Manager
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@ -200,20 +181,18 @@ Kubernetes v1.6.5.
export kube_version=v1.6.5
sudo sed -i "s|gcr.io/google_containers/kube-controller-manager-amd64:$kube_version|quay.io/attcomdev/kube-controller-manager:$kube_version|g" /etc/kubernetes/manifests/kube-controller-manager.yaml
Now you will want to ``restart`` your Kubernetes master server to
Now you will want to ``restart`` the Kubernetes master server to
continue.
Kube Controller Manager DNS Resolution
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Until the following `Kubernetes Pull
Request <https://github.com/kubernetes/kubernetes/issues/17406>`__ is
merged, you will need to allow the Kubernetes Controller to use the
internal container ``skydns`` endpoint as a DNS server, and add the
Kubernetes search suffix into the controller's resolv.conf. As of now,
the Kubernetes controller only mirrors the host's ``resolv.conf``. This
is not sufficient if you want the controller to know how to correctly
resolve container service endpoints (in the case of DaemonSets).
You will need to allow the Kubernetes Controller to use the
Kubernetes service DNS server, and add the Kubernetes search suffix
to the controller's resolv.conf. As of now, the Kubernetes controller
only mirrors the host's ``resolv.conf``. This is not sufficient if you
want the controller to know how to correctly resolve container service
endpoints.
First, find out what the IP Address of your ``kube-dns`` deployment is:
@ -224,82 +203,16 @@ First, find out what the IP Address of your ``kube-dns`` deployment is:
kube-dns 10.96.0.10 <none> 53/UDP,53/TCP 1d
admin@kubenode01:~$
As you can see by this example, ``10.96.0.10`` is the
``CLUSTER-IP``\ IP. Now, have a look at the current
``kube-controller-manager-kubenode01`` ``/etc/resolv.conf``:
Then update the controller manager configuration to match:
::
admin@kubenode01:~$ kubectl exec kube-controller-manager-kubenode01 -n kube-system -- cat /etc/resolv.conf
# Dynamic resolv.conf(5) file for glibc resolver(3) generated by resolvconf(8)
# DO NOT EDIT THIS FILE BY HAND -- YOUR CHANGES WILL BE OVERWRITTEN
nameserver 192.168.1.70
nameserver 8.8.8.8
search jinkit.com
admin@kubenode01:~$
What we need is for ``kube-controller-manager-kubenode01``
``/etc/resolv.conf`` to look like this:
::
admin@kubenode01:~$ kubectl exec kube-controller-manager-kubenode01 -n kube-system -- cat /etc/resolv.conf
admin@kubenode01:~$ CONTROLLER_MANAGER_POD=$(kubectl get -n kube-system pods -l component=kube-controller-manager --no-headers -o name | head -1 | awk -F '/' '{ print $NF }')
admin@kubenode01:~$ kubectl exec -n kube-system ${CONTROLLER_MANAGER_POD} -- sh -c "cat > /etc/resolv.conf <<EOF
nameserver 10.96.0.10
nameserver 192.168.1.70
nameserver 8.8.8.8
search svc.cluster.local jinkit.com
admin@kubenode01:~$
You can change this by doing the following:
::
admin@kubenode01:~$ kubectl exec kube-controller-manager-kubenode01 -it -n kube-system -- /bin/bash
root@kubenode01:/# cat <<EOF > /etc/resolv.conf
nameserver 10.96.0.10
nameserver 192.168.1.70
nameserver 8.8.8.8
search svc.cluster.local jinkit.com
EOF
root@kubenode01:/#
Now you can test your changes by deploying a service to your cluster,
and resolving this from the controller. As an example, lets deploy
something useful, like `Kubernetes
dashboard <https://github.com/kubernetes/dashboard>`__:
::
kubectl create -f https://rawgit.com/kubernetes/dashboard/master/src/deploy/kubernetes-dashboard.yaml
Note the ``IP`` field:
::
admin@kubenode01:~$ kubectl describe svc kubernetes-dashboard -n kube-system
Name: kubernetes-dashboard
Namespace: kube-system
Labels: app=kubernetes-dashboard
Selector: app=kubernetes-dashboard
Type: NodePort
IP: 10.110.207.144
Port: <unset> 80/TCP
NodePort: <unset> 32739/TCP
Endpoints: 10.25.178.65:9090
Session Affinity: None
No events.
admin@kubenode01:~$
Now you should be able to resolve the host
``kubernetes-dashboard.kube-system.svc.cluster.local``:
::
admin@kubenode01:~$ kubectl exec kube-controller-manager-kubenode01 -it -n kube-system -- ping kubernetes-dashboard.kube-system.svc.cluster.local
PING kubernetes-dashboard.kube-system.svc.cluster.local (10.110.207.144) 56(84) bytes of data.
.. note::
This host example above has ``iputils-ping`` installed.
search cluster.local svc.cluster.local
EOF"
Kubernetes Node DNS Resolution
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@ -358,42 +271,15 @@ Download the latest copy of Openstack-Helm:
Ceph Preparation and Installation
---------------------------------
Ceph must be aware of the OSX cluster and public networks. These CIDR
Ceph must be aware of the OSD cluster and public networks. These CIDR
ranges are the exact same ranges you used earlier in your Calico
deployment yaml (our example was 10.25.0.0/16 due to our 192.168.0.0/16
overlap). Explore this variable to your deployment environment by
deployment yaml. Export this variable to your deployment environment by
issuing the following commands:
::
export osd_cluster_network=10.25.0.0/16
export osd_public_network=10.25.0.0/16
Ceph Storage Volumes
--------------------
Ceph must also have volumes to mount on each host labeled for
``ceph-storage``. On each host that you labeled, create the following
directory (can be overriden):
::
mkdir -p /var/lib/openstack-helm/ceph
*Repeat this step for each node labeled: ``ceph-storage``*
Ceph Secrets Generation
-----------------------
Although you can bring your own secrets, we have conveniently created a
secret generation tool for you (for greenfield deployments). You can
create secrets for your project by issuing the following:
::
cd helm-toolkit/utils/secret-generator
./generate_secrets.sh all `./generate_secrets.sh fsid`
cd ../../..
export osd_cluster_network=192.168.0.0/16
export osd_public_network=192.168.0.0/16
Nova Compute Instance Storage
-----------------------------
@ -468,30 +354,55 @@ the following command to install Ceph:
::
helm install --set network.public=$osd_public_network --name=ceph local/ceph --namespace=ceph
helm install --namespace=ceph local/ceph --name=ceph \
--set manifests_enabled.client_secrets=false \
--set network.public=$osd_public_network \
--set network.cluster=$osd_cluster_network
Bootstrap Installation
----------------------
Activating Control-Plane Namespace for Ceph
-------------------------------------------
At this time (and before verification of Ceph) you'll need to install
the ``bootstrap`` chart. The ``bootstrap`` chart will install secrets
for both the ``ceph`` and ``openstack`` namespaces for the general
StorageClass:
In order for Ceph to fulfill PersistentVolumeClaims within Kubernetes namespaces
outside of Ceph's namespace, a client keyring needs to be present within that
namespace. For the rest of the OpenStack and supporting core services, this guide
will be deploying the control plane to a seperate namespace ``openstack``. To
deploy the aforementioned client keyring to the ``openstack`` namespace:
::
helm install --name=bootstrap-ceph local/bootstrap --namespace=ceph
helm install --name=bootstrap-openstack local/bootstrap --namespace=openstack
helm install --namespace=openstack local/ceph --name=ceph-openstack-config \
--set manifests_enabled.storage_secrets=false \
--set manifests_enabled.deployment=false \
--set ceph.namespace=ceph \
--set network.public=$osd_public_network \
--set network.cluster=$osd_cluster_network
This will load the client keyring as well as the same ``ceph.conf`` into
the specified namespace. Deploying ceph.conf into this namespace allows
OpenStack services to consume this ConfigMap for their Ceph-specific
configurations.
You may want to validate that Ceph is deployed successfully. For more
information on this, please see the section entitled `Ceph
Troubleshooting <../../operator/troubleshooting/persistent-storage.html>`__.
Ceph pool creation
------------------
You should now be ready to create the pools for OpenStack services to consume,
using the following commands:
::
kubectl exec -n ceph ceph-mon-0 -- ceph osd pool create volumes 8
kubectl exec -n ceph ceph-mon-0 -- ceph osd pool create images 8
kubectl exec -n ceph ceph-mon-0 -- ceph osd pool create vms 8
MariaDB Installation and Verification
-------------------------------------
We are using Galera to cluster MariaDB and establish a quorum. To
install the MariaDB, issue the following command:
We are using Galera to cluster MariaDB. To install MariaDB, issue the following
command:
::
@ -514,43 +425,49 @@ Now you can easily install the other services simply by going in order:
::
helm install --name=keystone local/keystone --set replicas=2 --namespace=openstack
helm install --namespace=openstack --name=keystone local/keystone \
--set replicas=2
**Install Horizon:**
::
helm install --name=horizon local/horizon --set network.enable_node_port=true --namespace=openstack
helm install --namespace=openstack --name=horizon local/horizon \
--set network.enable_node_port=true
**Install Glance:**
::
helm install --name=glance local/glance --set replicas.api=2,replicas.registry=2 --namespace=openstack
helm install --namespace=openstack --name=glance local/glance \
--set replicas.api=2,replicas.registry=2
**Install Heat:**
::
helm install --name=heat local/heat --namespace=openstack
helm install --namespace=openstack --name=heat local/heat
**Install Neutron:**
::
helm install --name=neutron local/neutron --set replicas.server=2 --namespace=openstack
helm install --namespace=openstack --name=neutron local/neutron \
--set replicas.server=2
**Install Nova:**
::
helm install --name=nova local/nova --set control_replicas=2 --namespace=openstack
helm install --namespace=openstack --name=nova local/nova \
--set control_replicas=2
**Install Cinder:**
::
helm install --name=cinder local/cinder --set replicas.api=2 --namespace=openstack
helm install --namespace=openstack --name=cinder local/cinder \
--set replicas.api=2
Final Checks
------------

11
glance/templates/etc/_ceph.client.glance.keyring.tpl → glance/templates/bin/_ceph-keyring.sh.tpl
View File

@ -1,3 +1,5 @@
#!/bin/bash
# Copyright 2017 The Openstack-Helm Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
@ -12,9 +14,16 @@
# See the License for the specific language governing permissions and
# limitations under the License.
set -ex
export HOME=/tmp
cat <<EOF > /etc/ceph/ceph.client.{{ .Values.conf.glance.glance_store.glance.store.rbd_store_user }}.keyring
[client.{{ .Values.conf.glance.glance_store.glance.store.rbd_store_user }}]
{{- if .Values.conf.ceph.keyring }}
key = {{ .Values.conf.ceph.keyring }}
{{- else }}
key = {{- include "secrets/ceph-client-key" . -}}
key = $(cat /tmp/client-keyring)
{{- end }}
EOF
exit 0

2
glance/templates/configmap-bin.yaml
View File

@ -35,3 +35,5 @@ data:
{{ tuple "bin/_glance-registry.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
bootstrap.sh: |+
{{ tuple "bin/_bootstrap.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
ceph-keyring.sh: |+
{{ tuple "bin/_ceph-keyring.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}

24
glance/templates/configmap-etc.yaml
View File

@ -78,30 +78,6 @@ data:
{{- end }}
{{- if .Values.conf.rally_tests.append -}}
{{ .Values.conf.rally_tests.append | indent 4 }}
{{- end }}
ceph.conf: |+
{{ if .Values.conf.ceph.override -}}
{{ .Values.conf.ceph.override | indent 4 }}
{{- else -}}
{{- if .Values.conf.ceph.prefix -}}
{{ .Values.conf.ceph.prefix | indent 4 }}
{{- end }}
{{ tuple "etc/_ceph.conf.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
{{- end }}
{{- if .Values.conf.ceph.append -}}
{{ .Values.conf.ceph.append | indent 4 }}
{{- end }}
ceph.client.{{ .Values.conf.glance.glance_store.glance.store.rbd_store_user }}.keyring: |+
{{ if .Values.conf.ceph_client.override -}}
{{ .Values.conf.ceph_client.override | indent 4 }}
{{- else -}}
{{- if .Values.conf.ceph_client.prefix -}}
{{ .Values.conf.ceph_client.prefix | indent 4 }}
{{- end }}
{{ tuple "etc/_ceph.client.glance.keyring.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
{{- end }}
{{- if .Values.conf.ceph_client.append -}}
{{ .Values.conf.ceph_client.append | indent 4 }}
{{- end }}
glance-api.conf: |+
{{ if .Values.conf.glance.override -}}

60
glance/templates/deployment-api.yaml
View File

@ -43,6 +43,47 @@ spec:
terminationGracePeriodSeconds: {{ .Values.termination_grace_period.api.timeout | default "600" }}
initContainers:
{{ tuple $envAll $dependencies $mounts_glance_api_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
{{- if eq .Values.storage "pvc" }}
- name: glance-perms
image: {{ .Values.images.api }}
imagePullPolicy: {{ .Values.images.pull_policy }}
securityContext:
runAsUser: 0
{{- if .Values.resources.enabled }}
resources:
limits:
cpu: {{ .Values.resources.api.limits.cpu | quote }}
memory: {{ .Values.resources.api.limits.memory | quote }}
requests:
cpu: {{ .Values.resources.api.requests.cpu | quote }}
memory: {{ .Values.resources.api.requests.memory | quote }}
{{- end }}
command:
- chown
- -R
- "glance:"
- {{ .Values.conf.glance.glance_store.glance.store.filesystem_store_datadir }}
volumeMounts:
- name: glance-images
mountPath: {{ .Values.conf.glance.glance_store.glance.store.filesystem_store_datadir }}
{{- end }}
{{ if eq .Values.storage "ceph" }}
- name: ceph-keyring-placement
image: {{ .Values.images.api }}
imagePullPolicy: {{ .Values.images.pull_policy }}
command:
- /tmp/ceph-keyring.sh
volumeMounts:
- name: etcceph
mountPath: /etc/ceph
- name: glance-bin
mountPath: /tmp/ceph-keyring.sh
subPath: ceph-keyring.sh
- name: ceph-keyring
mountPath: /tmp/client-keyring
subPath: key
readOnly: true
{{ end }}
containers:
- name: glance-api
image: {{ .Values.images.api }}
@ -93,13 +134,15 @@ spec:
- name: glance-images
mountPath: {{ .Values.conf.glance.glance_store.glance.store.filesystem_store_datadir }}
{{- else }}
- name: glance-etc
- name: etcceph
mountPath: /etc/ceph
- name: ceph-etc
mountPath: /etc/ceph/ceph.conf
subPath: ceph.conf
readOnly: true
- name: glance-etc
mountPath: /etc/ceph/ceph.client.{{ .Values.conf.glance.glance_store.glance.store.rbd_store_user }}.keyring
subPath: ceph.client.{{ .Values.conf.glance.glance_store.glance.store.rbd_store_user }}.keyring
- name: ceph-keyring
mountPath: /tmp/client-keyring
subPath: key
readOnly: true
{{- end }}
{{ if $mounts_glance_api.volumeMounts }}{{ toYaml $mounts_glance_api.volumeMounts | indent 12 }}{{ end }}
@ -117,5 +160,14 @@ spec:
- name: glance-images
persistentVolumeClaim:
claimName: glance-images
{{ else }}
- name: etcceph
emptyDir: {}
- name: ceph-etc
configMap:
name: ceph-etc
- name: ceph-keyring
secret:
secretName: pvc-ceph-client-key
{{- end }}
{{ if $mounts_glance_api.volumes }}{{ toYaml $mounts_glance_api.volumes | indent 8 }}{{ end }}

30
glance/templates/etc/_ceph.conf.tpl
View File

@ -1,30 +0,0 @@
# Copyright 2017 The Openstack-Helm Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
[global]
rgw_thread_pool_size = 1024
rgw_num_rados_handles = 100
{{- if .Values.conf.ceph.monitors }}
[mon]
{{ range .Values.conf.ceph.monitors }}
[mon.{{ . }}]
host = {{ . }}
mon_addr = {{ . }}
{{ end }}
{{- else }}
mon_host = ceph-mon.ceph
{{- end }}
[client]
rbd_cache_enabled = true
rbd_cache_writethrough_until_flush = true

0
helm-toolkit/secrets/.gitkeep
View File

78
helm-toolkit/utils/secret-generator/README.rst
View File

@ -1,78 +0,0 @@
Ceph Kubernetes Secret Generation
=================================
This script will generate ceph keyrings and configs as Kubernetes
secrets.
Sigil is required for template handling and must be installed in system
``PATH``. Instructions can be found`here
<https://github.com/gliderlabs/sigil>`__
The following functions are provided:
Generate raw FSID (can be used for other functions)
---------------------------------------------------
.. code:: bash
./generate_secrets.sh fsid
Generate raw ceph.conf (For verification)
-----------------------------------------
.. code:: bash
./generate_secrets.sh ceph-conf-raw <fsid> "overridekey=value"
Take a look at ``ceph/ceph.conf.tmpl`` for the default values
Generate encoded ceph.conf secret
---------------------------------
.. code:: bash
./generate_secrets.sh ceph-conf <fsid> "overridekey=value"
Generate encoded admin keyring secret
-------------------------------------
.. code:: bash
./generate_secrets.sh admin-keyring
Generate encoded mon keyring secret
-----------------------------------
.. code:: bash
./generate_secrets.sh mon-keyring
Generate a combined secret
--------------------------
Contains ceph.conf, admin keyring and mon keyring. Useful for generating
the ``/etc/ceph`` directory
.. code:: bash
./generate_secrets.sh combined-conf
Generate encoded boostrap keyring secret
----------------------------------------
.. code:: bash
./generate_secrets.sh bootstrap-keyring <osd|mds|rgw>
Kubernetes workflow
===================
.. code:: bash
./generator/generate_secrets.sh all `./generate_secrets.sh fsid`
kubectl create secret generic ceph-conf-combined --from-file=ceph.conf --from-file=ceph.client.admin.keyring --from-file=ceph.mon.keyring --namespace=ceph
kubectl create secret generic ceph-bootstrap-rgw-keyring --from-file=ceph.keyring=ceph.rgw.keyring --namespace=ceph
kubectl create secret generic ceph-bootstrap-mds-keyring --from-file=ceph.keyring=ceph.mds.keyring --namespace=ceph
kubectl create secret generic ceph-bootstrap-osd-keyring --from-file=ceph.keyring=ceph.osd.keyring --namespace=ceph
kubectl create secret generic ceph-client-key --from-file=ceph-client-key --namespace=ceph

96
helm-toolkit/utils/secret-generator/generate_secrets.sh
View File

@ -1,96 +0,0 @@
#!/bin/bash
# Copyright 2017 The Openstack-Helm Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
gen-fsid() {
echo "$(uuidgen)"
}
gen-ceph-conf-raw() {
fsid=${1:?}
shift
conf=$(sigil -p -f templates/ceph/ceph.conf.tmpl "fsid=${fsid}" $@)
echo "${conf}"
}
gen-ceph-conf() {
fsid=${1:?}
shift
conf=$(sigil -p -f templates/ceph/ceph.conf.tmpl "fsid=${fsid}" $@)
echo "${conf}"
}
gen-admin-keyring() {
key=$(python ceph-key.py)
keyring=$(sigil -f templates/ceph/admin.keyring.tmpl "key=${key}")
echo "${keyring}"
}
gen-mon-keyring() {
key=$(python ceph-key.py)
keyring=$(sigil -f templates/ceph/mon.keyring.tmpl "key=${key}")
echo "${keyring}"
}
gen-combined-conf() {
fsid=${1:?}
shift
conf=$(sigil -p -f templates/ceph/ceph.conf.tmpl "fsid=${fsid}" $@)
echo "${conf}" > ../../secrets/ceph.conf
key=$(python ceph-key.py)
keyring=$(sigil -f templates/ceph/admin.keyring.tmpl "key=${key}")
echo "${key}" > ../../secrets/ceph-client-key
echo "${keyring}" > ../../secrets/ceph.client.admin.keyring
key=$(python ceph-key.py)
keyring=$(sigil -f templates/ceph/mon.keyring.tmpl "key=${key}")
echo "${keyring}" > ../../secrets/ceph.mon.keyring
}
gen-bootstrap-keyring() {
service="${1:-osd}"
key=$(python ceph-key.py)
bootstrap=$(sigil -f templates/ceph/bootstrap.keyring.tmpl "key=${key}" "service=${service}")
echo "${bootstrap}"
}
gen-all-bootstrap-keyrings() {
gen-bootstrap-keyring osd > ../../secrets/ceph.osd.keyring
gen-bootstrap-keyring mds > ../../secrets/ceph.mds.keyring
gen-bootstrap-keyring rgw > ../../secrets/ceph.rgw.keyring
}
gen-all() {
gen-combined-conf $@
gen-all-bootstrap-keyrings
}
main() {
set -eo pipefail
case "$1" in
fsid) shift; gen-fsid $@;;
ceph-conf-raw) shift; gen-ceph-conf-raw $@;;
ceph-conf) shift; gen-ceph-conf $@;;
admin-keyring) shift; gen-admin-keyring $@;;
mon-keyring) shift; gen-mon-keyring $@;;
bootstrap-keyring) shift; gen-bootstrap-keyring $@;;
combined-conf) shift; gen-combined-conf $@;;
all) shift; gen-all $@;;
esac
}
main "$@"

20
helm-toolkit/utils/secret-generator/templates/ceph/admin.keyring.tmpl
View File

@ -1,20 +0,0 @@
# Copyright 2017 The Openstack-Helm Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
[client.admin]
key = {{ $key }}
auid = 0
caps mds = "allow"
caps mon = "allow *"
caps osd = "allow *"

17
helm-toolkit/utils/secret-generator/templates/ceph/bootstrap.keyring.tmpl
View File

@ -1,17 +0,0 @@
# Copyright 2017 The Openstack-Helm Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
[client.bootstrap-{{ $service }}]
key = {{ $key }}
caps mon = "allow profile bootstrap-{{ $service }}"

85
helm-toolkit/utils/secret-generator/templates/ceph/ceph.conf.tmpl
View File

@ -1,85 +0,0 @@
# Copyright 2017 The Openstack-Helm Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
[global]
fsid = ${fsid:?}
cephx = ${auth_cephx:-"true"}
cephx_require_signatures = ${auth_cephx_require_signatures:-"false"}
cephx_cluster_require_signatures = ${auth_cephx_cluster_require_signatures:-"true"}
cephx_service_require_signatures = ${auth_cephx_service_require_signatures:-"false"}
# auth
max_open_files = ${global_max_open_files:-"131072"}
osd_pool_default_pg_num = ${global_osd_pool_default_pg_num:-"128"}
osd_pool_default_pgp_num = ${global_osd_pool_default_pgp_num:-"128"}
osd_pool_default_size = ${global_osd_pool_default_size:-"3"}
osd_pool_default_min_size = ${global_osd_pool_default_min_size:-"1"}
mon_osd_full_ratio = ${global_mon_osd_full_ratio:-".95"}
mon_osd_nearfull_ratio = ${global_mon_osd_nearfull_ratio:-".85"}
mon_host = ${global_mon_host:-'ceph-mon'}
[mon]
mon_osd_down_out_interval = ${mon_mon_osd_down_out_interval:-"600"}
mon_osd_min_down_reporters = ${mon_mon_osd_min_down_reporters:-"4"}
mon_clock_drift_allowed = ${mon_mon_clock_drift_allowed:-".15"}
mon_clock_drift_warn_backoff = ${mon_mon_clock_drift_warn_backoff:-"30"}
mon_osd_report_timeout = ${mon_mon_osd_report_timeout:-"300"}
[osd]
journal_size = ${osd_journal_size:-"100"}
cluster_network = ${osd_cluster_network:-'192.168.0.0/16'}
public_network = ${osd_public_network:-'192.168.0.0/16'}
osd_mkfs_type = ${osd_osd_mkfs_type:-"xfs"}
osd_mkfs_options_xfs = ${osd_osd_mkfs_options_xfs:-"-f -i size=2048"}
osd_mon_heartbeat_interval = ${osd_osd_mon_heartbeat_interval:-"30"}
osd_max_object_name_len = ${osd_max_object_name_len:-"256"}
#crush
osd_pool_default_crush_rule = ${osd_pool_default_crush_rule:-"0"}
osd_crush_update_on_start = ${osd_osd_crush_update_on_start:-"true"}
#backend
osd_objectstore = ${osd_osd_objectstore:-"filestore"}
#performance tuning
filestore_merge_threshold = ${osd_filestore_merge_threshold:-"40"}
filestore_split_multiple = ${osd_filestore_split_multiple:-"8"}
osd_op_threads = ${osd_osd_op_threads:-"8"}
filestore_op_threads = ${osd_filestore_op_threads:-"8"}
filestore_max_sync_interval = ${osd_filestore_max_sync_interval:-"5"}
osd_max_scrubs = ${osd_osd_max_scrubs:-"1"}
#recovery tuning
osd_recovery_max_active = ${osd_osd_recovery_max_active:-"5"}
osd_max_backfills = ${osd_osd_max_backfills:-"2"}
osd_recovery_op_priority = ${osd_osd_recovery_op_priority:-"2"}
osd_client_op_priority = ${osd_osd_client_op_priority:-"63"}
osd_recovery_max_chunk = ${osd_osd_recovery_max_chunk:-"1048576"}
osd_recovery_threads = ${osd_osd_recovery_threads:-"1"}
#ports
ms_bind_port_min = ${osd_ms_bind_port_min:-"6800"}
ms_bind_port_max = ${osd_ms_bind_port_max:-"7100"}
[client]
rbd_cache_enabled = ${client_rbd_cache_enabled:-"true"}
rbd_cache_writethrough_until_flush = ${client_rbd_cache_writethrough_until_flush:-"true"}
rbd_default_features = ${client_rbd_default_features:-"1"}
[mds]
mds_cache_size = ${mds_mds_cache_size:-"100000"}

17
helm-toolkit/utils/secret-generator/templates/ceph/mon.keyring.tmpl
View File

@ -1,17 +0,0 @@
# Copyright 2017 The Openstack-Helm Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
[mon.]
key = {{ $key }}
caps mon = "allow *"

40
helm-toolkit/utils/test/ceph-rbd-test.yaml
View File

@ -1,40 +0,0 @@
# Copyright 2017 The Openstack-Helm Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
apiVersion: v1
kind: Pod
metadata:
name: ceph-rbd-test
spec:
containers:
- name: cephrbd-rw
image: busybox
command:
- sh
- -c
- while true; do sleep 1; done
volumeMounts:
- mountPath: "/mnt/cephrbd"
name: cephrbd
volumes:
- name: cephrbd
rbd:
monitors:
#This only works if you have skyDNS resolveable from the kubernetes node. Otherwise you must manually put in one or more mon pod ips.
- ceph-mon.ceph:6789
user: admin
image: ceph-rbd-test
pool: rbd
secretRef:
name: ceph-client-key

11
nova/templates/etc/_ceph.client.cinder.keyring.yaml.tpl → nova/templates/bin/_ceph-keyring.sh.tpl
View File

@ -1,3 +1,5 @@
#!/bin/bash
# Copyright 2017 The Openstack-Helm Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
@ -12,9 +14,16 @@
# See the License for the specific language governing permissions and
# limitations under the License.
set -ex
export HOME=/tmp
cat <<EOF > /etc/ceph/ceph.client.keyring
[client.{{ .Values.ceph.cinder_user }}]
{{- if .Values.ceph.cinder_keyring }}
key = {{ .Values.ceph.cinder_keyring }}
{{- else }}
key = {{- include "secrets/ceph-client-key" . -}}
key = $(cat /tmp/client-keyring)
{{- end }}
EOF
exit 0

4
nova/templates/configmap-bin.yaml
View File

@ -33,7 +33,7 @@ data:
{{ tuple "bin/_libvirt.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
{{- if .Values.ceph.enabled }}
ceph-secret-define.sh: |
{{ tuple "bin/_ceph-secret-define.sh.tpl" . | include "helm-toolkit.template" | indent 4 }}
{{ tuple "bin/_ceph-secret-define.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
{{- end }}
bootstrap.sh: |
{{ tuple "bin/_bootstrap.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
@ -51,3 +51,5 @@ data:
{{ tuple "bin/_nova-scheduler.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
fake-iptables.sh: |
{{ tuple "bin/_fake-iptables.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
ceph-keyring.sh: |+
{{ tuple "bin/_ceph-keyring.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}

2
nova/templates/configmap-etc.yaml
View File

@ -92,8 +92,6 @@ data:
{{- else -}}
{{ tuple "etc/_policy.yaml.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
{{- end }}
ceph.client.cinder.keyring.yaml: |+
{{ tuple "etc/_ceph.client.cinder.keyring.yaml.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
resolv.conf: |+
{{ tuple "etc/_resolv.conf.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
libvirtd.conf: |+

39
nova/templates/daemonset-compute.yaml
View File

@ -39,6 +39,23 @@ spec:
dnsPolicy: ClusterFirst
initContainers:
{{ tuple $envAll $dependencies $mounts_nova_compute_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
{{- if .Values.ceph.enabled }}
- name: ceph-keyring-placement
image: {{ .Values.images.compute }}
imagePullPolicy: {{ .Values.images.pull_policy }}
command:
- /tmp/ceph-keyring.sh
volumeMounts:
- name: etcceph
mountPath: /etc/ceph
- name: nova-bin
mountPath: /tmp/ceph-keyring.sh
subPath: ceph-keyring.sh
- name: ceph-keyring
mountPath: /tmp/client-keyring
subPath: key
readOnly: true
{{ end }}
containers:
- name: nova-compute
image: {{ .Values.images.compute }}
@ -77,14 +94,18 @@ spec:
mountPath: /etc/resolv.conf
subPath: resolv.conf
readOnly: true
- name: nova-etc
{{- if .Values.ceph.enabled }}
- name: etcceph
mountPath: /etc/ceph
- name: ceph-etc
mountPath: /etc/ceph/ceph.conf
subPath: ceph.conf
readOnly: true
- name: nova-etc
mountPath: /etc/ceph/ceph.client.keyring
subPath: ceph.client.keyring
- name: ceph-keyring
mountPath: /tmp/client-keyring
subPath: key
readOnly: true
{{ end }}
- mountPath: /lib/modules
name: libmodules
readOnly: true
@ -105,6 +126,16 @@ spec:
- name: nova-etc
configMap:
name: nova-etc
{{- if .Values.ceph.enabled }}
- name: etcceph
emptyDir: {}
- name: ceph-etc
configMap:
name: ceph-etc
- name: ceph-keyring
secret:
secretName: pvc-ceph-client-key
{{ end }}
- name: libmodules
hostPath:
path: /lib/modules

37
nova/templates/daemonset-libvirt.yaml
View File

@ -38,6 +38,23 @@ spec:
dnsPolicy: ClusterFirst
initContainers:
{{ tuple $envAll $dependencies $mounts_nova_libvirt_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }}
{{- if .Values.ceph.enabled }}
- name: ceph-keyring-placement
image: {{ .Values.images.libvirt }}
imagePullPolicy: {{ .Values.images.pull_policy }}
command:
- /tmp/ceph-keyring.sh
volumeMounts:
- name: etcceph
mountPath: /etc/ceph
- name: nova-bin
mountPath: /tmp/ceph-keyring.sh
subPath: ceph-keyring.sh
- name: ceph-keyring
mountPath: /tmp/client-keyring
subPath: key
readOnly: true
{{ end }}
containers:
- name: nova-libvirt
image: {{ .Values.images.libvirt }}
@ -101,13 +118,15 @@ spec:
- name: cgroup
mountPath: /sys/fs/cgroup
{{- if .Values.ceph.enabled }}
- name: nova-etc
- name: etcceph
mountPath: /etc/ceph
- name: ceph-etc
mountPath: /etc/ceph/ceph.conf
subPath: ceph.conf
readOnly: true
- name: nova-etc
mountPath: /etc/ceph/ceph.client.keyring
subPath: ceph.client.keyring
- name: ceph-keyring
mountPath: /tmp/client-keyring
subPath: key
readOnly: true
- name: nova-bin
mountPath: /tmp/ceph-secret-define.sh
@ -123,6 +142,16 @@ spec:
- name: nova-etc
configMap:
name: nova-etc
{{- if .Values.ceph.enabled }}
- name: etcceph
emptyDir: {}
- name: ceph-etc
configMap:
name: ceph-etc
- name: ceph-keyring
secret:
secretName: pvc-ceph-client-key
{{ end }}
- name: libmodules
hostPath:
path: /lib/modules

32
nova/templates/etc/_ceph.conf.tpl
View File

@ -1,32 +0,0 @@
# Copyright 2017 The Openstack-Helm Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
[global]
rgw_thread_pool_size = 1024
rgw_num_rados_handles = 100
{{- if .Values.ceph.enabled }}
[mon]
{{- if .Values.ceph.monitors }}
{{ range .Values.ceph.monitors }}
[mon.{{ . }}]
host = {{ . }}
mon_addr = {{ . }}
{{ end }}
{{- else }}
mon_host = ceph-mon.ceph
{{- end }}
{{- end }}
[client]
rbd_cache_enabled = true
rbd_cache_writethrough_until_flush = true

1
nova/values.yaml
View File

@ -130,7 +130,6 @@ keystone:
admin_region_name: "RegionOne"
nova_user_role: "admin"
ceph:
enabled: false
monitors: []

35
tests/pvc-test.yaml
View File

@ -12,15 +12,42 @@
# See the License for the specific language governing permissions and
# limitations under the License.
---
apiVersion: batch/v1
kind: Job
metadata:
labels:
version: v0.1.0
test: ceph
name: ceph-test-job
spec:
template:
spec:
restartPolicy: OnFailure
containers:
- name: test
image: docker.io/alpine:latest
imagePullPolicy: Always
command:
- /bin/sh
- -ec
- |
echo "Ceph PVC Mount Test Passed"
volumeMounts:
- name: ceph-mount
mountPath: /mnt/ceph
volumes:
- name: ceph-mount
persistentVolumeClaim:
claimName: ceph-test
---
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
name: pvc-test
name: ceph-test
spec:
accessModes:
- ReadWriteOnce
storageClassName: general
accessModes: [ "ReadWriteOnce" ]
resources:
requests:
storage: 1Gi
storageClassName: general

1
tools/gate/README.rst
View File

@ -17,6 +17,7 @@ integration test is below:
export INTEGRATION=aio
export INTEGRATION_TYPE=basic
export PVC_BACKEND=ceph
./tools/gate/setup_gate.sh
Supported Platforms

44
tools/gate/basic_launch.sh
View File

@ -29,14 +29,53 @@ if [ "x$HOST_OS" == "xfedora" ]; then
sudo modprobe ip6_tables
fi
if [ "x$PVC_BACKEND" == "xceph" ]; then
kubectl label nodes ceph-storage=enabled --all
CONTROLLER_MANAGER_POD=$(kubectl get -n kube-system pods -l component=kube-controller-manager --no-headers -o name | head -1 | awk -F '/' '{ print $NF }')
kubectl exec -n kube-system ${CONTROLLER_MANAGER_POD} -- sh -c "cat > /etc/resolv.conf <<EOF
nameserver 10.96.0.10
nameserver 8.8.8.8
search cluster.local svc.cluster.local
EOF"
export osd_cluster_network=192.168.0.0/16
export osd_public_network=192.168.0.0/16
helm install --namespace=ceph local/ceph --name=ceph2 \
--set manifests_enabled.client_secrets=false \
--set network.public=$osd_public_network \
--set network.cluster=$osd_cluster_network
kube_wait_for_pods ceph 420
kubectl exec -n ceph ceph-mon-0 -- ceph -s
helm install --namespace=openstack local/ceph --name=ceph-openstack-config \
--set manifests_enabled.storage_secrets=false \
--set manifests_enabled.deployment=false \
--set ceph.namespace=ceph \
--set network.public=$osd_public_network \
--set network.cluster=$osd_cluster_network
kube_wait_for_pods ceph 420
kubectl exec -n ceph ceph-mon-0 -- ceph osd pool create volumes 8
kubectl exec -n ceph ceph-mon-0 -- ceph osd pool create images 8
kubectl exec -n ceph ceph-mon-0 -- ceph osd pool create vms 8
fi
helm install --namespace=openstack local/mariadb --name=mariadb
helm install --namespace=openstack local/memcached --name=memcached
helm install --namespace=openstack local/etcd --name=etcd-rabbitmq
helm install --namespace=openstack local/rabbitmq --name=rabbitmq
kube_wait_for_pods openstack 420
helm install --namespace=openstack local/keystone --name=keystone
if [ "x$PVC_BACKEND" == "xceph" ]; then
helm install --namespace=openstack local/glance --name=glance
else
helm install --namespace=openstack local/glance --name=glance \
--values=${WORK_DIR}/tools/overrides/mvp/glance.yaml
fi
kube_wait_for_pods openstack 420
helm install --namespace=openstack local/nova --name=nova \
--values=${WORK_DIR}/tools/overrides/mvp/nova.yaml \
@ -44,7 +83,12 @@ helm install --namespace=openstack local/nova --name=nova \
helm install --namespace=openstack local/neutron --name=neutron \
--values=${WORK_DIR}/tools/overrides/mvp/neutron.yaml
kube_wait_for_pods openstack 420
if [ "x$PVC_BACKEND" == "xceph" ]; then
helm install --namespace=openstack local/cinder --name=cinder
else
helm install --namespace=openstack local/cinder --name=cinder \
--values=${WORK_DIR}/tools/overrides/mvp/cinder.yaml
fi
helm install --namespace=openstack local/heat --name=heat
helm install --namespace=openstack local/horizon --name=horizon
kube_wait_for_pods openstack 420

10
tools/gate/dump_logs.sh
View File

@ -53,6 +53,15 @@ kubectl get svc -o json --all-namespaces | jq -r \
${LOGS_DIR}/k8s/svc/$NAMESPACE-$NAME.txt
done
mkdir -p ${LOGS_DIR}/k8s/pvc
kubectl get pvc -o json --all-namespaces | jq -r \
'.items[].metadata | .namespace + " " + .name' | while read line; do
NAMESPACE=$(echo $line | awk '{print $1}')
NAME=$(echo $line | awk '{print $2}')
kubectl describe pvc $NAME --namespace $NAMESPACE > \
${LOGS_DIR}/k8s/pvc/$NAMESPACE-$NAME.txt
done
mkdir -p ${LOGS_DIR}/k8s/rbac
for OBJECT_TYPE in clusterroles \
roles \
@ -76,5 +85,6 @@ sudo iptables-save > ${LOGS_DIR}/nodes/$(hostname)/iptables.txt
sudo ip a > ${LOGS_DIR}/nodes/$(hostname)/ip.txt
sudo route -n > ${LOGS_DIR}/nodes/$(hostname)/routes.txt
arp -a > ${LOGS_DIR}/nodes/$(hostname)/arp.txt
cat /etc/resolv.conf > ${LOGS_DIR}/nodes/$(hostname)/resolv.conf
exit $1

14
tools/gate/funcs/common.sh
View File

@ -29,3 +29,17 @@ function base_install {
iptables
fi
}
function ceph_support_install {
if [ "x$HOST_OS" == "xubuntu" ]; then
sudo apt-get update -y
sudo apt-get install -y --no-install-recommends -qq \
ceph-common
elif [ "x$HOST_OS" == "xcentos" ]; then
sudo yum install -y \
ceph
elif [ "x$HOST_OS" == "xfedora" ]; then
sudo dnf install -y \
ceph
fi
}

6
tools/gate/funcs/kube.sh
View File

@ -130,3 +130,9 @@ function kubeadm_aio_launch {
kube_wait_for_pods kube-system 240
kube_wait_for_pods default 240
}
function ceph_kube_controller_manager_replace {
sudo docker pull ${CEPH_KUBE_CONTROLLER_MANAGER_IMAGE}
IMAGE_ID=$(sudo docker images ${CEPH_KUBE_CONTROLLER_MANAGER_IMAGE} -q)
sudo docker tag ${IMAGE_ID} ${BASE_KUBE_CONTROLLER_MANAGER_IMAGE}
}

1
tools/gate/funcs/network.sh
View File

@ -42,3 +42,4 @@ function net_hosts_pre_kube {
function net_hosts_post_kube {
sudo cp -f /etc/hosts-pre-kube /etc/hosts
}

6
tools/gate/kubeadm_aio.sh
View File

@ -18,5 +18,11 @@ source ${WORK_DIR}/tools/gate/funcs/kube.sh
kubeadm_aio_reqs_install
sudo docker pull ${KUBEADM_IMAGE} || kubeadm_aio_build
if [ "x$PVC_BACKEND" == "xceph" ]; then
ceph_kube_controller_manager_replace
sudo modprobe rbd
fi
kubeadm_aio_launch
net_resolv_kube

7
tools/gate/provision_gate_worker_node.sh
View File

@ -37,8 +37,15 @@ sudo rm -rfv \
/var/lib/etcd \
/var/etcd \
/var/lib/kubelet/* \
/var/lib/nova \
/var/lib/openstack-helm \
/run/openvswitch || true
# Load ceph kernel module if required
if [ "x$PVC_BACKEND" == "xceph" ]; then
sudo modprobe rbd
fi
# Launch Container
sudo docker run \
-dt \

16
tools/gate/setup_gate.sh
View File

@ -13,10 +13,12 @@
# limitations under the License.
set -ex
export HELM_VERSION=${2:-v2.3.1}
export HELM_VERSION=${2:-v2.4.1}
export KUBE_VERSION=${3:-v1.6.5}
export KUBECONFIG=${HOME}/.kubeadm-aio/admin.conf
export KUBEADM_IMAGE=openstackhelm/kubeadm-aio:${KUBE_VERSION}
export KUBEADM_IMAGE=openstackhelm/kubeadm-aio:${KUBE_VERSION}-ceph
export BASE_KUBE_CONTROLLER_MANAGER_IMAGE=gcr.io/google_containers/kube-controller-manager-amd64:${KUBE_VERSION}
export CEPH_KUBE_CONTROLLER_MANAGER_IMAGE=quay.io/attcomdev/kube-controller-manager:${KUBE_VERSION}
export WORK_DIR=$(pwd)
source /etc/os-release
@ -24,6 +26,7 @@ export HOST_OS=${ID}
source ${WORK_DIR}/tools/gate/funcs/common.sh
source ${WORK_DIR}/tools/gate/funcs/network.sh
source ${WORK_DIR}/tools/gate/funcs/helm.sh
export PVC_BACKEND=ceph
# Setup the logging location: by default use the working dir as the root.
export LOGS_DIR=${LOGS_DIR:-"${WORK_DIR}/logs"}
@ -34,14 +37,17 @@ function dump_logs () {
}
trap 'dump_logs "$?"' ERR
# Install base requirements
base_install
# Moving the ws-linter here to avoid it blocking all the jobs just for ws
if [ "x$INTEGRATION_TYPE" == "xlinter" ]; then
bash ${WORK_DIR}/tools/gate/whitespace.sh
fi
# Install base requirements
base_install
if [ "x$PVC_BACKEND" == "xceph" ]; then
ceph_support_install
fi
# We setup the network for pre kube here, to enable cluster restarts on
# development machines
net_resolv_pre_kube

1
tools/gate/setup_gate_worker_nodes.sh
View File

@ -31,6 +31,7 @@ cat /etc/nodepool/sub_nodes_private | while read SUB_NODE; do
export KUBEADM_TOKEN=${KUBEADM_TOKEN}; \
export PRIMARY_NODE_IP=${PRIMARY_NODE_IP}; \
export KUBEADM_IMAGE=${KUBEADM_IMAGE}; \
export PVC_BACKEND=${PVC_BACKEND}; \
bash ${WORK_DIR}/tools/gate/provision_gate_worker_node.sh"
EOS
done

20
tools/images/ceph-config-helper/Dockerfile Normal file
View File

@ -0,0 +1,20 @@
FROM ubuntu:16.04
MAINTAINER pete.birley@att.com
ARG KUBE_VERSION=v1.6.5
RUN set -x \
&& TMP_DIR=$(mktemp --directory) \
&& cd ${TMP_DIR} \
&& apt-get update \
&& apt-get install -y \
apt-transport-https \
ca-certificates \
curl \
python \
jq \
# Install kubectl:
&& curl -sSL https://dl.k8s.io/${KUBE_VERSION}/kubernetes-client-linux-amd64.tar.gz | tar -zxv --strip-components=1 \
&& mv ${TMP_DIR}/client/bin/kubectl /usr/bin/kubectl \
&& chmod +x /usr/bin/kubectl \
&& rm -rf ${TMP_DIR}

39
tools/images/ceph-config-helper/README.rst Normal file
View File

@ -0,0 +1,39 @@
Ceph Config Helper Container
=====================
This container builds a small image with kubectl and some other utilites for
use in the ceph-config chart.
Instructions
------------
OS Specific Host setup:
~~~~~~~~~~~~~~~~~~~~~~~
Ubuntu:
^^^^^^^aa
From a freshly provisioned Ubuntu 16.04 LTS host run:
.. code:: bash
sudo apt-get update -y
sudo apt-get install -y \
docker.io \
git
Build the Ceph-Helper Image environment (optional)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
A known good image is published to dockerhub on a fairly regular basis, but if
you wish to build your own image, from the root directory of the OpenStack-Helm
repo run:
.. code:: bash
export KUBE_VERSION=v1.6.5
sudo docker build \
--build-arg KUBE_VERSION=${KUBE_VERSION} \
-t docker.io/port/ceph-config-helper:${KUBE_VERSION} \
tools/images/ceph-config-helper
sudo docker push docker.io/port/ceph-config-helper:${KUBE_VERSION}

3
tools/kubeadm-aio/Dockerfile
View File

@ -61,8 +61,9 @@ RUN set -x \
git \
vim \
jq \
# Install nfs utils for development PVC provisioner
# Install utils for PVC provisioners
nfs-common \
ceph-common \
# Tweak Systemd units and targets for running in a container
&& find /lib/systemd/system/sysinit.target.wants/ ! -name 'systemd-tmpfiles-setup.service' -type l -exec rm -fv {} + \
&& rm -fv \

2
tools/kubeadm-aio/README.rst
View File

@ -57,7 +57,7 @@ repo run:
.. code:: bash
export KUBEADM_IMAGE=openstackhelm/kubeadm-aio:v1.6.4
export KUBEADM_IMAGE=openstackhelm/kubeadm-aio:v1.6.5
sudo docker build --pull -t ${KUBEADM_IMAGE} tools/kubeadm-aio
Deploy the AIO environment

3
tools/kubeadm-aio/assets/usr/bin/kubelet
View File

@ -36,7 +36,8 @@ else
--restart=always \
--volume=/sys/fs/cgroup:/sys/fs/cgroup:ro \
--volume=/:/rootfs:ro \
--volume=/dev/net:/dev/net:rw \
--volume=/dev:/dev:rshared \
--volume=/lib/modules:/lib/modules:ro \
--volume=/var/run/netns:/var/run/netns:rw \
--volume=/sys:/sys:ro \
--volume=/etc/machine-id:/etc/machine-id:ro \

1
tools/kubeadm-aio/kubeadm-aio-launcher.sh
View File

@ -33,6 +33,7 @@ sudo rm -rfv \
/run/openvswitch \
/var/lib/nova \
${HOME}/.kubeadm-aio/admin.conf \
/var/lib/openstack-helm \
/var/lib/nfs-provisioner || true
# Launch Container

30
helm-toolkit/utils/secret-generator/ceph-key.py → tools/overrides/mvp/cinder.yaml
View File

@ -1,5 +1,3 @@
#!/bin/python
# Copyright 2017 The Openstack-Helm Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
@ -14,17 +12,19 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import struct
import time
import base64
# MVP values for glance.
# This file contains overrides to launch a MVP deployment of glance for the
# OpenStack-Helm Single node gates, and local development use. It should be
# kept to the bare minimum required for this purpose.
storage: "gate"
conf:
cinder:
default:
cinder:
backup_driver: "cinder.backup.drivers.nfs"
backends:
rbd1:
volume_driver: "cinder.volume.drivers.nfs.NfsDriver"
key = os.urandom(16)
header = struct.pack(
'<hiih',
1, # le16 type: CEPH_CRYPTO_AES
int(time.time()), # le32 created: seconds
0, # le32 created: nanoseconds,
len(key), # le16: len(key)
)
print(base64.b64encode(header + key).decode('ascii'))

3
tools/overrides/mvp/nova.yaml
View File

@ -12,6 +12,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
ceph:
enabled: false
conf:
nova:
default: