diff --git a/mariadb/Chart.yaml b/mariadb/Chart.yaml index 736222e56..e472e5812 100644 --- a/mariadb/Chart.yaml +++ b/mariadb/Chart.yaml @@ -15,7 +15,7 @@ apiVersion: v1 appVersion: v10.6.7 description: OpenStack-Helm MariaDB name: mariadb -version: 0.2.44 +version: 0.2.45 home: https://mariadb.com/kb/en/ icon: http://badges.mariadb.org/mariadb-badge-180x60.png sources: diff --git a/mariadb/templates/bin/_mariadb_controller.py.tpl b/mariadb/templates/bin/_mariadb_controller.py.tpl new file mode 100644 index 000000000..faf5195a5 --- /dev/null +++ b/mariadb/templates/bin/_mariadb_controller.py.tpl @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 + +""" +Mariadb controller + +The script is responsible for set mariadb_role: primary to first +active pod in mariadb deployment. + +Env variables: +MARIADB_CONTROLLER_DEBUG: Flag to enable debug when set to 1. +MARIADB_CONTROLLER_CHECK_PODS_DELAY: The delay between check pod attempts. +MARIADB_CONTROLLER_PYKUBE_REQUEST_TIMEOUT: The timeout for kubernetes http session +MARIADB_CONTROLLER_PODS_NAMESPACE: The namespace to look for mariadb pods. +MARIADB_MASTER_SERVICE_NAME: The name of master service for mariadb. + +Changelog: +0.1.0: Initial varsion +""" + + +import logging +import os +import sys +import time + +import pykube + +MARIADB_CONTROLLER_DEBUG = os.getenv("MARIADB_CONTROLLER_DEBUG") +MARIADB_CONTROLLER_CHECK_PODS_DELAY = int( + os.getenv("MARIADB_CONTROLLER_CHECK_PODS_DELAY", 10) +) +MARIADB_CONTROLLER_PYKUBE_REQUEST_TIMEOUT = int( + os.getenv("MARIADB_CONTROLLER_PYKUBE_REQUEST_TIMEOUT", 60) +) +MARIADB_CONTROLLER_PODS_NAMESPACE = os.getenv( + "MARIADB_CONTROLLER_PODS_NAMESPACE", "openstack" +) +MARIADB_MASTER_SERVICE_NAME = os.getenv( + "MARIADB_MASTER_SERVICE_NAME", "mariadb" +) + +log_level = "DEBUG" if MARIADB_CONTROLLER_DEBUG else "INFO" +logging.basicConfig( + stream=sys.stdout, + format="%(asctime)s %(levelname)s %(name)s %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", +) +LOG = logging.getLogger("mariadb-controller") + +LOG.setLevel(log_level) + + +def login(): + config = pykube.KubeConfig.from_env() + client = pykube.HTTPClient( + config=config, timeout=MARIADB_CONTROLLER_PYKUBE_REQUEST_TIMEOUT + ) + LOG.info(f"Created k8s api client from context {config.current_context}") + return client + + +api = login() + + +def resource_list(klass, selector, namespace=None): + return klass.objects(api).filter(namespace=namespace, selector=selector) + + +def get_mariadb_pods(): + sorted_pods = sorted( + resource_list( + pykube.Pod, + {"application": "mariadb", "component": "server"}, + MARIADB_CONTROLLER_PODS_NAMESPACE, + ).iterator(), + key=lambda i: i.name, + ) + return sorted_pods + + +def get_mariadb_master_service(namespace): + return pykube.Service.objects(api).filter(namespace=namespace).get(name=MARIADB_MASTER_SERVICE_NAME) + + +def link_master_service(pod): + svc = get_mariadb_master_service(MARIADB_CONTROLLER_PODS_NAMESPACE) + svc.reload() + if svc.obj['spec']['selector'].get('statefulset.kubernetes.io/pod-name') == pod.name: + LOG.debug(f"Nothing to do, master service points to {pod.name}") + else: + svc.obj['spec']['selector']['statefulset.kubernetes.io/pod-name'] = pod.name + svc.update() + LOG.info(f"Link master service with {pod.name}") + + +def is_ready(pod): + if pod.ready and "deletionTimestamp" not in pod.metadata: + return True + + +def main(): + while True: + for pod in get_mariadb_pods(): + pod.reload() + if is_ready(pod): + link_master_service(pod) + break + LOG.debug(f"Sleeping for {MARIADB_CONTROLLER_CHECK_PODS_DELAY}") + time.sleep(MARIADB_CONTROLLER_CHECK_PODS_DELAY) + + +main() diff --git a/mariadb/templates/configmap-bin.yaml b/mariadb/templates/configmap-bin.yaml index cc92eb69e..7b6e18ab2 100644 --- a/mariadb/templates/configmap-bin.yaml +++ b/mariadb/templates/configmap-bin.yaml @@ -53,4 +53,8 @@ data: ks-user.sh: | {{ include "helm-toolkit.scripts.keystone_user" . | indent 4 }} {{- end }} +{{- if .Values.manifests.deployment_controller }} + mariadb_controller.py: | +{{ tuple "bin/_mariadb_controller.py.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} +{{- end }} {{- end }} diff --git a/mariadb/templates/deployment-controller.yaml b/mariadb/templates/deployment-controller.yaml new file mode 100644 index 000000000..598d084a4 --- /dev/null +++ b/mariadb/templates/deployment-controller.yaml @@ -0,0 +1,119 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.deployment_controller }} +{{- if .Values.manifests.deployment_ingress }} +{{- fail ".Values.manifests.deployment_ingress and .Values.manifests.deployment_controlle are mutually exclusive" }} +{{- end }} +{{- $envAll := . }} + +{{- $serviceAccountName := "mariadb-controller" }} +{{ tuple $envAll "controller" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ $envAll.Release.Name }}-{{ $serviceAccountName }}-pod + namespace: {{ $envAll.Release.Namespace }} +rules: + - apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - apiGroups: + - "" + resources: + - services + verbs: + - update + - patch + - get + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ $envAll.Release.Name }}-{{ $serviceAccountName }}-pod + namespace: {{ $envAll.Release.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ $envAll.Release.Name }}-{{ $serviceAccountName }}-pod +subjects: + - kind: ServiceAccount + name: {{ $serviceAccountName }} + namespace: {{ $envAll.Release.Namespace }} +--- + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mariadb-controller + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} + labels: +{{ tuple $envAll "mariadb" "controller" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + replicas: {{ .Values.pod.replicas.controller }} + selector: + matchLabels: +{{ tuple $envAll "mariadb" "controller" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }} +{{ tuple $envAll | include "helm-toolkit.snippets.kubernetes_upgrades_deployment" | indent 2 }} + template: + metadata: + labels: +{{ tuple $envAll "mariadb" "controller" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} + configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} + spec: + serviceAccountName: {{ $serviceAccountName }} +{{ dict "envAll" $envAll "application" "controller" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + affinity: +{{ tuple $envAll "mariadb" "controller" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }} + nodeSelector: + {{ .Values.labels.controller.node_selector_key }}: {{ .Values.labels.controller.node_selector_value }} + initContainers: +{{ tuple $envAll "controller" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: controller +{{ tuple $envAll "mariadb_controller" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ dict "envAll" $envAll "application" "controller" "container" "controller" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.controller | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + command: + - /tmp/mariadb_controller.py + env: +{{ include "helm-toolkit.utils.to_k8s_env_vars" .Values.pod.env.mariadb_controller | indent 12 }} + - name: MARIADB_CONTROLLER_PODS_NAMESPACE + value: {{ $envAll.Release.Namespace }} + - name: MARIADB_MASTER_SERVICE_NAME + value: {{ tuple "oslo_db" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - mountPath: /tmp/mariadb_controller.py + name: mariadb-bin + readOnly: true + subPath: mariadb_controller.py + volumes: + - name: pod-tmp + emptyDir: {} + - name: mariadb-bin + configMap: + name: mariadb-bin + defaultMode: 365 +{{- end }} diff --git a/mariadb/templates/service-discovery.yaml b/mariadb/templates/service-discovery.yaml index dec979ef3..378878c06 100644 --- a/mariadb/templates/service-discovery.yaml +++ b/mariadb/templates/service-discovery.yaml @@ -25,8 +25,13 @@ spec: port: {{ tuple "oslo_db" "direct" "mysql" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} - name: wsrep port: {{ tuple "oslo_db" "direct" "wsrep" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + - name: ist + port: {{ tuple "oslo_db" "direct" "ist" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + - name: sst + port: {{ tuple "oslo_db" "direct" "sst" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} clusterIP: None publishNotReadyAddresses: true selector: {{ tuple $envAll "mariadb" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +{{ .Values.network.mariadb_discovery | include "helm-toolkit.snippets.service_params" | indent 2 }} {{- end }} diff --git a/mariadb/templates/service-master.yaml b/mariadb/templates/service-master.yaml new file mode 100644 index 000000000..1472e6a32 --- /dev/null +++ b/mariadb/templates/service-master.yaml @@ -0,0 +1,33 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.service_master }} +{{- if .Values.manifests.service_ingress }} +{{- fail ".Values.manifests.service_ingress and .Values.manifests.service_master are mutually exclusive" }} +{{- end }} + +{{- $envAll := . }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ tuple "oslo_db" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} +spec: + ports: + - name: mysql + port: {{ tuple "oslo_db" "direct" "mysql" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + selector: +{{ tuple $envAll "mariadb" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +{{ .Values.network.mariadb_master | include "helm-toolkit.snippets.service_params" | indent 2 }} +{{- end }} diff --git a/mariadb/templates/service.yaml b/mariadb/templates/service.yaml index 3f7a71908..e68cbc49d 100644 --- a/mariadb/templates/service.yaml +++ b/mariadb/templates/service.yaml @@ -25,4 +25,5 @@ spec: port: {{ tuple "oslo_db" "direct" "mysql" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} selector: {{ tuple $envAll "mariadb" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +{{ .Values.network.mariadb | include "helm-toolkit.snippets.service_params" | indent 2 }} {{- end }} diff --git a/mariadb/templates/statefulset.yaml b/mariadb/templates/statefulset.yaml index 42521f190..d706a2772 100644 --- a/mariadb/templates/statefulset.yaml +++ b/mariadb/templates/statefulset.yaml @@ -226,6 +226,12 @@ spec: - name: wsrep protocol: TCP containerPort: {{ tuple "oslo_db" "direct" "wsrep" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + - name: ist + protocol: TCP + containerPort: {{ tuple "oslo_db" "direct" "ist" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + - name: sst + protocol: TCP + containerPort: {{ tuple "oslo_db" "direct" "sst" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} command: - /tmp/start.py lifecycle: diff --git a/mariadb/values.yaml b/mariadb/values.yaml index 4d7ee7563..e2153d583 100644 --- a/mariadb/values.yaml +++ b/mariadb/values.yaml @@ -31,6 +31,7 @@ images: mariadb_backup: quay.io/airshipit/porthole-mysqlclient-utility:latest-ubuntu_focal ks_user: docker.io/openstackhelm/heat:wallaby-ubuntu_focal scripted_test: docker.io/openstackhelm/mariadb:ubuntu_focal-20210415 + mariadb_controller: docker.io/openstackhelm/mariadb:latest-ubuntu_focal pull_policy: "IfNotPresent" local_registry: active: false @@ -57,8 +58,16 @@ labels: test: node_selector_key: openstack-control-plane node_selector_value: enabled + controller: + node_selector_key: openstack-control-plane + node_selector_value: enabled pod: + env: + mariadb_controller: + MARIADB_CONTROLLER_DEBUG: 0 + MARIADB_CONTROLLER_CHECK_PODS_DELAY: 10 + MARIADB_CONTROLLER_PYKUBE_REQUEST_TIMEOUT: 60 probes: server: mariadb: @@ -136,6 +145,13 @@ pod: test: runAsUser: 999 readOnlyRootFilesystem: true + controller: + pod: + runAsUser: 65534 + container: + controller: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true affinity: anti: type: @@ -159,6 +175,7 @@ pod: ingress: 2 error_page: 1 prometheus_mysql_exporter: 1 + controller: 1 lifecycle: upgrades: deployments: @@ -282,7 +299,8 @@ dependencies: services: - endpoint: internal service: oslo_db - + controller: + services: null volume: # this value is used for single pod deployments of mariadb to prevent losing all data # if the pod is restarted @@ -615,6 +633,10 @@ endpoints: default: 3306 wsrep: default: 4567 + ist: + default: 4568 + sst: + default: 4444 kube_dns: namespace: kube-system name: kubernetes-dns @@ -670,6 +692,13 @@ endpoints: default: 80 internal: 5000 +network: + mariadb: {} + mariadb_discovery: {} + mariadb_ingress: {} + mariadb_ingress_error_pages: {} + mariadb_master: {} + network_policy: mariadb: ingress: @@ -721,4 +750,6 @@ manifests: service_error: false service: true statefulset: true + deployment_controller: false + service_master: false ... diff --git a/releasenotes/notes/mariadb.yaml b/releasenotes/notes/mariadb.yaml index 7e5512f39..4f8a769d1 100644 --- a/releasenotes/notes/mariadb.yaml +++ b/releasenotes/notes/mariadb.yaml @@ -60,4 +60,5 @@ mariadb: - 0.2.42 Use quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal by default - 0.2.43 Add 2024.1 Ubuntu Jammy overrides - 0.2.44 Uplift ingress controller image to 1.11.2 + - 0.2.45 Add mariadb controller support ...