From ec29020b32eecd81e9863b7d518f04b72a18e20b Mon Sep 17 00:00:00 2001 From: Mohammed Naser Date: Fri, 21 Jul 2023 15:40:09 +0000 Subject: [PATCH] feat(ovn): enable ha for OVN control plane This is a really big refactor which implements and adds OVN HA for the control plane which can enable production deployments. Depends-On: https://review.opendev.org/c/openstack/openstack-helm-images/+/889181 Change-Id: Idce896148b33a87467cd5656918c5c7377a29504 --- ovn/Chart.yaml | 2 +- ovn/templates/bin/_ovn-controller-init.sh.tpl | 89 ++ ovn/templates/bin/_ovn-controller.sh.tpl | 39 + ovn/templates/bin/_ovn-northd.sh.tpl | 57 + .../bin/_ovn-setup-bridges-init.sh.tpl | 29 - ovn/templates/bin/_ovn.sh.tpl | 1393 ----------------- ovn/templates/bin/_ovsdb-server.sh.tpl | 72 + ovn/templates/configmap-bin.yaml | 12 +- ovn/templates/daemonset-controller.yaml | 38 +- ovn/templates/deployment-northd.yaml | 41 +- ...rvice-nb-db.yaml => service-ovsdb-nb.yaml} | 12 +- ...rvice-sb-db.yaml => service-ovsdb-sb.yaml} | 12 +- ovn/templates/statefulset-nb-db.yaml | 85 - ovn/templates/statefulset-ovsdb-nb.yaml | 102 ++ ovn/templates/statefulset-ovsdb-sb.yaml | 102 ++ ovn/templates/statefulset-sb-db.yaml | 85 - ovn/values.yaml | 124 +- releasenotes/notes/ovn.yaml | 1 + 18 files changed, 612 insertions(+), 1683 deletions(-) create mode 100644 ovn/templates/bin/_ovn-controller-init.sh.tpl create mode 100644 ovn/templates/bin/_ovn-controller.sh.tpl create mode 100644 ovn/templates/bin/_ovn-northd.sh.tpl delete mode 100644 ovn/templates/bin/_ovn-setup-bridges-init.sh.tpl delete mode 100644 ovn/templates/bin/_ovn.sh.tpl create mode 100644 ovn/templates/bin/_ovsdb-server.sh.tpl rename ovn/templates/{service-nb-db.yaml => service-ovsdb-nb.yaml} (54%) rename ovn/templates/{service-sb-db.yaml => service-ovsdb-sb.yaml} (54%) delete mode 100644 ovn/templates/statefulset-nb-db.yaml create mode 100644 ovn/templates/statefulset-ovsdb-nb.yaml create mode 100644 ovn/templates/statefulset-ovsdb-sb.yaml delete mode 100644 ovn/templates/statefulset-sb-db.yaml diff --git a/ovn/Chart.yaml b/ovn/Chart.yaml index 6a9e88cad..6ca3cc827 100644 --- a/ovn/Chart.yaml +++ b/ovn/Chart.yaml @@ -15,7 +15,7 @@ apiVersion: v1 appVersion: v23.3.0 description: OpenStack-Helm OVN name: ovn -version: 0.1.3 +version: 0.1.4 home: https://www.ovn.org icon: https://www.ovn.org/images/ovn-logo.png sources: diff --git a/ovn/templates/bin/_ovn-controller-init.sh.tpl b/ovn/templates/bin/_ovn-controller-init.sh.tpl new file mode 100644 index 000000000..248cfc97d --- /dev/null +++ b/ovn/templates/bin/_ovn-controller-init.sh.tpl @@ -0,0 +1,89 @@ +#!/bin/bash -xe + +# Copyright 2023 VEXXHOST, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +function get_ip_address_from_interface { + local interface=$1 + local ip=$(ip -4 -o addr s "${interface}" | awk '{ print $4; exit }' | awk -F '/' '{print $1}') + if [ -z "${ip}" ] ; then + exit 1 + fi + echo ${ip} +} + +# Detect tunnel interface +tunnel_interface="{{- .Values.network.interface.tunnel -}}" +if [ -z "${tunnel_interface}" ] ; then + # search for interface with tunnel network routing + tunnel_network_cidr="{{- .Values.network.interface.tunnel_network_cidr -}}" + if [ -z "${tunnel_network_cidr}" ] ; then + tunnel_network_cidr="0/0" + fi + # If there is not tunnel network gateway, exit + tunnel_interface=$(ip -4 route list ${tunnel_network_cidr} | awk -F 'dev' '{ print $2; exit }' \ + | awk '{ print $1 }') || exit 1 +fi +ovs-vsctl set open . external_ids:ovn-encap-ip="$(get_ip_address_from_interface ${tunnel_interface})" + +# Configure system ID +set +e +ovs-vsctl get open . external-ids:system-id +if [ $? -eq 1 ]; then + ovs-vsctl set open . external-ids:system-id="$(uuidgen)" +fi +set -e + +# Configure OVN remote +{{- if empty .Values.conf.ovn_remote -}} +{{- $sb_svc_name := "ovn-ovsdb-sb" -}} +{{- $sb_svc := (tuple $sb_svc_name "internal" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup") -}} +{{- $sb_port := (tuple "ovn-ovsdb-sb" "internal" "raft" . | include "helm-toolkit.endpoints.endpoint_port_lookup") -}} +{{- $sb_service_list := list -}} +{{- range $i := until (.Values.pod.replicas.ovn_ovsdb_sb | int) -}} + {{- $sb_service_list = printf "tcp:%s-%d.%s:%s" $sb_svc_name $i $sb_svc $sb_port | append $sb_service_list -}} +{{- end }} + +ovs-vsctl set open . external-ids:ovn-remote="{{ include "helm-toolkit.utils.joinListWithComma" $sb_service_list }}" +{{- else -}} +ovs-vsctl set open . external-ids:ovn-remote="{{ .Values.conf.ovn_remote }}" +{{- end }} + +# Configure OVN values +ovs-vsctl set open . external-ids:rundir="/var/run/openvswitch" +ovs-vsctl set open . external-ids:ovn-encap-type="{{ .Values.conf.ovn_encap_type }}" +ovs-vsctl set open . external-ids:ovn-bridge="{{ .Values.conf.ovn_bridge }}" +ovs-vsctl set open . external-ids:ovn-bridge-mappings="{{ .Values.conf.ovn_bridge_mappings }}" +ovs-vsctl set open . external-ids:ovn-cms-options="{{ .Values.conf.ovn_cms_options }}" + +# Configure hostname +{{- if .Values.conf.use_fqdn.compute }} + ovs-vsctl set open . external-ids:hostname="$(hostname -f)" +{{- else }} + ovs-vsctl set open . external-ids:hostname="$(hostname)" +{{- end }} + +# Create bridges and create ports +# handle any bridge mappings +# /tmp/auto_bridge_add is one line json file: {"br-ex1":"eth1","br-ex2":"eth2"} +for bmap in `sed 's/[{}"]//g' /tmp/auto_bridge_add | tr "," "\n"` +do + bridge=${bmap%:*} + iface=${bmap#*:} + ovs-vsctl --may-exist add-br $bridge -- set bridge $bridge protocols=OpenFlow13 + if [ -n "$iface" ] && [ "$iface" != "null" ] + then + ovs-vsctl --may-exist add-port $bridge $iface + fi +done diff --git a/ovn/templates/bin/_ovn-controller.sh.tpl b/ovn/templates/bin/_ovn-controller.sh.tpl new file mode 100644 index 000000000..ecb659d26 --- /dev/null +++ b/ovn/templates/bin/_ovn-controller.sh.tpl @@ -0,0 +1,39 @@ +#!/bin/bash -xe + +# Copyright 2023 VEXXHOST, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +COMMAND="${@:-start}" + +function start () { + /usr/share/ovn/scripts/ovn-ctl start_controller \ + --ovn-manage-ovsdb=no + + tail --follow=name /var/log/ovn/ovn-controller.log +} + +function stop () { + /usr/share/ovn/scripts/ovn-ctl stop_controller + pkill tail +} + +function liveness () { + ovs-appctl -t /var/run/ovn/ovn-controller.$(cat /var/run/ovn/ovn-controller.pid).ctl status +} + +function readiness () { + ovs-appctl -t /var/run/ovn/ovn-controller.$(cat /var/run/ovn/ovn-controller.pid).ctl status +} + +$COMMAND diff --git a/ovn/templates/bin/_ovn-northd.sh.tpl b/ovn/templates/bin/_ovn-northd.sh.tpl new file mode 100644 index 000000000..bb61c581a --- /dev/null +++ b/ovn/templates/bin/_ovn-northd.sh.tpl @@ -0,0 +1,57 @@ +#!/bin/bash -xe + +# Copyright 2023 VEXXHOST, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +COMMAND="${@:-start}" + +{{- $nb_svc_name := "ovn-ovsdb-nb" -}} +{{- $nb_svc := (tuple $nb_svc_name "internal" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup") -}} +{{- $nb_port := (tuple "ovn-ovsdb-nb" "internal" "raft" . | include "helm-toolkit.endpoints.endpoint_port_lookup") -}} +{{- $nb_service_list := list -}} +{{- range $i := until (.Values.pod.replicas.ovn_ovsdb_nb | int) -}} + {{- $nb_service_list = printf "tcp:%s-%d.%s:%s" $nb_svc_name $i $nb_svc $nb_port | append $nb_service_list -}} +{{- end -}} + +{{- $sb_svc_name := "ovn-ovsdb-sb" -}} +{{- $sb_svc := (tuple $sb_svc_name "internal" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup") -}} +{{- $sb_port := (tuple "ovn-ovsdb-sb" "internal" "raft" . | include "helm-toolkit.endpoints.endpoint_port_lookup") -}} +{{- $sb_service_list := list -}} +{{- range $i := until (.Values.pod.replicas.ovn_ovsdb_sb | int) -}} + {{- $sb_service_list = printf "tcp:%s-%d.%s:%s" $sb_svc_name $i $sb_svc $sb_port | append $sb_service_list -}} +{{- end }} + +function start () { + /usr/share/ovn/scripts/ovn-ctl start_northd \ + --ovn-manage-ovsdb=no \ + --ovn-northd-nb-db={{ include "helm-toolkit.utils.joinListWithComma" $nb_service_list }} \ + --ovn-northd-sb-db={{ include "helm-toolkit.utils.joinListWithComma" $sb_service_list }} + + tail --follow=name /var/log/ovn/ovn-northd.log +} + +function stop () { + /usr/share/ovn/scripts/ovn-ctl stop_northd + pkill tail +} + +function liveness () { + ovs-appctl -t /var/run/ovn/ovn-northd.$(cat /var/run/ovn/ovn-northd.pid).ctl status +} + +function readiness () { + ovs-appctl -t /var/run/ovn/ovn-northd.$(cat /var/run/ovn/ovn-northd.pid).ctl status +} + +$COMMAND diff --git a/ovn/templates/bin/_ovn-setup-bridges-init.sh.tpl b/ovn/templates/bin/_ovn-setup-bridges-init.sh.tpl deleted file mode 100644 index c474f1c69..000000000 --- a/ovn/templates/bin/_ovn-setup-bridges-init.sh.tpl +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash - -{{/* -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/}} -set -ex - -# handle any bridge mappings -# /tmp/auto_bridge_add is one line json file: {"br-ex1":"eth1","br-ex2":"eth2"} -for bmap in `sed 's/[{}"]//g' /tmp/auto_bridge_add | tr "," "\n"` -do - bridge=${bmap%:*} - iface=${bmap#*:} - ovs-vsctl --may-exist add-br $bridge -- set bridge $bridge protocols=OpenFlow13 - if [ -n "$iface" ] && [ "$iface" != "null" ] - then - ovs-vsctl --may-exist add-port $bridge $iface - fi -done diff --git a/ovn/templates/bin/_ovn.sh.tpl b/ovn/templates/bin/_ovn.sh.tpl deleted file mode 100644 index a6128d42a..000000000 --- a/ovn/templates/bin/_ovn.sh.tpl +++ /dev/null @@ -1,1393 +0,0 @@ -#!/bin/bash -# set -x - -bracketify() { case "$1" in *:*) echo "[$1]" ;; *) echo "$1" ;; esac } - -OVN_NORTH="tcp:${OVN_NB_DB_SERVICE_HOST}:${OVN_NB_DB_SERVICE_PORT_OVN_NB_DB}" -OVN_SOUTH="tcp:${OVN_SB_DB_SERVICE_HOST}:${OVN_SB_DB_SERVICE_PORT_OVN_SB_DB}" - -# This script is the entrypoint to the image. -# Supports version 3 daemonsets -# $1 is the daemon to start. -# In version 3 each process has a separate container. Some daemons start -# more than 1 process. Also, where possible, output is to stdout and -# The script waits for prerquisite deamons to come up first. -# Commands ($1 values) -# ovs-server Runs the ovs daemons - ovsdb-server and ovs-switchd (v3) -# run-ovn-northd Runs ovn-northd as a process does not run nb_ovsdb or sb_ovsdb (v3) -# nb-ovsdb Runs nb_ovsdb as a process (no detach or monitor) (v3) -# sb-ovsdb Runs sb_ovsdb as a process (no detach or monitor) (v3) -# ovn-master Runs ovnkube in master mode (v3) -# ovn-controller Runs ovn controller (v3) -# ovn-node Runs ovnkube in node mode (v3) -# cleanup-ovn-node Runs ovnkube to cleanup the node (v3) -# cleanup-ovs-server Cleanup ovs-server (v3) -# display Displays log files -# display_env Displays environment variables -# ovn_debug Displays ovn/ovs configuration and flows - -# ==================== -# Environment variables are used to customize operation -# K8S_APISERVER - hostname:port (URL)of the real apiserver, not the service address - v3 -# OVN_NET_CIDR - the network cidr - v3 -# OVN_SVC_CIDR - the cluster-service-cidr - v3 -# OVN_KUBERNETES_NAMESPACE - k8s namespace - v3 -# K8S_NODE - hostname of the node - v3 -# -# OVN_DAEMONSET_VERSION - version match daemonset and image - v3 -# K8S_TOKEN - the apiserver token. Automatically detected when running in a pod - v3 -# K8S_CACERT - the apiserver CA. Automatically detected when running in a pod - v3 -# OVN_CONTROLLER_OPTS - the options for ovn-ctl -# OVN_NORTHD_OPTS - the options for the ovn northbound db -# OVN_GATEWAY_MODE - the gateway mode (shared or local) - v3 -# OVN_GATEWAY_OPTS - the options for the ovn gateway -# OVN_GATEWAY_ROUTER_SUBNET - the gateway router subnet (shared mode, DPU only) - v3 -# OVNKUBE_LOGLEVEL - log level for ovnkube (0..5, default 4) - v3 -# OVN_LOGLEVEL_NORTHD - log level (ovn-ctl default: -vconsole:emer -vsyslog:err -vfile:info) - v3 -# OVN_LOGLEVEL_NB - log level (ovn-ctl default: -vconsole:off -vfile:info) - v3 -# OVN_LOGLEVEL_SB - log level (ovn-ctl default: -vconsole:off -vfile:info) - v3 -# OVN_LOGLEVEL_CONTROLLER - log level (ovn-ctl default: -vconsole:off -vfile:info) - v3 -# OVN_LOGLEVEL_NBCTLD - log level (ovn-ctl default: -vconsole:off -vfile:info) - v3 -# OVNKUBE_LOGFILE_MAXSIZE - log file max size in MB(default 100 MB) -# OVNKUBE_LOGFILE_MAXBACKUPS - log file max backups (default 5) -# OVNKUBE_LOGFILE_MAXAGE - log file max age in days (default 5 days) -# OVN_ACL_LOGGING_RATE_LIMIT - specify default ACL logging rate limit in messages per second (default: 20) -# OVN_NB_PORT - ovn north db port (default 6640) -# OVN_SB_PORT - ovn south db port (default 6640) -# OVN_NB_RAFT_PORT - ovn north db raft port (default 6643) -# OVN_SB_RAFT_PORT - ovn south db raft port (default 6644) -# OVN_NB_RAFT_ELECTION_TIMER - ovn north db election timer in ms (default 1000) -# OVN_SB_RAFT_ELECTION_TIMER - ovn south db election timer in ms (default 1000) -# OVN_SSL_ENABLE - use SSL transport to NB/SB db and northd (default: no) -# OVN_REMOTE_PROBE_INTERVAL - ovn remote probe interval in ms (default 100000) -# OVN_MONITOR_ALL - ovn-controller monitor all data in SB DB -# OVN_OFCTRL_WAIT_BEFORE_CLEAR - ovn-controller wait time in ms before clearing OpenFlow rules during start up -# OVN_ENABLE_LFLOW_CACHE - enable ovn-controller lflow-cache -# OVN_LFLOW_CACHE_LIMIT - maximum number of logical flow cache entries of ovn-controller -# OVN_LFLOW_CACHE_LIMIT_KB - maximum size of the logical flow cache of ovn-controller -# OVN_EGRESSIP_ENABLE - enable egress IP for ovn-kubernetes -# OVN_EGRESSIP_HEALTHCHECK_PORT - egress IP node check to use grpc on this port (0 ==> dial to port 9 instead) -# OVN_EGRESSFIREWALL_ENABLE - enable egressFirewall for ovn-kubernetes -# OVN_EGRESSQOS_ENABLE - enable egress QoS for ovn-kubernetes -# OVN_UNPRIVILEGED_MODE - execute CNI ovs/netns commands from host (default no) -# OVNKUBE_NODE_MODE - ovnkube node mode of operation, one of: full, dpu, dpu-host (default: full) -# OVNKUBE_NODE_MGMT_PORT_NETDEV - ovnkube node management port netdev. -# OVN_ENCAP_IP - encap IP to be used for OVN traffic on the node. mandatory in case ovnkube-node-mode=="dpu" -# OVN_HOST_NETWORK_NAMESPACE - namespace to classify host network traffic for applying network policies - -# The argument to the command is the operation to be performed -# ovn-master ovn-controller ovn-node display display_env ovn_debug -# a cmd must be provided, there is no default -cmd=${1:-""} - -# ovn daemon log levels -ovn_loglevel_northd=${OVN_LOGLEVEL_NORTHD:-"-vconsole:info"} -ovn_loglevel_nb=${OVN_LOGLEVEL_NB:-"-vconsole:info"} -ovn_loglevel_sb=${OVN_LOGLEVEL_SB:-"-vconsole:info"} -ovn_loglevel_controller=${OVN_LOGLEVEL_CONTROLLER:-"-vconsole:info"} - -ovnkubelogdir=/var/log/ovn-kubernetes - -# logfile rotation parameters -ovnkube_logfile_maxsize=${OVNKUBE_LOGFILE_MAXSIZE:-"100"} -ovnkube_logfile_maxbackups=${OVNKUBE_LOGFILE_MAXBACKUPS:-"5"} -ovnkube_logfile_maxage=${OVNKUBE_LOGFILE_MAXAGE:-"5"} - -# ovnkube.sh version (update when API between daemonset and script changes - v.x.y) -ovnkube_version="3" - -# The daemonset version must be compatible with this script. -# The default when OVN_DAEMONSET_VERSION is not set is version 3 -ovn_daemonset_version=${OVN_DAEMONSET_VERSION:-"3"} - -# hostname is the host's hostname when using host networking, -# This is useful on the master -# otherwise it is the container ID (useful for debugging). -ovn_pod_host=${K8S_NODE:-$(hostname)} - -# The ovs user id, by default it is going to be root:root -ovs_user_id=${OVS_USER_ID:-""} - -# ovs options -ovs_options=${OVS_OPTIONS:-""} - -if [[ -f /var/run/secrets/kubernetes.io/serviceaccount/token ]]; then - k8s_token=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token) -else - k8s_token=${K8S_TOKEN} -fi - -# certs and private keys for k8s and OVN -K8S_CACERT=${K8S_CACERT:-/var/run/secrets/kubernetes.io/serviceaccount/ca.crt} - -ovn_ca_cert=/ovn-cert/ca-cert.pem -ovn_nb_pk=/ovn-cert/ovnnb-privkey.pem -ovn_nb_cert=/ovn-cert/ovnnb-cert.pem -ovn_sb_pk=/ovn-cert/ovnsb-privkey.pem -ovn_sb_cert=/ovn-cert/ovnsb-cert.pem -ovn_northd_pk=/ovn-cert/ovnnorthd-privkey.pem -ovn_northd_cert=/ovn-cert/ovnnorthd-cert.pem -ovn_controller_pk=/ovn-cert/ovncontroller-privkey.pem -ovn_controller_cert=/ovn-cert/ovncontroller-cert.pem -ovn_controller_cname="ovncontroller" - -transport="tcp" -ovndb_ctl_ssl_opts="" -if [[ "yes" == ${OVN_SSL_ENABLE} ]]; then - transport="ssl" - ovndb_ctl_ssl_opts="-p ${ovn_controller_pk} -c ${ovn_controller_cert} -C ${ovn_ca_cert}" -fi - -# ovn-northd - /etc/sysconfig/ovn-northd -ovn_northd_opts=${OVN_NORTHD_OPTS:-""} - -# ovn-controller -ovn_controller_opts=${OVN_CONTROLLER_OPTS:-""} - -# set the log level for ovnkube -ovnkube_loglevel=${OVNKUBE_LOGLEVEL:-4} - -# by default it is going to be a shared gateway mode, however this can be overridden to any of the other -# two gateway modes that we support using `images/daemonset.sh` tool -ovn_gateway_mode=${OVN_GATEWAY_MODE:-"shared"} -ovn_gateway_opts=${OVN_GATEWAY_OPTS:-""} -ovn_gateway_router_subnet=${OVN_GATEWAY_ROUTER_SUBNET:-""} - -net_cidr=${OVN_NET_CIDR:-10.128.0.0/14/23} -svc_cidr=${OVN_SVC_CIDR:-172.30.0.0/16} -mtu=${OVN_MTU:-1400} -routable_mtu=${OVN_ROUTABLE_MTU:-} - -# set metrics endpoint bind to K8S_NODE_IP. -metrics_endpoint_ip=${K8S_NODE_IP:-0.0.0.0} -metrics_endpoint_ip=$(bracketify $metrics_endpoint_ip) -ovn_kubernetes_namespace=${OVN_KUBERNETES_NAMESPACE:-ovn-kubernetes} -# namespace used for classifying host network traffic -ovn_host_network_namespace=${OVN_HOST_NETWORK_NAMESPACE:-ovn-host-network} - -# host on which ovnkube-db POD is running and this POD contains both -# OVN NB and SB DB running in their own container. -ovn_db_host=$(hostname -i) - -# OVN_NB_PORT - ovn north db port (default 6640) -ovn_nb_port=${OVN_NB_PORT:-6640} -# OVN_SB_PORT - ovn south db port (default 6640) -ovn_sb_port=${OVN_SB_PORT:-6640} -# OVN_NB_RAFT_PORT - ovn north db port used for raft communication (default 6643) -ovn_nb_raft_port=${OVN_NB_RAFT_PORT:-6643} -# OVN_SB_RAFT_PORT - ovn south db port used for raft communication (default 6644) -ovn_sb_raft_port=${OVN_SB_RAFT_PORT:-6644} -# OVN_ENCAP_PORT - GENEVE UDP port (default 6081) -ovn_encap_port=${OVN_ENCAP_PORT:-6081} -# OVN_NB_RAFT_ELECTION_TIMER - ovn north db election timer in ms (default 1000) -ovn_nb_raft_election_timer=${OVN_NB_RAFT_ELECTION_TIMER:-1000} -# OVN_SB_RAFT_ELECTION_TIMER - ovn south db election timer in ms (default 1000) -ovn_sb_raft_election_timer=${OVN_SB_RAFT_ELECTION_TIMER:-1000} - -ovn_hybrid_overlay_enable=${OVN_HYBRID_OVERLAY_ENABLE:-} -ovn_hybrid_overlay_net_cidr=${OVN_HYBRID_OVERLAY_NET_CIDR:-} -ovn_disable_snat_multiple_gws=${OVN_DISABLE_SNAT_MULTIPLE_GWS:-} -ovn_disable_pkt_mtu_check=${OVN_DISABLE_PKT_MTU_CHECK:-} -ovn_empty_lb_events=${OVN_EMPTY_LB_EVENTS:-} -# OVN_V4_JOIN_SUBNET - v4 join subnet -ovn_v4_join_subnet=${OVN_V4_JOIN_SUBNET:-} -# OVN_V6_JOIN_SUBNET - v6 join subnet -ovn_v6_join_subnet=${OVN_V6_JOIN_SUBNET:-} -#OVN_REMOTE_PROBE_INTERVAL - ovn remote probe interval in ms (default 100000) -ovn_remote_probe_interval=${OVN_REMOTE_PROBE_INTERVAL:-100000} -#OVN_MONITOR_ALL - ovn-controller monitor all data in SB DB -ovn_monitor_all=${OVN_MONITOR_ALL:-} -#OVN_OFCTRL_WAIT_BEFORE_CLEAR - ovn-controller wait time in ms before clearing OpenFlow rules during start up -ovn_ofctrl_wait_before_clear=${OVN_OFCTRL_WAIT_BEFORE_CLEAR:-} -ovn_enable_lflow_cache=${OVN_ENABLE_LFLOW_CACHE:-} -ovn_lflow_cache_limit=${OVN_LFLOW_CACHE_LIMIT:-} -ovn_lflow_cache_limit_kb=${OVN_LFLOW_CACHE_LIMIT_KB:-} -ovn_multicast_enable=${OVN_MULTICAST_ENABLE:-} -#OVN_EGRESSIP_ENABLE - enable egress IP for ovn-kubernetes -ovn_egressip_enable=${OVN_EGRESSIP_ENABLE:-false} -#OVN_EGRESSIP_HEALTHCHECK_PORT - egress IP node check to use grpc on this port -ovn_egress_ip_healthcheck_port=${OVN_EGRESSIP_HEALTHCHECK_PORT:-9107} -#OVN_EGRESSFIREWALL_ENABLE - enable egressFirewall for ovn-kubernetes -ovn_egressfirewall_enable=${OVN_EGRESSFIREWALL_ENABLE:-false} -#OVN_EGRESSQOS_ENABLE - enable egress QoS for ovn-kubernetes -ovn_egressqos_enable=${OVN_EGRESSQOS_ENABLE:-false} -#OVN_DISABLE_OVN_IFACE_ID_VER - disable usage of the OVN iface-id-ver option -ovn_disable_ovn_iface_id_ver=${OVN_DISABLE_OVN_IFACE_ID_VER:-false} -ovn_acl_logging_rate_limit=${OVN_ACL_LOGGING_RATE_LIMIT:-"20"} -ovn_netflow_targets=${OVN_NETFLOW_TARGETS:-} -ovn_sflow_targets=${OVN_SFLOW_TARGETS:-} -ovn_ipfix_targets=${OVN_IPFIX_TARGETS:-} -ovn_ipfix_sampling=${OVN_IPFIX_SAMPLING:-} \ -ovn_ipfix_cache_max_flows=${OVN_IPFIX_CACHE_MAX_FLOWS:-} \ -ovn_ipfix_cache_active_timeout=${OVN_IPFIX_CACHE_ACTIVE_TIMEOUT:-} \ - -# OVNKUBE_NODE_MODE - is the mode which ovnkube node operates -ovnkube_node_mode=${OVNKUBE_NODE_MODE:-"full"} -# OVNKUBE_NODE_MGMT_PORT_NETDEV - is the net device to be used for management port -ovnkube_node_mgmt_port_netdev=${OVNKUBE_NODE_MGMT_PORT_NETDEV:-} -ovnkube_config_duration_enable=${OVNKUBE_CONFIG_DURATION_ENABLE:-false} -# OVN_ENCAP_IP - encap IP to be used for OVN traffic on the node -ovn_encap_ip=${OVN_ENCAP_IP:-} - -ovn_ex_gw_network_interface=${OVN_EX_GW_NETWORK_INTERFACE:-} - -# Determine the ovn rundir. -if [[ -f /usr/bin/ovn-appctl ]]; then - # ovn-appctl is present. Use new ovn run dir path. - OVN_RUNDIR=/var/run/ovn - OVNCTL_PATH=/usr/share/ovn/scripts/ovn-ctl - OVN_LOGDIR=/var/log/ovn - OVN_ETCDIR=/etc/ovn -else - # ovn-appctl is not present. Use openvswitch run dir path. - OVN_RUNDIR=/var/run/openvswitch - OVNCTL_PATH=/usr/share/openvswitch/scripts/ovn-ctl - OVN_LOGDIR=/var/log/openvswitch - OVN_ETCDIR=/etc/openvswitch -fi - -OVS_RUNDIR=/var/run/openvswitch -OVS_LOGDIR=/var/log/openvswitch - -# ========================================= - -setup_ovs_permissions() { - if [ ${ovs_user_id:-XX} != "XX" ]; then - chown -R ${ovs_user_id} /etc/openvswitch - chown -R ${ovs_user_id} ${OVS_RUNDIR} - chown -R ${ovs_user_id} ${OVS_LOGDIR} - chown -R ${ovs_user_id} ${OVN_ETCDIR} - chown -R ${ovs_user_id} ${OVN_RUNDIR} - chown -R ${ovs_user_id} ${OVN_LOGDIR} - fi -} - -run_as_ovs_user_if_needed() { - setup_ovs_permissions - - if [ ${ovs_user_id:-XX} != "XX" ]; then - local uid=$(id -u "${ovs_user_id%:*}") - local gid=$(id -g "${ovs_user_id%:*}") - local groups=$(id -G "${ovs_user_id%:*}" | tr ' ' ',') - - setpriv --reuid $uid --regid $gid --groups $groups "$@" - echo "run as: setpriv --reuid $uid --regid $gid --groups $groups $@" - else - "$@" - echo "run as: $@" - fi -} - -# wait_for_event [attempts=] function_to_call [arguments_to_function] -# -# Processes running inside the container should immediately start, so we -# shouldn't be making 80 attempts (default value). The "attempts=" -# argument will help us in configuring that value. -wait_for_event() { - retries=0 - sleeper=1 - attempts=80 - if [[ $1 =~ ^attempts= ]]; then - eval $1 - shift - fi - while true; do - $@ - if [[ $? != 0 ]]; then - ((retries += 1)) - if [[ "${retries}" -gt ${attempts} ]]; then - echo "error: $@ did not come up, exiting" - exit 1 - fi - echo "info: Waiting for $@ to come up, waiting ${sleeper}s ..." - sleep ${sleeper} - sleeper=5 - else - if [[ "${retries}" != 0 ]]; then - echo "$@ came up in ${retries} ${sleeper} sec tries" - fi - break - fi - done -} - -# check that daemonset version is among expected versions -check_ovn_daemonset_version() { - ok=$1 - for v in ${ok}; do - if [[ $v == ${ovn_daemonset_version} ]]; then - return 0 - fi - done - echo "VERSION MISMATCH expect ${ok}, daemonset is version ${ovn_daemonset_version}" - exit 1 -} - - -ovsdb_cleanup() { - local db=${1} - ovs-appctl -t ${OVN_RUNDIR}/ovn${db}_db.ctl exit >/dev/null 2>&1 - kill $(jobs -p) >/dev/null 2>&1 - exit 0 -} - -get_ovn_db_vars() { - ovn_nbdb_str="" - ovn_sbdb_str="" - for i in "${ovn_db_hosts[@]}"; do - if [ -n "$ovn_nbdb_str" ]; then - ovn_nbdb_str=${ovn_nbdb_str}"," - ovn_sbdb_str=${ovn_sbdb_str}"," - fi - ip=$(bracketify $i) - ovn_nbdb_str=${ovn_nbdb_str}${transport}://${ip}:${ovn_nb_port} - ovn_sbdb_str=${ovn_sbdb_str}${transport}://${ip}:${ovn_sb_port} - done - # OVN_NORTH and OVN_SOUTH override derived host - ovn_nbdb=${OVN_NORTH:-$ovn_nbdb_str} - ovn_sbdb=${OVN_SOUTH:-$ovn_sbdb_str} - - # ovsdb server connection method :: - ovn_nbdb_conn=$(echo ${ovn_nbdb} | sed 's;//;;g') - ovn_sbdb_conn=$(echo ${ovn_sbdb} | sed 's;//;;g') -} - -# OVS must be up before OVN comes up. -# This checks if OVS is up and running -ovs_ready() { - for daemon in $(echo ovsdb-server ovs-vswitchd); do - pidfile=${OVS_RUNDIR}/${daemon}.pid - if [[ -f ${pidfile} ]]; then - check_health $daemon $(cat $pidfile) - if [[ $? == 0 ]]; then - continue - fi - fi - return 1 - done - return 0 -} - -# Verify that the process is running either by checking for the PID in `ps` output -# or by using `ovs-appctl` utility for the processes that support it. -# $1 is the name of the process -process_ready() { - case ${1} in - "ovsdb-server" | "ovs-vswitchd") - pidfile=${OVS_RUNDIR}/${1}.pid - ;; - *) - pidfile=${OVN_RUNDIR}/${1}.pid - ;; - esac - - if [[ -f ${pidfile} ]]; then - check_health $1 $(cat $pidfile) - if [[ $? == 0 ]]; then - return 0 - fi - fi - return 1 -} - -# continuously checks if process is healthy. Exits if process terminates. -# $1 is the name of the process -# $2 is the pid of an another process to kill before exiting -process_healthy() { - case ${1} in - "ovsdb-server" | "ovs-vswitchd") - pid=$(cat ${OVS_RUNDIR}/${1}.pid) - ;; - *) - pid=$(cat ${OVN_RUNDIR}/${1}.pid) - ;; - esac - - while true; do - check_health $1 ${pid} - if [[ $? != 0 ]]; then - echo "=============== pid ${pid} terminated ========== " - # kill the tail -f - if [[ $2 != "" ]]; then - kill $2 - fi - exit 6 - fi - sleep 15 - done -} - -# checks for the health of the process either using `ps` or `ovs-appctl` -# $1 is the name of the process -# $2 is the process pid -check_health() { - ctl_file="" - case ${1} in - "ovnkube" | "ovnkube-master" | "ovn-dbchecker") - # just check for presence of pid - ;; - "ovnnb_db" | "ovnsb_db") - ctl_file=${OVN_RUNDIR}/${1}.ctl - ;; - "ovn-northd" | "ovn-controller") - ctl_file=${OVN_RUNDIR}/${1}.${2}.ctl - ;; - "ovsdb-server" | "ovs-vswitchd") - ctl_file=${OVS_RUNDIR}/${1}.${2}.ctl - ;; - *) - echo "Unknown service ${1} specified. Exiting.. " - exit 1 - ;; - esac - - if [[ ${ctl_file} == "" ]]; then - # no control file, so just do the PID check - pid=${2} - pidTest=$(ps ax | awk '{ print $1 }' | grep "^${pid:-XX}$") - if [[ ${pid:-XX} == ${pidTest} ]]; then - return 0 - fi - else - # use ovs-appctl to do the check - ovs-appctl -t ${ctl_file} version >/dev/null - if [[ $? == 0 ]]; then - return 0 - fi - fi - - return 1 -} - -display_file() { - if [[ -f $3 ]]; then - echo "====================== $1 pid " - cat $2 - echo "====================== $1 log " - cat $3 - echo " " - fi -} - -# pid and log file for each container -display() { - echo "==================== display for ${ovn_pod_host} =================== " - date - display_file "nb-ovsdb" ${OVN_RUNDIR}/ovnnb_db.pid ${OVN_LOGDIR}/ovsdb-server-nb.log - display_file "sb-ovsdb" ${OVN_RUNDIR}/ovnsb_db.pid ${OVN_LOGDIR}/ovsdb-server-sb.log - display_file "run-ovn-northd" ${OVN_RUNDIR}/ovn-northd.pid ${OVN_LOGDIR}/ovn-northd.log - display_file "ovn-master" ${OVN_RUNDIR}/ovnkube-master.pid ${ovnkubelogdir}/ovnkube-master.log - display_file "ovs-vswitchd" ${OVS_RUNDIR}/ovs-vswitchd.pid ${OVS_LOGDIR}/ovs-vswitchd.log - display_file "ovsdb-server" ${OVS_RUNDIR}/ovsdb-server.pid ${OVS_LOGDIR}/ovsdb-server.log - display_file "ovn-controller" ${OVN_RUNDIR}/ovn-controller.pid ${OVN_LOGDIR}/ovn-controller.log - display_file "ovnkube" ${OVN_RUNDIR}/ovnkube.pid ${ovnkubelogdir}/ovnkube.log - display_file "ovn-dbchecker" ${OVN_RUNDIR}/ovn-dbchecker.pid ${OVN_LOGDIR}/ovn-dbchecker.log -} - -setup_cni() { - cp -f /usr/libexec/cni/ovn-k8s-cni-overlay /opt/cni/bin/ovn-k8s-cni-overlay -} - -display_version() { - echo " =================== hostname: ${ovn_pod_host}" - echo " =================== daemonset version ${ovn_daemonset_version}" - if [[ -f /root/git_info ]]; then - disp_ver=$(cat /root/git_info) - echo " =================== Image built from ovn-kubernetes ${disp_ver}" - return - fi -} - -display_env() { - echo OVS_USER_ID ${ovs_user_id} - echo OVS_OPTIONS ${ovs_options} - echo OVN_NORTH ${ovn_nbdb} - echo OVN_NORTHD_OPTS ${ovn_northd_opts} - echo OVN_SOUTH ${ovn_sbdb} - echo OVN_CONTROLLER_OPTS ${ovn_controller_opts} - echo OVN_LOGLEVEL_CONTROLLER ${ovn_loglevel_controller} - echo OVN_GATEWAY_MODE ${ovn_gateway_mode} - echo OVN_GATEWAY_OPTS ${ovn_gateway_opts} - echo OVN_GATEWAY_ROUTER_SUBNET ${ovn_gateway_router_subnet} - echo OVN_NET_CIDR ${net_cidr} - echo OVN_SVC_CIDR ${svc_cidr} - echo OVN_NB_PORT ${ovn_nb_port} - echo OVN_SB_PORT ${ovn_sb_port} - echo K8S_APISERVER ${K8S_APISERVER} - echo OVNKUBE_LOGLEVEL ${ovnkube_loglevel} - echo OVN_DAEMONSET_VERSION ${ovn_daemonset_version} - echo OVNKUBE_NODE_MODE ${ovnkube_node_mode} - echo OVN_ENCAP_IP ${ovn_encap_ip} - echo ovnkube.sh version ${ovnkube_version} - echo OVN_HOST_NETWORK_NAMESPACE ${ovn_host_network_namespace} -} - -ovn_debug() { - echo "ovn_nbdb ${ovn_nbdb} ovn_sbdb ${ovn_sbdb}" - echo "ovn_nbdb_conn ${ovn_nbdb_conn}" - echo "ovn_sbdb_conn ${ovn_sbdb_conn}" - - # get ovs/ovn info from the node for debug purposes - echo "=========== ovn_debug hostname: ${ovn_pod_host} =============" - echo "=========== ovn-nbctl --db=${ovn_nbdb_conn} show =============" - ovn-nbctl --db=${ovn_nbdb_conn} show - echo " " - echo "=========== ovn-nbctl list ACL =============" - ovn-nbctl --db=${ovn_nbdb_conn} list ACL - echo " " - echo "=========== ovn-nbctl list address_set =============" - ovn-nbctl --db=${ovn_nbdb_conn} list address_set - echo " " - echo "=========== ovs-vsctl show =============" - ovs-vsctl show - echo " " - echo "=========== ovs-ofctl -O OpenFlow13 dump-ports br-int =============" - ovs-ofctl -O OpenFlow13 dump-ports br-int - echo " " - echo "=========== ovs-ofctl -O OpenFlow13 dump-ports-desc br-int =============" - ovs-ofctl -O OpenFlow13 dump-ports-desc br-int - echo " " - echo "=========== ovs-ofctl dump-flows br-int =============" - ovs-ofctl dump-flows br-int - echo " " - echo "=========== ovn-sbctl --db=${ovn_sbdb_conn} show =============" - ovn-sbctl --db=${ovn_sbdb_conn} show - echo " " - echo "=========== ovn-sbctl --db=${ovn_sbdb_conn} lflow-list =============" - ovn-sbctl --db=${ovn_sbdb_conn} lflow-list - echo " " - echo "=========== ovn-sbctl --db=${ovn_sbdb_conn} list datapath =============" - ovn-sbctl --db=${ovn_sbdb_conn} list datapath - echo " " - echo "=========== ovn-sbctl --db=${ovn_sbdb_conn} list port_binding =============" - ovn-sbctl --db=${ovn_sbdb_conn} list port_binding -} - -ovs-server() { - # start ovs ovsdb-server and ovs-vswitchd - set -euo pipefail - - # if another process is listening on the cni-server socket, wait until it exits - trap 'kill $(jobs -p); exit 0' TERM - retries=0 - while true; do - if /usr/share/openvswitch/scripts/ovs-ctl status >/dev/null; then - echo "warning: Another process is currently managing OVS, waiting 10s ..." 2>&1 - sleep 10 & - wait - ((retries += 1)) - else - break - fi - if [[ "${retries}" -gt 60 ]]; then - echo "error: Another process is currently managing OVS, exiting" 2>&1 - exit 1 - fi - done - rm -f ${OVS_RUNDIR}/ovs-vswitchd.pid - rm -f ${OVS_RUNDIR}/ovsdb-server.pid - - # launch OVS - function quit() { - /usr/share/openvswitch/scripts/ovs-ctl stop - exit 1 - } - trap quit SIGTERM - - setup_ovs_permissions - - USER_ARGS="" - if [ ${ovs_user_id:-XX} != "XX" ]; then - USER_ARGS="--ovs-user=${ovs_user_id}" - fi - - /usr/share/openvswitch/scripts/ovs-ctl start --no-ovs-vswitchd \ - --system-id=random ${ovs_options} ${USER_ARGS} "$@" - - # Restrict the number of pthreads ovs-vswitchd creates to reduce the - # amount of RSS it uses on hosts with many cores - # https://bugzilla.redhat.com/show_bug.cgi?id=1571379 - # https://bugzilla.redhat.com/show_bug.cgi?id=1572797 - if [[ $(nproc) -gt 12 ]]; then - ovs-vsctl --no-wait set Open_vSwitch . other_config:n-revalidator-threads=4 - ovs-vsctl --no-wait set Open_vSwitch . other_config:n-handler-threads=10 - fi - /usr/share/openvswitch/scripts/ovs-ctl start --no-ovsdb-server \ - --system-id=random ${ovs_options} ${USER_ARGS} "$@" - - tail --follow=name ${OVS_LOGDIR}/ovs-vswitchd.log ${OVS_LOGDIR}/ovsdb-server.log & - ovs_tail_pid=$! - sleep 10 - while true; do - if ! /usr/share/openvswitch/scripts/ovs-ctl status >/dev/null; then - echo "OVS seems to have crashed, exiting" - kill ${ovs_tail_pid} - quit - fi - sleep 15 - done -} - -cleanup-ovs-server() { - echo "=============== time: $(date +%d-%m-%H:%M:%S:%N) cleanup-ovs-server (wait for ovn-node to exit) =======" - retries=0 - while [[ ${retries} -lt 80 ]]; do - if [[ ! -e ${OVN_RUNDIR}/ovnkube.pid ]]; then - break - fi - echo "=============== time: $(date +%d-%m-%H:%M:%S:%N) cleanup-ovs-server ovn-node still running, wait) =======" - sleep 1 - ((retries += 1)) - done - echo "=============== time: $(date +%d-%m-%H:%M:%S:%N) cleanup-ovs-server (ovs-ctl stop) =======" - /usr/share/openvswitch/scripts/ovs-ctl stop -} - -function memory_trim_on_compaction_supported { - if [[ $1 == "nbdb" ]]; then - mem_trim_check=$(ovn-appctl -t ${OVN_RUNDIR}/ovnnb_db.ctl list-commands | grep "memory-trim-on-compaction") - elif [[ $1 == "sbdb" ]]; then - mem_trim_check=$(ovn-appctl -t ${OVN_RUNDIR}/ovnsb_db.ctl list-commands | grep "memory-trim-on-compaction") - fi - if [[ ${mem_trim_check} != "" ]]; then - return $(/bin/true) - else - return $(/bin/false) - fi -} - -# v3 - run nb_ovsdb in a separate container -nb-ovsdb() { - trap 'ovsdb_cleanup nb' TERM - check_ovn_daemonset_version "3" - rm -f ${OVN_RUNDIR}/ovnnb_db.pid - - if [[ ${ovn_db_host} == "" ]]; then - echo "The IP address of the host $(hostname) could not be determined. Exiting..." - exit 1 - fi - - echo "=============== run nb_ovsdb ========== MASTER ONLY" - run_as_ovs_user_if_needed \ - ${OVNCTL_PATH} run_nb_ovsdb --no-monitor \ - --ovn-nb-log="${ovn_loglevel_nb}" & - - wait_for_event attempts=3 process_ready ovnnb_db - echo "=============== nb-ovsdb ========== RUNNING" - - # setting northd probe interval - set_northd_probe_interval - [[ "yes" == ${OVN_SSL_ENABLE} ]] && { - ovn-nbctl set-ssl ${ovn_nb_pk} ${ovn_nb_cert} ${ovn_ca_cert} - echo "=============== nb-ovsdb ========== reconfigured for SSL" - } - [[ "true" == "${ENABLE_IPSEC}" ]] && { - ovn-nbctl set nb_global . ipsec=true - echo "=============== nb-ovsdb ========== reconfigured for ipsec" - } - ovn-nbctl --inactivity-probe=0 set-connection p${transport}:${ovn_nb_port}:$(bracketify ${ovn_db_host}) - if memory_trim_on_compaction_supported "nbdb" - then - # Enable NBDB memory trimming on DB compaction, Every 10mins DBs are compacted - # memory on the heap is freed, when enable memory trimmming freed memory will go back to OS. - ovn-appctl -t ${OVN_RUNDIR}/ovnnb_db.ctl ovsdb-server/memory-trim-on-compaction on - fi - tail --follow=name ${OVN_LOGDIR}/ovsdb-server-nb.log & - ovn_tail_pid=$! - process_healthy ovnnb_db ${ovn_tail_pid} - echo "=============== run nb_ovsdb ========== terminated" -} - -# v3 - run sb_ovsdb in a separate container -sb-ovsdb() { - trap 'ovsdb_cleanup sb' TERM - check_ovn_daemonset_version "3" - rm -f ${OVN_RUNDIR}/ovnsb_db.pid - - if [[ ${ovn_db_host} == "" ]]; then - echo "The IP address of the host $(hostname) could not be determined. Exiting..." - exit 1 - fi - - echo "=============== run sb_ovsdb ========== MASTER ONLY" - run_as_ovs_user_if_needed \ - ${OVNCTL_PATH} run_sb_ovsdb --no-monitor \ - --ovn-sb-log="${ovn_loglevel_sb}" & - - wait_for_event attempts=3 process_ready ovnsb_db - echo "=============== sb-ovsdb ========== RUNNING" - - [[ "yes" == ${OVN_SSL_ENABLE} ]] && { - ovn-sbctl set-ssl ${ovn_sb_pk} ${ovn_sb_cert} ${ovn_ca_cert} - echo "=============== sb-ovsdb ========== reconfigured for SSL" - } - ovn-sbctl --inactivity-probe=0 set-connection p${transport}:${ovn_sb_port}:$(bracketify ${ovn_db_host}) - - # create the ovnkube-db endpoints - if memory_trim_on_compaction_supported "sbdb" - then - # Enable SBDB memory trimming on DB compaction, Every 10mins DBs are compacted - # memory on the heap is freed, when enable memory trimmming freed memory will go back to OS. - ovn-appctl -t ${OVN_RUNDIR}/ovnsb_db.ctl ovsdb-server/memory-trim-on-compaction on - fi - tail --follow=name ${OVN_LOGDIR}/ovsdb-server-sb.log & - ovn_tail_pid=$! - - process_healthy ovnsb_db ${ovn_tail_pid} - echo "=============== run sb_ovsdb ========== terminated" -} - -# v3 - Runs ovn-dbchecker on ovnkube-db pod. -ovn-dbchecker() { - trap 'kill $(jobs -p); exit 0' TERM - check_ovn_daemonset_version "3" - rm -f ${OVN_RUNDIR}/ovn-dbchecker.pid - - echo "ovn_nbdb ${ovn_nbdb} ovn_sbdb ${ovn_sbdb}" - - # wait for nb-ovsdb and sb-ovsdb to start - echo "=============== ovn-dbchecker (wait for nb-ovsdb) ========== OVNKUBE_DB" - wait_for_event attempts=15 process_ready ovnnb_db - - echo "=============== ovn-dbchecker (wait for sb-ovsdb) ========== OVNKUBE_DB" - wait_for_event attempts=15 process_ready ovnsb_db - - local ovn_db_ssl_opts="" - [[ "yes" == ${OVN_SSL_ENABLE} ]] && { - ovn_db_ssl_opts=" - --nb-client-privkey ${ovn_controller_pk} - --nb-client-cert ${ovn_controller_cert} - --nb-client-cacert ${ovn_ca_cert} - --nb-cert-common-name ${ovn_controller_cname} - --sb-client-privkey ${ovn_controller_pk} - --sb-client-cert ${ovn_controller_cert} - --sb-client-cacert ${ovn_ca_cert} - --sb-cert-common-name ${ovn_controller_cname} - " - } - - echo "=============== ovn-dbchecker ========== OVNKUBE_DB" - /usr/bin/ovndbchecker \ - --nb-address=${ovn_nbdb} --sb-address=${ovn_sbdb} \ - ${ovn_db_ssl_opts} \ - --loglevel=${ovnkube_loglevel} \ - --logfile-maxsize=${ovnkube_logfile_maxsize} \ - --logfile-maxbackups=${ovnkube_logfile_maxbackups} \ - --logfile-maxage=${ovnkube_logfile_maxage} \ - --pidfile ${OVN_RUNDIR}/ovn-dbchecker.pid \ - --logfile /var/log/ovn-kubernetes/ovn-dbchecker.log & - - echo "=============== ovn-dbchecker ========== running" - wait_for_event attempts=3 process_ready ovn-dbchecker - - process_healthy ovn-dbchecker - exit 11 -} - -# v3 - Runs northd on master. Does not run nb_ovsdb, and sb_ovsdb -run-ovn-northd() { - trap 'ovs-appctl -t ovn-northd exit >/dev/null 2>&1; exit 0' TERM - check_ovn_daemonset_version "3" - rm -f ${OVN_RUNDIR}/ovn-northd.pid - rm -f ${OVN_RUNDIR}/ovn-northd.*.ctl - mkdir -p ${OVN_RUNDIR} - - echo "=============== run_ovn_northd ========== MASTER ONLY" - echo "ovn_nbdb ${ovn_nbdb} ovn_sbdb ${ovn_sbdb}" - echo "ovn_northd_opts=${ovn_northd_opts}" - echo "ovn_loglevel_northd=${ovn_loglevel_northd}" - - # no monitor (and no detach), start northd which connects to the - # ovnkube-db service - local ovn_northd_ssl_opts="" - [[ "yes" == ${OVN_SSL_ENABLE} ]] && { - ovn_northd_ssl_opts=" - --ovn-northd-ssl-key=${ovn_northd_pk} - --ovn-northd-ssl-cert=${ovn_northd_cert} - --ovn-northd-ssl-ca-cert=${ovn_ca_cert} - " - } - - run_as_ovs_user_if_needed \ - ${OVNCTL_PATH} start_northd \ - --ovn-northd-priority=0 \ - --no-monitor --ovn-manage-ovsdb=no \ - --ovn-northd-nb-db=${ovn_nbdb_conn} --ovn-northd-sb-db=${ovn_sbdb_conn} \ - ${ovn_northd_ssl_opts} \ - --ovn-northd-log="${ovn_loglevel_northd}" \ - ${ovn_northd_opts} - - wait_for_event attempts=3 process_ready ovn-northd - echo "=============== run_ovn_northd ========== RUNNING" - - tail --follow=name ${OVN_LOGDIR}/ovn-northd.log & - ovn_tail_pid=$! - - process_healthy ovn-northd ${ovn_tail_pid} - exit 8 -} - -# v3 - run ovnkube --master -ovn-master() { - trap 'kill $(jobs -p); exit 0' TERM - check_ovn_daemonset_version "3" - rm -f ${OVN_RUNDIR}/ovnkube-master.pid - - echo "ovn_nbdb ${ovn_nbdb} ovn_sbdb ${ovn_sbdb}" - - # wait for northd to start - wait_for_event process_ready ovn-northd - - # wait for ovs-servers to start since ovn-master sets some fields in OVS DB - echo "=============== ovn-master - (wait for ovs)" - wait_for_event ovs_ready - - hybrid_overlay_flags= - if [[ ${ovn_hybrid_overlay_enable} == "true" ]]; then - hybrid_overlay_flags="--enable-hybrid-overlay" - if [[ -n "${ovn_hybrid_overlay_net_cidr}" ]]; then - hybrid_overlay_flags="${hybrid_overlay_flags} --hybrid-overlay-cluster-subnets=${ovn_hybrid_overlay_net_cidr}" - fi - fi - disable_snat_multiple_gws_flag= - if [[ ${ovn_disable_snat_multiple_gws} == "true" ]]; then - disable_snat_multiple_gws_flag="--disable-snat-multiple-gws" - fi - - disable_pkt_mtu_check_flag= - if [[ ${ovn_disable_pkt_mtu_check} == "true" ]]; then - disable_pkt_mtu_check_flag="--disable-pkt-mtu-check" - fi - - empty_lb_events_flag= - if [[ ${ovn_empty_lb_events} == "true" ]]; then - empty_lb_events_flag="--ovn-empty-lb-events" - fi - - ovn_v4_join_subnet_opt= - if [[ -n ${ovn_v4_join_subnet} ]]; then - ovn_v4_join_subnet_opt="--gateway-v4-join-subnet=${ovn_v4_join_subnet}" - fi - - ovn_v6_join_subnet_opt= - if [[ -n ${ovn_v6_join_subnet} ]]; then - ovn_v6_join_subnet_opt="--gateway-v6-join-subnet=${ovn_v6_join_subnet}" - fi - - local ovn_master_ssl_opts="" - [[ "yes" == ${OVN_SSL_ENABLE} ]] && { - ovn_master_ssl_opts=" - --nb-client-privkey ${ovn_controller_pk} - --nb-client-cert ${ovn_controller_cert} - --nb-client-cacert ${ovn_ca_cert} - --nb-cert-common-name ${ovn_controller_cname} - --sb-client-privkey ${ovn_controller_pk} - --sb-client-cert ${ovn_controller_cert} - --sb-client-cacert ${ovn_ca_cert} - --sb-cert-common-name ${ovn_controller_cname} - " - } - - ovn_acl_logging_rate_limit_flag= - if [[ -n ${ovn_acl_logging_rate_limit} ]]; then - ovn_acl_logging_rate_limit_flag="--acl-logging-rate-limit ${ovn_acl_logging_rate_limit}" - fi - - multicast_enabled_flag= - if [[ ${ovn_multicast_enable} == "true" ]]; then - multicast_enabled_flag="--enable-multicast" - fi - - egressip_enabled_flag= - if [[ ${ovn_egressip_enable} == "true" ]]; then - egressip_enabled_flag="--enable-egress-ip" - fi - - egressip_healthcheck_port_flag= - if [[ -n "${ovn_egress_ip_healthcheck_port}" ]]; then - egressip_healthcheck_port_flag="--egressip-node-healthcheck-port=${ovn_egress_ip_healthcheck_port}" - fi - - egressfirewall_enabled_flag= - if [[ ${ovn_egressfirewall_enable} == "true" ]]; then - egressfirewall_enabled_flag="--enable-egress-firewall" - fi - echo "egressfirewall_enabled_flag=${egressfirewall_enabled_flag}" - egressqos_enabled_flag= - if [[ ${ovn_egressqos_enable} == "true" ]]; then - egressqos_enabled_flag="--enable-egress-qos" - fi - - ovnkube_master_metrics_bind_address="${metrics_endpoint_ip}:9409" - local ovnkube_metrics_tls_opts="" - if [[ ${OVNKUBE_METRICS_PK} != "" && ${OVNKUBE_METRICS_CERT} != "" ]]; then - ovnkube_metrics_tls_opts=" - --node-server-privkey ${OVNKUBE_METRICS_PK} - --node-server-cert ${OVNKUBE_METRICS_CERT} - " - fi - - ovnkube_config_duration_enable_flag= - if [[ ${ovnkube_config_duration_enable} == "true" ]]; then - ovnkube_config_duration_enable_flag="--metrics-enable-config-duration" - fi - echo "ovnkube_config_duration_enable_flag: ${ovnkube_config_duration_enable_flag}" - - echo "=============== ovn-master ========== MASTER ONLY" - /usr/bin/ovnkube \ - --init-master ${K8S_NODE} \ - --cluster-subnets ${net_cidr} --k8s-service-cidr=${svc_cidr} \ - --nb-address=${ovn_nbdb} --sb-address=${ovn_sbdb} \ - --gateway-mode=${ovn_gateway_mode} \ - --loglevel=${ovnkube_loglevel} \ - --logfile-maxsize=${ovnkube_logfile_maxsize} \ - --logfile-maxbackups=${ovnkube_logfile_maxbackups} \ - --logfile-maxage=${ovnkube_logfile_maxage} \ - ${hybrid_overlay_flags} \ - ${disable_snat_multiple_gws_flag} \ - ${empty_lb_events_flag} \ - ${ovn_v4_join_subnet_opt} \ - ${ovn_v6_join_subnet_opt} \ - --pidfile ${OVN_RUNDIR}/ovnkube-master.pid \ - --logfile /var/log/ovn-kubernetes/ovnkube-master.log \ - ${ovn_master_ssl_opts} \ - ${ovnkube_metrics_tls_opts} \ - ${multicast_enabled_flag} \ - ${ovn_acl_logging_rate_limit_flag} \ - ${egressip_enabled_flag} \ - ${egressip_healthcheck_port_flag} \ - ${egressfirewall_enabled_flag} \ - ${egressqos_enabled_flag} \ - ${ovnkube_config_duration_enable_flag} \ - --metrics-bind-address ${ovnkube_master_metrics_bind_address} \ - --host-network-namespace ${ovn_host_network_namespace} & - - echo "=============== ovn-master ========== running" - wait_for_event attempts=3 process_ready ovnkube-master - - process_healthy ovnkube-master - exit 9 -} - -add-external-id-configs() { - ovs-vsctl get open . external-ids:system-id 2>&1 | grep -q "no key" - if [ $? -eq 0 ]; then - ovs-vsctl set open . external-ids:system-id="$(uuidgen)" - fi - - ovs-vsctl set open . external-ids:rundir="/var/run/openvswitch" - ovs-vsctl set open . external_ids:ovn-encap-ip="$ovn_encap_ip" - ovs-vsctl set open . external-ids:ovn-remote="{{ .Values.conf.ovn_remote }}" - ovs-vsctl set open . external-ids:ovn-encap-type="{{ .Values.conf.ovn_encap_type }}" - ovs-vsctl set open . external-ids:ovn-bridge="{{ .Values.conf.ovn_bridge }}" - ovs-vsctl set open . external-ids:ovn-bridge-mappings="{{ .Values.conf.ovn_bridge_mappings }}" - ovs-vsctl set open . external-ids:ovn-cms-options="{{ .Values.conf.ovn_cms_options }}" - - {{- if .Values.conf.use_fqdn.compute }} - ovs-vsctl set open . external-ids:hostname="$ovn_pod_host.compute" - {{- else }} - ovs-vsctl set open . external-ids:hostname="$ovn_pod_host" - {{- end }} -} - -# ovn-controller - all nodes -ovn-controller() { - add-external-id-configs - - check_ovn_daemonset_version "3" - rm -f ${OVN_RUNDIR}/ovn-controller.pid - - echo "ovn_nbdb ${ovn_nbdb} ovn_sbdb ${ovn_sbdb}" - echo "ovn_nbdb_conn ${ovn_nbdb_conn}" - - echo "=============== ovn-controller start_controller" - rm -f /var/run/ovn-kubernetes/cni/* - rm -f ${OVN_RUNDIR}/ovn-controller.*.ctl - - local ovn_controller_ssl_opts="" - [[ "yes" == ${OVN_SSL_ENABLE} ]] && { - ovn_controller_ssl_opts=" - --ovn-controller-ssl-key=${ovn_controller_pk} - --ovn-controller-ssl-cert=${ovn_controller_cert} - --ovn-controller-ssl-ca-cert=${ovn_ca_cert} - " - } - run_as_ovs_user_if_needed \ - ${OVNCTL_PATH} --no-monitor start_controller \ - --ovn-controller-priority=0 \ - ${ovn_controller_ssl_opts} \ - --ovn-controller-log="${ovn_loglevel_controller}" \ - ${ovn_controller_opts} - - tail --follow=name ${OVN_LOGDIR}/ovn-controller.log & - controller_tail_pid=$! - - wait_for_event attempts=3 process_ready ovn-controller - echo "=============== ovn-controller ========== running" - - process_healthy ovn-controller ${controller_tail_pid} - exit 10 -} - -# ovn-node - all nodes -ovn-node() { - trap 'kill $(jobs -p) ; rm -f /etc/cni/net.d/10-ovn-kubernetes.conf ; exit 0' TERM - check_ovn_daemonset_version "3" - rm -f ${OVN_RUNDIR}/ovnkube.pid - - if [[ ${ovnkube_node_mode} != "dpu-host" ]]; then - echo "=============== ovn-node - (wait for ovs)" - wait_for_event ovs_ready - fi - - echo "ovn_nbdb ${ovn_nbdb} ovn_sbdb ${ovn_sbdb} ovn_nbdb_conn ${ovn_nbdb_conn}" - - if [[ ${ovnkube_node_mode} != "dpu-host" ]]; then - echo "=============== ovn-node - (ovn-node wait for ovn-controller.pid)" - wait_for_event process_ready ovn-controller - fi - - ovn_routable_mtu_flag= - if [[ -n "${routable_mtu}" ]]; then - routable_mtu_flag="--routable-mtu ${routable_mtu}" - fi - - hybrid_overlay_flags= - if [[ ${ovn_hybrid_overlay_enable} == "true" ]]; then - hybrid_overlay_flags="--enable-hybrid-overlay" - if [[ -n "${ovn_hybrid_overlay_net_cidr}" ]]; then - hybrid_overlay_flags="${hybrid_overlay_flags} --hybrid-overlay-cluster-subnets=${ovn_hybrid_overlay_net_cidr}" - fi - fi - - disable_snat_multiple_gws_flag= - if [[ ${ovn_disable_snat_multiple_gws} == "true" ]]; then - disable_snat_multiple_gws_flag="--disable-snat-multiple-gws" - fi - - disable_pkt_mtu_check_flag= - if [[ ${ovn_disable_pkt_mtu_check} == "true" ]]; then - disable_pkt_mtu_check_flag="--disable-pkt-mtu-check" - fi - - multicast_enabled_flag= - if [[ ${ovn_multicast_enable} == "true" ]]; then - multicast_enabled_flag="--enable-multicast" - fi - - egressip_enabled_flag= - if [[ ${ovn_egressip_enable} == "true" ]]; then - egressip_enabled_flag="--enable-egress-ip" - fi - - egressip_healthcheck_port_flag= - if [[ -n "${ovn_egress_ip_healthcheck_port}" ]]; then - egressip_healthcheck_port_flag="--egressip-node-healthcheck-port=${ovn_egress_ip_healthcheck_port}" - fi - - disable_ovn_iface_id_ver_flag= - if [[ ${ovn_disable_ovn_iface_id_ver} == "true" ]]; then - disable_ovn_iface_id_ver_flag="--disable-ovn-iface-id-ver" - fi - - netflow_targets= - if [[ -n ${ovn_netflow_targets} ]]; then - netflow_targets="--netflow-targets ${ovn_netflow_targets}" - fi - - sflow_targets= - if [[ -n ${ovn_sflow_targets} ]]; then - sflow_targets="--sflow-targets ${ovn_sflow_targets}" - fi - - ipfix_targets= - if [[ -n ${ovn_ipfix_targets} ]]; then - ipfix_targets="--ipfix-targets ${ovn_ipfix_targets}" - fi - - ipfix_config= - if [[ -n ${ovn_ipfix_sampling} ]]; then - ipfix_config="--ipfix-sampling ${ovn_ipfix_sampling}" - fi - if [[ -n ${ovn_ipfix_cache_max_flows} ]]; then - ipfix_config="${ipfix_config} --ipfix-cache-max-flows ${ovn_ipfix_cache_max_flows}" - fi - if [[ -n ${ovn_ipfix_cache_active_timeout} ]]; then - ipfix_config="${ipfix_config} --ipfix-cache-active-timeout ${ovn_ipfix_cache_active_timeout}" - fi - - monitor_all= - if [[ -n ${ovn_monitor_all} ]]; then - monitor_all="--monitor-all=${ovn_monitor_all}" - fi - - ofctrl_wait_before_clear= - if [[ -n ${ovn_ofctrl_wait_before_clear} ]]; then - ofctrl_wait_before_clear="--ofctrl-wait-before-clear=${ovn_ofctrl_wait_before_clear}" - fi - - enable_lflow_cache= - if [[ -n ${ovn_enable_lflow_cache} ]]; then - enable_lflow_cache="--enable-lflow-cache=${ovn_enable_lflow_cache}" - fi - - lflow_cache_limit= - if [[ -n ${ovn_lflow_cache_limit} ]]; then - lflow_cache_limit="--lflow-cache-limit=${ovn_lflow_cache_limit}" - fi - - lflow_cache_limit_kb= - if [[ -n ${ovn_lflow_cache_limit_kb} ]]; then - lflow_cache_limit_kb="--lflow-cache-limit-kb=${ovn_lflow_cache_limit_kb}" - fi - - egress_interface= - if [[ -n ${ovn_ex_gw_network_interface} ]]; then - egress_interface="--exgw-interface ${ovn_ex_gw_network_interface}" - fi - - ovn_encap_ip_flag= - if [[ ${ovn_encap_ip} != "" ]]; then - ovn_encap_ip_flag="--encap-ip=${ovn_encap_ip}" - else - ovn_encap_ip=$(ovs-vsctl --if-exists get Open_vSwitch . external_ids:ovn-encap-ip) - if [[ $? == 0 ]]; then - ovn_encap_ip=$(echo ${ovn_encap_ip} | tr -d '\"') - if [[ "${ovn_encap_ip}" != "" ]]; then - ovn_encap_ip_flag="--encap-ip=${ovn_encap_ip}" - fi - fi - fi - - ovnkube_node_mode_flag= - if [[ ${ovnkube_node_mode} != "" ]]; then - ovnkube_node_mode_flag="--ovnkube-node-mode=${ovnkube_node_mode}" - if [[ ${ovnkube_node_mode} == "dpu" ]]; then - # encap IP is required for dpu, this is either provided via OVN_ENCAP_IP env variable or taken from ovs - if [[ ${ovn_encap_ip} == "" ]]; then - echo "ovn encap IP must be provided if \"ovnkube-node-mode\" set to \"dpu\". Exiting..." - exit 1 - fi - fi - fi - - ovnkube_node_mgmt_port_netdev_flag= - if [[ ${ovnkube_node_mgmt_port_netdev} != "" ]]; then - ovnkube_node_mgmt_port_netdev_flag="--ovnkube-node-mgmt-port-netdev=${ovnkube_node_mgmt_port_netdev}" - fi - - local ovn_node_ssl_opts="" - if [[ ${ovnkube_node_mode} != "dpu-host" ]]; then - [[ "yes" == ${OVN_SSL_ENABLE} ]] && { - ovn_node_ssl_opts=" - --nb-client-privkey ${ovn_controller_pk} - --nb-client-cert ${ovn_controller_cert} - --nb-client-cacert ${ovn_ca_cert} - --nb-cert-common-name ${ovn_controller_cname} - --sb-client-privkey ${ovn_controller_pk} - --sb-client-cert ${ovn_controller_cert} - --sb-client-cacert ${ovn_ca_cert} - --sb-cert-common-name ${ovn_controller_cname} - " - } - fi - - ovn_unprivileged_flag="--unprivileged-mode" - if test -z "${OVN_UNPRIVILEGED_MODE+x}" -o "x${OVN_UNPRIVILEGED_MODE}" = xno; then - ovn_unprivileged_flag="" - fi - - ovn_metrics_bind_address="${metrics_endpoint_ip}:9476" - ovnkube_node_metrics_bind_address="${metrics_endpoint_ip}:9410" - - local ovnkube_metrics_tls_opts="" - if [[ ${OVNKUBE_METRICS_PK} != "" && ${OVNKUBE_METRICS_CERT} != "" ]]; then - ovnkube_metrics_tls_opts=" - --node-server-privkey ${OVNKUBE_METRICS_PK} - --node-server-cert ${OVNKUBE_METRICS_CERT} - " - fi - - echo "=============== ovn-node --init-node" - /usr/bin/ovnkube --init-node ${K8S_NODE} \ - --cluster-subnets ${net_cidr} --k8s-service-cidr=${svc_cidr} \ - --nb-address=${ovn_nbdb} --sb-address=${ovn_sbdb} \ - ${ovn_unprivileged_flag} \ - --nodeport \ - --mtu=${mtu} \ - ${routable_mtu_flag} \ - ${ovn_encap_ip_flag} \ - --loglevel=${ovnkube_loglevel} \ - --logfile-maxsize=${ovnkube_logfile_maxsize} \ - --logfile-maxbackups=${ovnkube_logfile_maxbackups} \ - --logfile-maxage=${ovnkube_logfile_maxage} \ - ${hybrid_overlay_flags} \ - ${disable_snat_multiple_gws_flag} \ - ${disable_pkt_mtu_check_flag} \ - --gateway-mode=${ovn_gateway_mode} ${ovn_gateway_opts} \ - --gateway-router-subnet=${ovn_gateway_router_subnet} \ - --pidfile ${OVN_RUNDIR}/ovnkube.pid \ - --logfile /var/log/ovn-kubernetes/ovnkube.log \ - ${ovn_node_ssl_opts} \ - ${ovnkube_metrics_tls_opts} \ - --inactivity-probe=${ovn_remote_probe_interval} \ - ${monitor_all} \ - ${ofctrl_wait_before_clear} \ - ${enable_lflow_cache} \ - ${lflow_cache_limit} \ - ${lflow_cache_limit_kb} \ - ${multicast_enabled_flag} \ - ${egressip_enabled_flag} \ - ${egressip_healthcheck_port_flag} \ - ${disable_ovn_iface_id_ver_flag} \ - ${netflow_targets} \ - ${sflow_targets} \ - ${ipfix_targets} \ - ${ipfix_config} \ - --ovn-metrics-bind-address ${ovn_metrics_bind_address} \ - --metrics-bind-address ${ovnkube_node_metrics_bind_address} \ - ${ovnkube_node_mode_flag} \ - ${egress_interface} \ - --host-network-namespace ${ovn_host_network_namespace} \ - ${ovnkube_node_mgmt_port_netdev_flag} & - - wait_for_event attempts=3 process_ready ovnkube - if [[ ${ovnkube_node_mode} != "dpu" ]]; then - setup_cni - fi - echo "=============== ovn-node ========== running" - - process_healthy ovnkube - exit 7 -} - -# cleanup-ovn-node - all nodes -cleanup-ovn-node() { - check_ovn_daemonset_version "3" - - rm -f /etc/cni/net.d/10-ovn-kubernetes.conf - - echo "=============== time: $(date +%d-%m-%H:%M:%S:%N) cleanup-ovn-node - (wait for ovn-controller to exit)" - retries=0 - while [[ ${retries} -lt 80 ]]; do - process_ready ovn-controller - if [[ $? != 0 ]]; then - break - fi - echo "=============== time: $(date +%d-%m-%H:%M:%S:%N) cleanup-ovn-node - (ovn-controller still running, wait)" - sleep 1 - ((retries += 1)) - done - - echo "=============== time: $(date +%d-%m-%H:%M:%S:%N) cleanup-ovn-node --cleanup-node" - /usr/bin/ovnkube --cleanup-node ${K8S_NODE} --gateway-mode=${ovn_gateway_mode} ${ovn_gateway_opts} \ - --k8s-token=${k8s_token} --k8s-apiserver=${K8S_APISERVER} --k8s-cacert=${K8S_CACERT} \ - --loglevel=${ovnkube_loglevel} \ - --logfile /var/log/ovn-kubernetes/ovnkube.log - -} - -# v3 - Runs ovn-kube-util in daemon mode to export prometheus metrics related to OVS. -ovs-metrics() { - check_ovn_daemonset_version "3" - - echo "=============== ovs-metrics - (wait for ovs_ready)" - wait_for_event ovs_ready - - ovs_exporter_bind_address="${metrics_endpoint_ip}:9310" - /usr/bin/ovn-kube-util \ - --loglevel=${ovnkube_loglevel} \ - ovs-exporter \ - --metrics-bind-address ${ovs_exporter_bind_address} - - echo "=============== ovs-metrics with pid ${?} terminated ========== " - exit 1 -} - -echo "================== ovnkube.sh --- version: ${ovnkube_version} ================" - -echo " ==================== command: ${cmd}" -display_version - -# display_env - -# Start the requested daemons -# daemons come up in order -# ovs-db-server - all nodes -- not done by this script (v3) -# ovs-vswitchd - all nodes -- not done by this script (v3) -# run-ovn-northd Runs ovn-northd as a process does not run nb_ovsdb or sb_ovsdb (v3) -# nb-ovsdb Runs nb_ovsdb as a process (no detach or monitor) (v3) -# sb-ovsdb Runs sb_ovsdb as a process (no detach or monitor) (v3) -# ovn-dbchecker Runs ovndb checker alongside nb-ovsdb and sb-ovsdb containers (v3) -# ovn-master - master only (v3) -# ovn-controller - all nodes (v3) -# ovn-node - all nodes (v3) -# cleanup-ovn-node - all nodes (v3) - -get_ovn_db_vars - -case ${cmd} in -"nb-ovsdb") # pod ovnkube-db container nb-ovsdb - nb-ovsdb - ;; -"sb-ovsdb") # pod ovnkube-db container sb-ovsdb - sb-ovsdb - ;; -"ovn-dbchecker") # pod ovnkube-db container ovn-dbchecker - ovn-dbchecker - ;; -"run-ovn-northd") # pod ovnkube-master container run-ovn-northd - run-ovn-northd - ;; -"ovn-master") # pod ovnkube-master container ovnkube-master - ovn-master - ;; -"ovs-server") # pod ovnkube-node container ovs-daemons - ovs-server - ;; -"ovn-controller") # pod ovnkube-node container ovn-controller - ovn-controller - ;; -"ovn-node") # pod ovnkube-node container ovn-node - ovn-node - ;; -"ovn-northd") - ovn-northd - ;; -"display_env") - display_env - exit 0 - ;; -"display") - display - exit 0 - ;; -"ovn_debug") - ovn_debug - exit 0 - ;; -"cleanup-ovs-server") - cleanup-ovs-server - ;; -"cleanup-ovn-node") - cleanup-ovn-node - ;; -"nb-ovsdb-raft") - ovsdb-raft nb ${ovn_nb_port} ${ovn_nb_raft_port} ${ovn_nb_raft_election_timer} - ;; -"sb-ovsdb-raft") - ovsdb-raft sb ${ovn_sb_port} ${ovn_sb_raft_port} ${ovn_sb_raft_election_timer} - ;; -"ovs-metrics") - ovs-metrics - ;; -*) - echo "invalid command ${cmd}" - echo "valid v3 commands: ovs-server nb-ovsdb sb-ovsdb run-ovn-northd ovn-master " \ - "ovn-controller ovn-node display_env display ovn_debug cleanup-ovs-server " \ - "cleanup-ovn-node nb-ovsdb-raft sb-ovsdb-raft" - exit 0 - ;; -esac - -exit 0 diff --git a/ovn/templates/bin/_ovsdb-server.sh.tpl b/ovn/templates/bin/_ovsdb-server.sh.tpl new file mode 100644 index 000000000..e023505be --- /dev/null +++ b/ovn/templates/bin/_ovsdb-server.sh.tpl @@ -0,0 +1,72 @@ +#!/bin/bash -xe + +# Copyright 2023 VEXXHOST, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +COMMAND="${@:-start}" + +OVSDB_HOST=$(hostname -f) +ARGS=( + --db-${OVS_DATABASE}-create-insecure-remote=yes + --db-${OVS_DATABASE}-cluster-local-proto=tcp + --db-${OVS_DATABASE}-cluster-local-addr=$(hostname -f) +) + +if [[ ! $HOSTNAME == *-0 && $OVSDB_HOST =~ (.+)-([0-9]+)\. ]]; then + OVSDB_BOOTSTRAP_HOST="${BASH_REMATCH[1]}-0.${OVSDB_HOST#*.}" + + ARGS+=( + --db-${OVS_DATABASE}-cluster-remote-proto=tcp + --db-${OVS_DATABASE}-cluster-remote-addr=${OVSDB_BOOTSTRAP_HOST} + ) +fi + +function start () { + /usr/share/ovn/scripts/ovn-ctl start_${OVS_DATABASE}_ovsdb ${ARGS[@]} + + tail --follow=name /var/log/ovn/ovsdb-server-${OVS_DATABASE}.log +} + +function stop () { + /usr/share/ovn/scripts/ovn-ctl stop_${OVS_DATABASE}_ovsdb + pkill tail +} + +function liveness () { + if [[ $OVS_DATABASE == "nb" ]]; then + OVN_DATABASE="Northbound" + elif [[ $OVS_DATABASE == "sb" ]]; then + OVN_DATABASE="Southbound" + else + echo "OVS_DATABASE must be nb or sb" + exit 1 + fi + + ovs-appctl -t /var/run/ovn/ovn${OVS_DATABASE}_db.ctl cluster/status OVN_${OVN_DATABASE} +} + +function readiness () { + if [[ $OVS_DATABASE == "nb" ]]; then + OVN_DATABASE="Northbound" + elif [[ $OVS_DATABASE == "sb" ]]; then + OVN_DATABASE="Southbound" + else + echo "OVS_DATABASE must be nb or sb" + exit 1 + fi + + ovs-appctl -t /var/run/ovn/ovn${OVS_DATABASE}_db.ctl cluster/status OVN_${OVN_DATABASE} +} + +$COMMAND diff --git a/ovn/templates/configmap-bin.yaml b/ovn/templates/configmap-bin.yaml index 6da9af1e5..a849dd8ae 100644 --- a/ovn/templates/configmap-bin.yaml +++ b/ovn/templates/configmap-bin.yaml @@ -24,8 +24,12 @@ data: image-repo-sync.sh: | {{- include "helm-toolkit.scripts.image_repo_sync" . | indent 4 }} {{- end }} - ovn.sh: | -{{ tuple "bin/_ovn.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} - ovn-setup-bridges-init.sh: | -{{ tuple "bin/_ovn-setup-bridges-init.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + ovsdb-server.sh: | +{{ tuple "bin/_ovsdb-server.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + ovn-northd.sh: | +{{ tuple "bin/_ovn-northd.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + ovn-controller-init.sh: | +{{ tuple "bin/_ovn-controller-init.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + ovn-controller.sh: | +{{ tuple "bin/_ovn-controller.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} {{- end }} diff --git a/ovn/templates/daemonset-controller.yaml b/ovn/templates/daemonset-controller.yaml index c1bcda3f5..32222ee3f 100644 --- a/ovn/templates/daemonset-controller.yaml +++ b/ovn/templates/daemonset-controller.yaml @@ -38,20 +38,22 @@ spec: {{ tuple $envAll "ovn" "ovn-controller" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} annotations: {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} + configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} + configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }} spec: serviceAccountName: {{ $serviceAccountName }} nodeSelector: {{ .Values.labels.ovn_controller.node_selector_key }}: {{ .Values.labels.ovn_controller.node_selector_value }} initContainers: {{- tuple $envAll "ovn_controller" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} - - name: ovn-setup-bridge + - name: controller-init {{ tuple $envAll "ovn_controller" | include "helm-toolkit.snippets.image" | indent 10 }} command: - - /tmp/ovn-setup-bridges-init.sh + - /tmp/ovn-controller-init.sh volumeMounts: - name: ovn-bin - mountPath: /tmp/ovn-setup-bridges-init.sh - subPath: ovn-setup-bridges-init.sh + mountPath: /tmp/ovn-controller-init.sh + subPath: ovn-controller-init.sh readOnly: true - name: run-openvswitch mountPath: /run/openvswitch @@ -60,25 +62,23 @@ spec: subPath: auto_bridge_add readOnly: true containers: - - name: ovn-controller + - name: controller {{ tuple $envAll "ovn_controller" | include "helm-toolkit.snippets.image" | indent 10 }} - command: - - /tmp/start.sh - - ovn-controller {{ tuple $envAll $envAll.Values.pod.resources.server | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} - env: - - name: K8S_NODE - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: OVN_ENCAP_IP - valueFrom: - fieldRef: - fieldPath: status.hostIP +{{ dict "envAll" $envAll "application" "ovn_controller" "container" "controller" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + command: + - /tmp/ovn-controller.sh + - start + lifecycle: + preStop: + exec: + command: + - /tmp/ovn-controller.sh + - stop volumeMounts: - name: ovn-bin - mountPath: /tmp/start.sh - subPath: ovn.sh + mountPath: /tmp/ovn-controller.sh + subPath: ovn-controller.sh readOnly: true - name: run-openvswitch mountPath: /run/openvswitch diff --git a/ovn/templates/deployment-northd.yaml b/ovn/templates/deployment-northd.yaml index e7c30dba4..e3afdd05b 100644 --- a/ovn/templates/deployment-northd.yaml +++ b/ovn/templates/deployment-northd.yaml @@ -12,6 +12,20 @@ See the License for the specific language governing permissions and limitations under the License. */}} +{{- define "livenessProbeTemplate" }} +exec: + command: + - /tmp/ovn-northd.sh + - liveness +{{- end }} + +{{- define "readinessProbeTemplate" }} +exec: + command: + - /tmp/ovn-northd.sh + - readiness +{{- end }} + {{- if .Values.manifests.deployment_northd }} {{- $envAll := . }} @@ -24,13 +38,10 @@ metadata: name: ovn-northd annotations: {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} - configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} labels: {{ tuple $envAll "ovn" "ovn-northd" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} spec: - replicas: 1 - strategy: - type: Recreate + replicas: {{ .Values.pod.replicas.ovn_northd }} selector: matchLabels: {{ tuple $envAll "ovn" "ovn-northd" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }} @@ -40,6 +51,7 @@ spec: {{ tuple $envAll "ovn" "ovn-northd" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} annotations: {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} + configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} spec: serviceAccountName: {{ $serviceAccountName }} nodeSelector: @@ -47,16 +59,25 @@ spec: initContainers: {{- tuple $envAll "ovn_northd" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} containers: - - name: ovn-northd + - name: northd {{ tuple $envAll "ovn_northd" | include "helm-toolkit.snippets.image" | indent 10 }} - command: - - /tmp/start.sh - - run-ovn-northd {{ tuple $envAll $envAll.Values.pod.resources.server | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "ovn_northd" "container" "northd" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} +{{ dict "envAll" . "component" "ovn_northd" "container" "northd" "type" "liveness" "probeTemplate" (include "livenessProbeTemplate" . | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }} +{{ dict "envAll" . "component" "ovn_northd" "container" "northd" "type" "readiness" "probeTemplate" (include "readinessProbeTemplate" . | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }} + command: + - /tmp/ovn-northd.sh + - start + lifecycle: + preStop: + exec: + command: + - /tmp/ovn-northd.sh + - stop volumeMounts: - name: ovn-bin - mountPath: /tmp/start.sh - subPath: ovn.sh + mountPath: /tmp/ovn-northd.sh + subPath: ovn-northd.sh readOnly: true volumes: - name: ovn-bin diff --git a/ovn/templates/service-nb-db.yaml b/ovn/templates/service-ovsdb-nb.yaml similarity index 54% rename from ovn/templates/service-nb-db.yaml rename to ovn/templates/service-ovsdb-nb.yaml index 7599c30d9..b93da9b8b 100644 --- a/ovn/templates/service-nb-db.yaml +++ b/ovn/templates/service-ovsdb-nb.yaml @@ -12,17 +12,19 @@ See the License for the specific language governing permissions and limitations under the License. */}} -{{- if .Values.manifests.service_ovn_nb_db }} +{{- if .Values.manifests.service_ovn_ovsdb_nb }} {{- $envAll := . }} --- apiVersion: v1 kind: Service metadata: - name: {{ tuple "ovn-nb-db" "direct" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} + name: {{ tuple "ovn-ovsdb-nb" "direct" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} spec: ports: - - name: ovn-nb-db - port: {{ tuple "ovn-nb-db" "internal" "db" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + - name: ovsdb + port: {{ tuple "ovn-ovsdb-nb" "internal" "ovsdb" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + - name: raft + port: {{ tuple "ovn-ovsdb-nb" "internal" "raft" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} selector: -{{ tuple $envAll "ovn" "ovn-nb-db" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +{{ tuple $envAll "ovn" "ovn-ovsdb-nb" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} {{- end }} diff --git a/ovn/templates/service-sb-db.yaml b/ovn/templates/service-ovsdb-sb.yaml similarity index 54% rename from ovn/templates/service-sb-db.yaml rename to ovn/templates/service-ovsdb-sb.yaml index c3723f9d6..70f62c6e4 100644 --- a/ovn/templates/service-sb-db.yaml +++ b/ovn/templates/service-ovsdb-sb.yaml @@ -12,17 +12,19 @@ See the License for the specific language governing permissions and limitations under the License. */}} -{{- if .Values.manifests.service_ovn_sb_db }} +{{- if .Values.manifests.service_ovn_ovsdb_sb }} {{- $envAll := . }} --- apiVersion: v1 kind: Service metadata: - name: {{ tuple "ovn-sb-db" "direct" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} + name: {{ tuple "ovn-ovsdb-sb" "direct" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} spec: ports: - - name: ovn-sb-db - port: {{ tuple "ovn-sb-db" "internal" "db" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + - name: ovsdb + port: {{ tuple "ovn-ovsdb-sb" "internal" "ovsdb" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + - name: raft + port: {{ tuple "ovn-ovsdb-sb" "internal" "raft" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} selector: -{{ tuple $envAll "ovn" "ovn-sb-db" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +{{ tuple $envAll "ovn" "ovn-ovsdb-sb" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} {{- end }} diff --git a/ovn/templates/statefulset-nb-db.yaml b/ovn/templates/statefulset-nb-db.yaml deleted file mode 100644 index 78d7b56a2..000000000 --- a/ovn/templates/statefulset-nb-db.yaml +++ /dev/null @@ -1,85 +0,0 @@ -{{/* -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/}} - -{{- if .Values.manifests.statefulset_ovn_nb_db }} -{{- $envAll := . }} - -{{- $serviceAccountName := "ovn-nb-db" }} -{{ tuple $envAll "ovn_nb_db" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} ---- -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: ovn-nb-db - annotations: - {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} - labels: -{{ tuple $envAll "ovn" "ovn-nb-db" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} -spec: - serviceName: {{ tuple "ovn-nb-db" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} - replicas: 1 - selector: - matchLabels: -{{ tuple $envAll "ovn" "ovn-nb-db" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }} - template: - metadata: - labels: -{{ tuple $envAll "ovn" "ovn-nb-db" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} - annotations: -{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} - spec: - serviceAccountName: {{ $serviceAccountName }} - affinity: -{{- tuple $envAll "ovn" "ovn-nb-db" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }} - nodeSelector: - {{ .Values.labels.ovn_nb_db.node_selector_key }}: {{ .Values.labels.ovn_nb_db.node_selector_value }} - initContainers: -{{- tuple $envAll "ovn_nb_db" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} - containers: - - name: ovn-nb-db -{{ tuple $envAll "ovn_nb_db" | include "helm-toolkit.snippets.image" | indent 10 }} - ports: - - containerPort: {{ tuple "ovn-nb-db" "internal" "db" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} - command: - - /tmp/start.sh - - nb-ovsdb -{{ tuple $envAll $envAll.Values.pod.resources.server | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} - volumeMounts: - - name: ovn-bin - mountPath: /tmp/start.sh - subPath: ovn.sh - readOnly: true - - name: ovn-nb-db-data - mountPath: /var/lib/ovn - volumes: - - name: ovn-bin - configMap: - name: ovn-bin - defaultMode: 0555 -{{- if not .Values.volume.ovn_nb_db.enabled }} - - name: ovn-nb-db-data - emptyDir: {} -{{- else }} - volumeClaimTemplates: - - metadata: - name: ovn-nb-db-data - spec: - accessModes: ["ReadWriteOnce"] - resources: - requests: - storage: {{ $envAll.Values.volume.ovn_nb_db.size }} - storageClassName: {{ $envAll.Values.volume.ovn_nb_db.class_name }} -{{- end }} - -{{- end }} diff --git a/ovn/templates/statefulset-ovsdb-nb.yaml b/ovn/templates/statefulset-ovsdb-nb.yaml new file mode 100644 index 000000000..bda1ab962 --- /dev/null +++ b/ovn/templates/statefulset-ovsdb-nb.yaml @@ -0,0 +1,102 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.statefulset_ovn_ovsdb_nb }} +{{- $envAll := . }} + +{{- $serviceAccountName := "ovn-ovsdb-nb" }} +{{ tuple $envAll "ovn_ovsdb_nb" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: ovn-ovsdb-nb + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} + labels: +{{ tuple $envAll "ovn" "ovn-ovsdb-nb" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + serviceName: {{ tuple "ovn-ovsdb-nb" "direct" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} + replicas: {{ .Values.pod.replicas.ovn_ovsdb_nb }} + selector: + matchLabels: +{{ tuple $envAll "ovn" "ovn-ovsdb-nb" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }} + template: + metadata: + labels: +{{ tuple $envAll "ovn" "ovn-ovsdb-nb" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} + configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} + spec: + serviceAccountName: {{ $serviceAccountName }} + affinity: +{{- tuple $envAll "ovn" "ovn-ovsdb-nb" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }} + nodeSelector: + {{ .Values.labels.ovn_ovsdb_nb.node_selector_key }}: {{ .Values.labels.ovn_ovsdb_nb.node_selector_value }} + initContainers: +{{- tuple $envAll "ovn_ovsdb_nb" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: ovsdb +{{ tuple $envAll "ovn_ovsdb_nb" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.server | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + ports: + - containerPort: {{ tuple "ovn-ovsdb-nb" "internal" "ovsdb" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + - containerPort: {{ tuple "ovn-ovsdb-nb" "internal" "raft" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + env: + - name: OVS_DATABASE + value: nb + - name: OVS_PORT + value: "{{ tuple "ovn-ovsdb-nb" "internal" "ovsdb" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}" + command: + - /tmp/ovsdb-server.sh + - start + lifecycle: + preStop: + exec: + command: + - /tmp/ovsdb-server.sh + - stop + volumeMounts: + - name: ovn-bin + mountPath: /tmp/ovsdb-server.sh + subPath: ovsdb-server.sh + readOnly: true + - name: run-openvswitch + mountPath: /run/openvswitch + - name: data + mountPath: /var/lib/ovn + volumes: + - name: run-openvswitch + emptyDir: {} + - name: ovn-bin + configMap: + name: ovn-bin + defaultMode: 0555 +{{- if not .Values.volume.ovn_ovsdb_nb.enabled }} + - name: data + emptyDir: {} +{{- else }} + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: ["ReadWriteOnce"] + storageClassName: {{ $envAll.Values.volume.ovn_ovsdb_nb.class_name }} + resources: + requests: + storage: {{ $envAll.Values.volume.ovn_ovsdb_nb.size }} +{{- end }} + +{{- end }} diff --git a/ovn/templates/statefulset-ovsdb-sb.yaml b/ovn/templates/statefulset-ovsdb-sb.yaml new file mode 100644 index 000000000..3f4c6b945 --- /dev/null +++ b/ovn/templates/statefulset-ovsdb-sb.yaml @@ -0,0 +1,102 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.statefulset_ovn_ovsdb_sb }} +{{- $envAll := . }} + +{{- $serviceAccountName := "ovn-ovsdb-sb" }} +{{ tuple $envAll "ovn_ovsdb_sb" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: ovn-ovsdb-sb + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} + labels: +{{ tuple $envAll "ovn" "ovn-ovsdb-sb" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + serviceName: {{ tuple "ovn-ovsdb-sb" "direct" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} + replicas: {{ .Values.pod.replicas.ovn_ovsdb_sb }} + selector: + matchLabels: +{{ tuple $envAll "ovn" "ovn-ovsdb-sb" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }} + template: + metadata: + labels: +{{ tuple $envAll "ovn" "ovn-ovsdb-sb" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} + configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} + spec: + serviceAccountName: {{ $serviceAccountName }} + affinity: +{{- tuple $envAll "ovn" "ovn-ovsdb-sb" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }} + nodeSelector: + {{ .Values.labels.ovn_ovsdb_sb.node_selector_key }}: {{ .Values.labels.ovn_ovsdb_sb.node_selector_value }} + initContainers: +{{- tuple $envAll "ovn_ovsdb_sb" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: ovsdb +{{ tuple $envAll "ovn_ovsdb_sb" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.server | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + ports: + - containerPort: {{ tuple "ovn-ovsdb-sb" "internal" "ovsdb" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + - containerPort: {{ tuple "ovn-ovsdb-sb" "internal" "raft" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + env: + - name: OVS_DATABASE + value: sb + - name: OVS_PORT + value: "{{ tuple "ovn-ovsdb-sb" "internal" "ovsdb" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}" + command: + - /tmp/ovsdb-server.sh + - start + lifecycle: + preStop: + exec: + command: + - /tmp/ovsdb-server.sh + - stop + volumeMounts: + - name: ovn-bin + mountPath: /tmp/ovsdb-server.sh + subPath: ovsdb-server.sh + readOnly: true + - name: run-openvswitch + mountPath: /run/openvswitch + - name: data + mountPath: /var/lib/ovn + volumes: + - name: run-openvswitch + emptyDir: {} + - name: ovn-bin + configMap: + name: ovn-bin + defaultMode: 0555 +{{- if not .Values.volume.ovn_ovsdb_sb.enabled }} + - name: data + emptyDir: {} +{{- else }} + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: {{ $envAll.Values.volume.ovn_ovsdb_sb.size }} + storageClassName: {{ $envAll.Values.volume.ovn_ovsdb_sb.class_name }} +{{- end }} + +{{- end }} diff --git a/ovn/templates/statefulset-sb-db.yaml b/ovn/templates/statefulset-sb-db.yaml deleted file mode 100644 index 37c2ee0a6..000000000 --- a/ovn/templates/statefulset-sb-db.yaml +++ /dev/null @@ -1,85 +0,0 @@ -{{/* -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/}} - -{{- if .Values.manifests.statefulset_ovn_sb_db }} -{{- $envAll := . }} - -{{- $serviceAccountName := "ovn-sb-db" }} -{{ tuple $envAll "ovn_sb_db" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} ---- -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: ovn-sb-db - annotations: - {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} - labels: -{{ tuple $envAll "ovn" "ovn-sb-db" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} -spec: - serviceName: {{ tuple "ovn-sb-db" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} - replicas: 1 - selector: - matchLabels: -{{ tuple $envAll "ovn" "ovn-sb-db" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }} - template: - metadata: - labels: -{{ tuple $envAll "ovn" "ovn-sb-db" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} - annotations: -{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} - spec: - serviceAccountName: {{ $serviceAccountName }} - affinity: -{{- tuple $envAll "ovn" "ovn-sb-db" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }} - nodeSelector: - {{ .Values.labels.ovn_sb_db.node_selector_key }}: {{ .Values.labels.ovn_sb_db.node_selector_value }} - initContainers: -{{- tuple $envAll "ovn_sb_db" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} - containers: - - name: ovn-sb-db -{{ tuple $envAll "ovn_sb_db" | include "helm-toolkit.snippets.image" | indent 10 }} - ports: - - containerPort: {{ tuple "ovn-sb-db" "internal" "db" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} - command: - - /tmp/start.sh - - sb-ovsdb -{{ tuple $envAll $envAll.Values.pod.resources.server | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} - volumeMounts: - - name: ovn-bin - mountPath: /tmp/start.sh - subPath: ovn.sh - readOnly: true - - name: ovn-sb-db-data - mountPath: /var/lib/ovn - volumes: - - name: ovn-bin - configMap: - name: ovn-bin - defaultMode: 0555 -{{- if not .Values.volume.ovn_sb_db.enabled }} - - name: ovn-sb-db-data - emptyDir: {} -{{- else }} - volumeClaimTemplates: - - metadata: - name: ovn-sb-db-data - spec: - accessModes: ["ReadWriteOnce"] - resources: - requests: - storage: {{ $envAll.Values.volume.ovn_sb_db.size }} - storageClassName: {{ $envAll.Values.volume.ovn_sb_db.class_name }} -{{- end }} - -{{- end }} diff --git a/ovn/values.yaml b/ovn/values.yaml index 4c8148c1b..3e3d69da2 100644 --- a/ovn/values.yaml +++ b/ovn/values.yaml @@ -20,8 +20,8 @@ release_group: null images: tags: - ovn_nb_db: docker.io/openstackhelm/ovn:latest-ubuntu_focal - ovn_sb_db: docker.io/openstackhelm/ovn:latest-ubuntu_focal + ovn_ovsdb_nb: docker.io/openstackhelm/ovn:latest-ubuntu_focal + ovn_ovsdb_sb: docker.io/openstackhelm/ovn:latest-ubuntu_focal ovn_northd: docker.io/openstackhelm/ovn:latest-ubuntu_focal ovn_controller: docker.io/openstackhelm/ovn:latest-ubuntu_focal dep_check: quay.io/airshipit/kubernetes-entrypoint:v1.0.0 @@ -34,10 +34,10 @@ images: - image_repo_sync labels: - ovn_nb_db: + ovn_ovsdb_nb: node_selector_key: openstack-network-node node_selector_value: enabled - ovn_sb_db: + ovn_ovsdb_sb: node_selector_key: openstack-network-node node_selector_value: enabled ovn_northd: @@ -48,30 +48,28 @@ labels: node_selector_value: enabled volume: - ovn_nb_db: - use_local_path: - enabled: false - host_path: /var/lib/rabbitmq - chown_on_start: true + ovn_ovsdb_nb: enabled: true class_name: general size: 5Gi - ovn_sb_db: - use_local_path: - enabled: false - host_path: /var/lib/rabbitmq - chown_on_start: true + ovn_ovsdb_sb: enabled: true class_name: general size: 5Gi +network: + interface: + # Tunnel interface will be used for VXLAN tunneling. + tunnel: null + # If tunnel is null there is a fallback mechanism to search + # for interface with routing using tunnel network cidr. + tunnel_network_cidr: "0/0" + conf: ovn_cms_options: "enable-chassis-as-gw,availability-zones=nova" - ovn_remote: tcp:ovn-sb-db.openstack.svc.cluster.local:6640 ovn_encap_type: geneve ovn_bridge: br-int - # ovn_bridge_mappings: "physnet-public:br-public,physnet-private:br-private" - ovn_bridge_mappings: "" + ovn_bridge_mappings: external:br-ex # auto_bridge_add: # br-private: eth0 @@ -83,10 +81,23 @@ conf: compute: true pod: + security_context: + ovn_northd: + container: + northd: + capabilities: + add: + - SYS_NICE + ovn_controller: + container: + controller: + capabilities: + add: + - SYS_NICE tolerations: - ovn_nb_db: + ovn_ovsdb_nb: enabled: false - ovn_sb_db: + ovn_ovsdb_sb: enabled: false ovn_northd: enabled: false @@ -102,17 +113,32 @@ pod: default: 10 probes: - # TODO: Add healthchecks + ovn_northd: + northd: + readiness: + enabled: true + params: + initialDelaySeconds: 5 + timeoutSeconds: 10 + liveness: + enabled: true + params: + initialDelaySeconds: 5 + timeoutSeconds: 10 dns_policy: "ClusterFirstWithHostNet" + replicas: + ovn_ovsdb_nb: 1 + ovn_ovsdb_sb: 1 + ovn_northd: 1 lifecycle: upgrades: daemonsets: pod_replacement_strategy: RollingUpdate - ovn_nb_db: + ovn_ovsdb_nb: enabled: true min_ready_seconds: 0 max_unavailable: 1 - ovn_sb_db: + ovn_ovsdb_sb: enabled: true min_ready_seconds: 0 max_unavailable: 1 @@ -127,14 +153,14 @@ pod: resources: enabled: false ovs: - ovn_nb_db: + ovn_ovsdb_nb: requests: memory: "128Mi" cpu: "100m" limits: memory: "1024Mi" cpu: "2000m" - ovn_sb_db: + ovn_ovsdb_sb: requests: memory: "128Mi" cpu: "100m" @@ -166,8 +192,8 @@ pod: secrets: oci_image_registry: - ovn_nb_db: ovn-nb-db-oci-image-registry-key - ovn_sb_db: ovn-sb-db-oci-image-registry-key + ovn_ovsdb_nb: ovn-ovsdb-nb-oci-image-registry-key + ovn_ovsdb_sb: ovn-ovsdb-sb-oci-image-registry-key ovn_northd: ovn-northd-oci-image-registry-key ovn_controller: ovn-controller-oci-image-registry-key @@ -201,34 +227,38 @@ endpoints: port: registry: default: null - ovn_nb_db: - name: ovn-nb-db + ovn_ovsdb_nb: + name: ovn-ovsdb-nb namespace: null hosts: - default: ovn-nb-db + default: ovn-ovsdb-nb host_fqdn_override: default: null port: - db: - default: 6640 - ovn_sb_db: - name: ovn-sb-db + ovsdb: + default: 6641 + raft: + default: 6643 + ovn_ovsdb_sb: + name: ovn-ovsdb-sb namespace: null hosts: - default: ovn-sb-db + default: ovn-ovsdb-sb host_fqdn_override: default: null port: - db: - default: 6640 + ovsdb: + default: 6642 + raft: + default: 6644 network_policy: - ovn_nb_db: + ovn_ovsdb_nb: ingress: - {} egress: - {} - ovn_sb_db: + ovn_ovsdb_sb: ingress: - {} egress: @@ -254,18 +284,18 @@ dependencies: - endpoint: node service: local_image_registry static: - ovn_nb_db: null - ovn_sb_db: null + ovn_ovsdb_nb: null + ovn_ovsdb_sb: null ovn_northd: services: - endpoint: internal - service: ovn-nb-db + service: ovn-ovsdb-nb - endpoint: internal - service: ovn-sb-db + service: ovn-ovsdb-sb ovn_controller: services: - endpoint: internal - service: ovn-sb-db + service: ovn-ovsdb-sb pod: - requireSameNode: true labels: @@ -281,10 +311,10 @@ manifests: configmap_etc: true deployment_northd: true daemonset_controller: true - service_ovn_nb_db: true - service_ovn_sb_db: true - statefulset_ovn_nb_db: true - statefulset_ovn_sb_db: true + service_ovn_ovsdb_nb: true + service_ovn_ovsdb_sb: true + statefulset_ovn_ovsdb_nb: true + statefulset_ovn_ovsdb_sb: true deployment_ovn_northd: true daemonset_ovn_controller: true job_image_repo_sync: true diff --git a/releasenotes/notes/ovn.yaml b/releasenotes/notes/ovn.yaml index e60055fef..8c3b7c4f1 100644 --- a/releasenotes/notes/ovn.yaml +++ b/releasenotes/notes/ovn.yaml @@ -4,4 +4,5 @@ ovn: - 0.1.1 Fix ovn db persistence issue - 0.1.2 Add bridge-mapping configuration - 0.1.3 Fix system-id reuse + - 0.1.4 Add support for OVN HA + refactor ...