From fb90642b1809418e042cf988b777eff31e339af2 Mon Sep 17 00:00:00 2001 From: Vladimir Kozhukalov Date: Tue, 19 Mar 2024 15:35:02 -0500 Subject: [PATCH] Update ovn controller init script - OVN init script must be able to attach an interface to the provider network bridge and migrate IP from the interface to the bridge exactly like Neutron OVS agent init script does it. - OVN init script sets gateway option to those OVN controller instances which are running on nodes with l3-agent=enabled label. Change-Id: I24345c1f85c1e75af6e804f09d35abf530ddd6b4 --- ovn/Chart.yaml | 2 +- ovn/templates/bin/_ovn-controller-init.sh.tpl | 65 ++++++++++++++++++- ovn/templates/daemonset-controller.yaml | 47 ++++++++++++++ ovn/values.yaml | 4 +- releasenotes/notes/ovn.yaml | 1 + zuul.d/jobs.yaml | 5 ++ zuul.d/project.yaml | 1 + 7 files changed, 120 insertions(+), 5 deletions(-) diff --git a/ovn/Chart.yaml b/ovn/Chart.yaml index 38c8fb519..9f1fbd010 100644 --- a/ovn/Chart.yaml +++ b/ovn/Chart.yaml @@ -15,7 +15,7 @@ apiVersion: v1 appVersion: v23.3.0 description: OpenStack-Helm OVN name: ovn -version: 0.1.7 +version: 0.1.8 home: https://www.ovn.org icon: https://www.ovn.org/images/ovn-logo.png sources: diff --git a/ovn/templates/bin/_ovn-controller-init.sh.tpl b/ovn/templates/bin/_ovn-controller-init.sh.tpl index 67e3cccc8..a4d8130ff 100644 --- a/ovn/templates/bin/_ovn-controller-init.sh.tpl +++ b/ovn/templates/bin/_ovn-controller-init.sh.tpl @@ -16,13 +16,65 @@ function get_ip_address_from_interface { local interface=$1 - local ip=$(ip -4 -o addr s "${interface}" | awk '{ print $4; exit }' | awk -F '/' '{print $1}') + local ip=$(ip -4 -o addr s "${interface}" | awk '{ print $4; exit }' | awk -F '/' 'NR==1 {print $1}') if [ -z "${ip}" ] ; then exit 1 fi echo ${ip} } +function get_ip_prefix_from_interface { + local interface=$1 + local prefix=$(ip -4 -o addr s "${interface}" | awk '{ print $4; exit }' | awk -F '/' 'NR==1 {print $2}') + if [ -z "${prefix}" ] ; then + exit 1 + fi + echo ${prefix} +} + +function migrate_ip_from_nic { + src_nic=$1 + bridge_name=$2 + + # Enabling explicit error handling: We must avoid to lose the IP + # address in the migration process. Hence, on every error, we + # attempt to assign the IP back to the original NIC and exit. + set +e + + ip=$(get_ip_address_from_interface ${src_nic}) + prefix=$(get_ip_prefix_from_interface ${src_nic}) + + bridge_ip=$(get_ip_address_from_interface "${bridge_name}") + bridge_prefix=$(get_ip_prefix_from_interface "${bridge_name}") + + ip link set ${bridge_name} up + + if [[ -n "${ip}" && -n "${prefix}" ]]; then + ip addr flush dev ${src_nic} + if [ $? -ne 0 ] ; then + ip addr add ${ip}/${prefix} dev ${src_nic} + echo "Error while flushing IP from ${src_nic}." + exit 1 + fi + + ip addr add ${ip}/${prefix} dev "${bridge_name}" + if [ $? -ne 0 ] ; then + echo "Error assigning IP to bridge "${bridge_name}"." + ip addr add ${ip}/${prefix} dev ${src_nic} + exit 1 + fi + elif [[ -n "${bridge_ip}" && -n "${bridge_prefix}" ]]; then + echo "Bridge '${bridge_name}' already has IP assigned. Keeping the same:: IP:[${bridge_ip}]; Prefix:[${bridge_prefix}]..." + elif [[ -z "${bridge_ip}" && -z "${ip}" ]]; then + echo "Interface and bridge have no ips configured. Leaving as is." + else + echo "Interface ${src_nic} has invalid IP address. IP:[${ip}]; Prefix:[${prefix}]..." + exit 1 + fi + + set -e +} + # Detect tunnel interface tunnel_interface="{{- .Values.network.interface.tunnel -}}" if [ -z "${tunnel_interface}" ] ; then @@ -65,7 +117,13 @@ ovs-vsctl set open . external-ids:rundir="/var/run/openvswitch" ovs-vsctl set open . external-ids:ovn-encap-type="{{ .Values.conf.ovn_encap_type }}" ovs-vsctl set open . external-ids:ovn-bridge="{{ .Values.conf.ovn_bridge }}" ovs-vsctl set open . external-ids:ovn-bridge-mappings="{{ .Values.conf.ovn_bridge_mappings }}" -ovs-vsctl set open . external-ids:ovn-cms-options="{{ .Values.conf.ovn_cms_options }}" + +GW_ENABLED=$(cat /tmp/gw-enabled/gw-enabled) +if [[ ${GW_ENABLED} == enabled ]]; then + ovs-vsctl set open . external-ids:ovn-cms-options={{ .Values.conf.onv_cms_options_gw_enabled }} +else + ovs-vsctl set open . external-ids:ovn-cms-options={{ .Values.conf.ovn_cms_options }} +fi # Configure hostname {{- if .Values.pod.use_fqdn.compute }} @@ -82,8 +140,9 @@ do bridge=${bmap%:*} iface=${bmap#*:} ovs-vsctl --may-exist add-br $bridge -- set bridge $bridge protocols=OpenFlow13 - if [ -n "$iface" ] && [ "$iface" != "null" ] + if [ -n "$iface" ] && [ "$iface" != "null" ] && ( ip link show $iface 1>/dev/null 2>&1 ); then ovs-vsctl --may-exist add-port $bridge $iface + migrate_ip_from_nic $iface $bridge fi done diff --git a/ovn/templates/daemonset-controller.yaml b/ovn/templates/daemonset-controller.yaml index ff77d0767..f27903fca 100644 --- a/ovn/templates/daemonset-controller.yaml +++ b/ovn/templates/daemonset-controller.yaml @@ -17,7 +17,33 @@ limitations under the License. {{- $configMapName := "ovn-etc" }} {{- $serviceAccountName := "ovn-controller" }} +{{- $serviceAccountNamespace := $envAll.Release.Namespace }} {{ tuple $envAll "ovn_controller" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: ovn-controller-list-nodes-role-{{ $serviceAccountNamespace }} +rules: +- apiGroups: [""] + resources: ["nodes"] + verbs: ["list", "get"] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: ovn-controller-list-nodes-rolebinding-{{ $serviceAccountNamespace }} +subjects: +- kind: ServiceAccount + name: {{ $serviceAccountName }} + namespace: {{ $serviceAccountNamespace }} +roleRef: + kind: ClusterRole + name: ovn-controller-list-nodes-role-{{ $serviceAccountNamespace }} + apiGroup: rbac.authorization.k8s.io + --- kind: DaemonSet apiVersion: apps/v1 @@ -50,6 +76,22 @@ spec: {{ .Values.labels.ovn_controller.node_selector_key }}: {{ .Values.labels.ovn_controller.node_selector_value }} initContainers: {{- tuple $envAll "ovn_controller" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + - name: get-gw-enabled +{{ tuple $envAll "ovn_controller_kubectl" | include "helm-toolkit.snippets.image" | indent 10 }} + command: + - /bin/bash + - -c + - | + kubectl get node ${NODENAME} -o jsonpath='{.metadata.labels.l3-agent}' > /tmp/gw-enabled/gw-enabled + env: + - name: NODENAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - name: gw-enabled + mountPath: /tmp/gw-enabled + readOnly: false - name: controller-init {{ dict "envAll" $envAll "application" "ovn_controller" "container" "controller_init" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} {{ tuple $envAll "ovn_controller" | include "helm-toolkit.snippets.image" | indent 10 }} @@ -66,6 +108,9 @@ spec: mountPath: /tmp/auto_bridge_add subPath: auto_bridge_add readOnly: true + - name: gw-enabled + mountPath: /tmp/gw-enabled + readOnly: true containers: - name: controller {{ tuple $envAll "ovn_controller" | include "helm-toolkit.snippets.image" | indent 10 }} @@ -112,4 +157,6 @@ spec: hostPath: path: /run/ovn type: DirectoryOrCreate + - name: gw-enabled + emptyDir: {} {{- end }} diff --git a/ovn/values.yaml b/ovn/values.yaml index 69adb6bf1..97a9c4a43 100644 --- a/ovn/values.yaml +++ b/ovn/values.yaml @@ -24,6 +24,7 @@ images: ovn_ovsdb_sb: docker.io/openstackhelm/ovn:ubuntu_focal ovn_northd: docker.io/openstackhelm/ovn:ubuntu_focal ovn_controller: docker.io/openstackhelm/ovn:ubuntu_focal + ovn_controller_kubectl: docker.io/openstackhelm/ceph-config-helper:latest-ubuntu_jammy dep_check: quay.io/airshipit/kubernetes-entrypoint:v1.0.0 image_repo_sync: docker.io/library/docker:17.07.0 pull_policy: "IfNotPresent" @@ -66,7 +67,8 @@ network: tunnel_network_cidr: "0/0" conf: - ovn_cms_options: "enable-chassis-as-gw,availability-zones=nova" + ovn_cms_options: "availability-zones=nova" + onv_cms_options_gw_enabled: "enable-chassis-as-gw,availability-zones=nova" ovn_encap_type: geneve ovn_bridge: br-int ovn_bridge_mappings: external:br-ex diff --git a/releasenotes/notes/ovn.yaml b/releasenotes/notes/ovn.yaml index e7b33f713..efa86a0c8 100644 --- a/releasenotes/notes/ovn.yaml +++ b/releasenotes/notes/ovn.yaml @@ -8,4 +8,5 @@ ovn: - 0.1.5 Add ubuntu_focal and ubuntu_jammy overrides - 0.1.6 Fix ovsdb port number - 0.1.7 Use host network for ovn controller pods + - 0.1.8 Fix attaching interfaces to the bridge ... diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml index 3446c0452..8f6b62af2 100644 --- a/zuul.d/jobs.yaml +++ b/zuul.d/jobs.yaml @@ -303,4 +303,9 @@ container_distro_version: jammy feature_gates: dpdk +- job: + name: openstack-helm-infra-compute-kit-ovn-2023-2-ubuntu_jammy + parent: openstack-helm-compute-kit-ovn-2023-2-ubuntu_jammy + files: + - ^ovn/.* ... diff --git a/zuul.d/project.yaml b/zuul.d/project.yaml index a547d937c..9c548d90f 100644 --- a/zuul.d/project.yaml +++ b/zuul.d/project.yaml @@ -30,6 +30,7 @@ - openstack-helm-infra-metacontroller - openstack-helm-infra-mariadb-operator - openstack-helm-compute-kit-dpdk-ubuntu_jammy + - openstack-helm-infra-compute-kit-ovn-2023-2-ubuntu_jammy gate: jobs: - openstack-helm-lint