From 9861dc2361f6bcdec239453bb3aabcebf41620f3 Mon Sep 17 00:00:00 2001 From: Boovan Rajendran Date: Thu, 9 Feb 2023 09:32:34 -0500 Subject: [PATCH] kubernetes: Revert kubeadm control-plane changes for 1.25 Upstream has deprecated 'node-role.kubernetes.io/master' to use 'node-role.kubernetes.io/control-plane' in k8s 1.25. Revert the commit 80e5bcae9b885179446fa27fa3b4da0992c1b8a8 to remove the old control-plane label. Partially revert the commit ddd046f3dd88186cbc83b57e83144db96eae4af4 to add new control-plane taint. Test-plan: Pass: Verified by performing the make test WHAT commands. Note: Runtime testing will have to wait until we enable K8s upgrades from 1.24 to 1.25 Story: 2010368 Task: 47376 Signed-off-by: Boovan Rajendran Change-Id: Ie85ca92df9679e822457d9b258ee454d847b9e7a --- ...leanup-the-master-taint-on-CP-nodes-.patch | 96 +++++++++++++++++++ ...dm-remove-RemoveOldControlPlaneLabel.patch | 50 ++++++++++ .../debian/deb_folder/patches/series | 3 + 3 files changed, 149 insertions(+) create mode 100644 kubernetes/kubernetes-1.25.3/debian/deb_folder/patches/Revert-kubeadm-cleanup-the-master-taint-on-CP-nodes-.patch create mode 100644 kubernetes/kubernetes-1.25.3/debian/deb_folder/patches/Revert-kubeadm-remove-RemoveOldControlPlaneLabel.patch diff --git a/kubernetes/kubernetes-1.25.3/debian/deb_folder/patches/Revert-kubeadm-cleanup-the-master-taint-on-CP-nodes-.patch b/kubernetes/kubernetes-1.25.3/debian/deb_folder/patches/Revert-kubeadm-cleanup-the-master-taint-on-CP-nodes-.patch new file mode 100644 index 000000000..60e7a1e1a --- /dev/null +++ b/kubernetes/kubernetes-1.25.3/debian/deb_folder/patches/Revert-kubeadm-cleanup-the-master-taint-on-CP-nodes-.patch @@ -0,0 +1,96 @@ +From 094f22ddb7c82523c716b6866c7f9135da6119a6 Mon Sep 17 00:00:00 2001 +From: Boovan Rajendran +Date: Wed, 15 Feb 2023 03:28:16 -0500 +Subject: [PATCH] Revert "kubeadm: cleanup the "master" taint on CP nodes + during upgrade" + +This partially reverts commit ddd046f3dd88186cbc83b57e83144db96eae4af4. + +Signed-off-by: Boovan Rajendran +--- + cmd/kubeadm/app/cmd/upgrade/apply.go | 17 ++++++++ + cmd/kubeadm/app/phases/upgrade/postupgrade.go | 40 +++++++++++++++++++ + 2 files changed, 57 insertions(+) + +diff --git a/cmd/kubeadm/app/cmd/upgrade/apply.go b/cmd/kubeadm/app/cmd/upgrade/apply.go +index 4687c5bcdd2..03c1bf19f14 100644 +--- a/cmd/kubeadm/app/cmd/upgrade/apply.go ++++ b/cmd/kubeadm/app/cmd/upgrade/apply.go +@@ -158,6 +158,23 @@ func runApply(flags *applyFlags, args []string) error { + return errors.Wrap(err, "[upgrade/apply] FATAL") + } + ++ // TODO: https://github.com/kubernetes/kubeadm/issues/2200 ++ fmt.Printf("[upgrade/postupgrade] Removing the deprecated label %s='' from all control plane Nodes. "+ ++ "After this step only the label %s='' will be present on control plane Nodes.\n", ++ kubeadmconstants.LabelNodeRoleOldControlPlane, kubeadmconstants.LabelNodeRoleControlPlane) ++ if err := upgrade.RemoveOldControlPlaneLabel(client); err != nil { ++ return err ++ } ++ ++ // TODO: https://github.com/kubernetes/kubeadm/issues/2200 ++ fmt.Printf("[upgrade/postupgrade] Adding the new taint %s to all control plane Nodes. "+ ++ "After this step both taints %s and %s should be present on control plane Nodes.\n", ++ kubeadmconstants.ControlPlaneTaint.String(), kubeadmconstants.ControlPlaneTaint.String(), ++ kubeadmconstants.OldControlPlaneTaint.String()) ++ if err := upgrade.AddNewControlPlaneTaint(client); err != nil { ++ return err ++ } ++ + // Clean this up in 1.26 + // TODO: https://github.com/kubernetes/kubeadm/issues/2200 + fmt.Printf("[upgrade/postupgrade] Removing the old taint %s from all control plane Nodes. "+ +diff --git a/cmd/kubeadm/app/phases/upgrade/postupgrade.go b/cmd/kubeadm/app/phases/upgrade/postupgrade.go +index eb67ccffaf8..95752b609b7 100644 +--- a/cmd/kubeadm/app/phases/upgrade/postupgrade.go ++++ b/cmd/kubeadm/app/phases/upgrade/postupgrade.go +@@ -232,6 +232,46 @@ func RemoveOldControlPlaneLabel(client clientset.Interface) error { + return nil + } + ++// AddNewControlPlaneTaint finds all nodes with the new "control-plane" node-role label ++// and adds the new "control-plane" taint to them. ++// TODO: https://github.com/kubernetes/kubeadm/issues/2200 ++func AddNewControlPlaneTaint(client clientset.Interface) error { ++ selectorControlPlane := labels.SelectorFromSet(labels.Set(map[string]string{ ++ kubeadmconstants.LabelNodeRoleControlPlane: "", ++ })) ++ nodes, err := client.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{ ++ LabelSelector: selectorControlPlane.String(), ++ }) ++ if err != nil { ++ return errors.Wrapf(err, "could not list nodes labeled with %q", kubeadmconstants.LabelNodeRoleControlPlane) ++ } ++ ++ for _, n := range nodes.Items { ++ // Check if the node has the old / new taints ++ hasOldTaint := false ++ hasNewTaint := false ++ for _, t := range n.Spec.Taints { ++ switch t.String() { ++ case kubeadmconstants.OldControlPlaneTaint.String(): ++ hasOldTaint = true ++ case kubeadmconstants.ControlPlaneTaint.String(): ++ hasNewTaint = true ++ } ++ } ++ // If the old taint is present and the new taint is missing, patch the node with the new taint. ++ // When the old taint is missing, assume the user has manually untainted the node and take no action. ++ if !hasNewTaint && hasOldTaint { ++ err = apiclient.PatchNode(client, n.Name, func(n *v1.Node) { ++ n.Spec.Taints = append(n.Spec.Taints, kubeadmconstants.ControlPlaneTaint) ++ }) ++ if err != nil { ++ return err ++ } ++ } ++ } ++ return nil ++} ++ + // RemoveOldControlPlaneTaint finds all nodes with the new "control-plane" node-role label + // and removes the old "control-plane" taint to them. + // TODO: https://github.com/kubernetes/kubeadm/issues/2200 +-- +2.25.1 + diff --git a/kubernetes/kubernetes-1.25.3/debian/deb_folder/patches/Revert-kubeadm-remove-RemoveOldControlPlaneLabel.patch b/kubernetes/kubernetes-1.25.3/debian/deb_folder/patches/Revert-kubeadm-remove-RemoveOldControlPlaneLabel.patch new file mode 100644 index 000000000..103a1c309 --- /dev/null +++ b/kubernetes/kubernetes-1.25.3/debian/deb_folder/patches/Revert-kubeadm-remove-RemoveOldControlPlaneLabel.patch @@ -0,0 +1,50 @@ +From dbf1b405fd31c548992fb73bafcb44c8ffe208ee Mon Sep 17 00:00:00 2001 +From: Boovan Rajendran +Date: Wed, 15 Feb 2023 02:47:26 -0500 +Subject: [PATCH] Revert "kubeadm: remove RemoveOldControlPlaneLabel" + +This reverts commit 80e5bcae9b885179446fa27fa3b4da0992c1b8a8. + +Signed-off-by: Boovan Rajendran +--- + cmd/kubeadm/app/phases/upgrade/postupgrade.go | 24 +++++++++++++++++++ + 1 file changed, 24 insertions(+) + +diff --git a/cmd/kubeadm/app/phases/upgrade/postupgrade.go b/cmd/kubeadm/app/phases/upgrade/postupgrade.go +index d6a5394ccde..eb67ccffaf8 100644 +--- a/cmd/kubeadm/app/phases/upgrade/postupgrade.go ++++ b/cmd/kubeadm/app/phases/upgrade/postupgrade.go +@@ -208,6 +208,30 @@ func rollbackFiles(files map[string]string, originalErr error) error { + return errors.Errorf("couldn't move these files: %v. Got errors: %v", files, errorsutil.NewAggregate(errs)) + } + ++// RemoveOldControlPlaneLabel finds all nodes with the legacy node-role label and removes it ++// TODO: https://github.com/kubernetes/kubeadm/issues/2200 ++func RemoveOldControlPlaneLabel(client clientset.Interface) error { ++ selectorOldControlPlane := labels.SelectorFromSet(labels.Set(map[string]string{ ++ kubeadmconstants.LabelNodeRoleOldControlPlane: "", ++ })) ++ nodesWithOldLabel, err := client.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{ ++ LabelSelector: selectorOldControlPlane.String(), ++ }) ++ if err != nil { ++ return errors.Wrapf(err, "could not list nodes labeled with %q", kubeadmconstants.LabelNodeRoleOldControlPlane) ++ } ++ ++ for _, n := range nodesWithOldLabel.Items { ++ err = apiclient.PatchNode(client, n.Name, func(n *v1.Node) { ++ delete(n.ObjectMeta.Labels, kubeadmconstants.LabelNodeRoleOldControlPlane) ++ }) ++ if err != nil { ++ return err ++ } ++ } ++ return nil ++} ++ + // RemoveOldControlPlaneTaint finds all nodes with the new "control-plane" node-role label + // and removes the old "control-plane" taint to them. + // TODO: https://github.com/kubernetes/kubeadm/issues/2200 +-- +2.25.1 + diff --git a/kubernetes/kubernetes-1.25.3/debian/deb_folder/patches/series b/kubernetes/kubernetes-1.25.3/debian/deb_folder/patches/series index 1aa14b3be..601343ac8 100644 --- a/kubernetes/kubernetes-1.25.3/debian/deb_folder/patches/series +++ b/kubernetes/kubernetes-1.25.3/debian/deb_folder/patches/series @@ -8,3 +8,6 @@ kubelet-cpumanager-infra-pods-use-system-reserved-CP.patch kubelet-cpumanager-introduce-concept-of-isolated-CPU.patch enable-support-for-kubernetes-to-ignore-isolcpus.patch kubelet-CFS-quota-throttling-for-non-integer-cpulimit.patch +Revert-kubeadm-remove-RemoveOldControlPlaneLabel.patch +Revert-kubeadm-cleanup-the-master-taint-on-CP-nodes-.patch +