From 4b77ab2b96063508d3618576082ac144c498bbf9 Mon Sep 17 00:00:00 2001 From: Jim Gauld Date: Thu, 13 Oct 2022 15:58:50 -0400 Subject: [PATCH] kubernetes: Revert kubeadm control-plane label changes for 1.24 Upstream has deprecated 'node-role.kubernetes.io/master' to use 'node-role.kubernetes.io/control-plane' in k8s 1.24. To preserve backwards compatibility we need to revert back to using the 'node-role.kubernetes.io/master' taint. Platform and applications need to be updated to use 'control-plane' with nodeSelector/Tolerations so we may upgrade from 'master'. Test-plan: PASS: kubernetes-1.24.4 package builds PASS: AIO-SX Fresh install of ISO with k8s 1.24.4 Story: 2010301 Task: 46564 Signed-off-by: Jim Gauld Change-Id: I660bf2bc0bbf50cdff85b9c72477e53b176c9ed9 --- ...work-test_context-add-control-plane-.patch | 32 ++++ ...pply-the-new-control-plane-taint-dur.patch | 86 +++++++++++ ...pply-the-new-control-plane-taint-on-.patch | 144 ++++++++++++++++++ ...elete-the-old-master-label-during-up.patch | 119 +++++++++++++++ ...nly-apply-the-new-control-plane-labe.patch | 123 +++++++++++++++ .../debian/deb_folder/patches/series | 5 + 6 files changed, 509 insertions(+) create mode 100644 kubernetes/kubernetes-1.24.4/debian/deb_folder/patches/Revert-e2e-framework-test_context-add-control-plane-.patch create mode 100644 kubernetes/kubernetes-1.24.4/debian/deb_folder/patches/Revert-kubeadm-apply-the-new-control-plane-taint-dur.patch create mode 100644 kubernetes/kubernetes-1.24.4/debian/deb_folder/patches/Revert-kubeadm-apply-the-new-control-plane-taint-on-.patch create mode 100644 kubernetes/kubernetes-1.24.4/debian/deb_folder/patches/Revert-kubeadm-delete-the-old-master-label-during-up.patch create mode 100644 kubernetes/kubernetes-1.24.4/debian/deb_folder/patches/Revert-kubeadm-only-apply-the-new-control-plane-labe.patch diff --git a/kubernetes/kubernetes-1.24.4/debian/deb_folder/patches/Revert-e2e-framework-test_context-add-control-plane-.patch b/kubernetes/kubernetes-1.24.4/debian/deb_folder/patches/Revert-e2e-framework-test_context-add-control-plane-.patch new file mode 100644 index 000000000..a6d07848c --- /dev/null +++ b/kubernetes/kubernetes-1.24.4/debian/deb_folder/patches/Revert-e2e-framework-test_context-add-control-plane-.patch @@ -0,0 +1,32 @@ +From a218316fb35a2ec67b7c8ad0fc2e8df537ee3434 Mon Sep 17 00:00:00 2001 +From: Jim Gauld +Date: Wed, 12 Oct 2022 13:57:45 -0400 +Subject: [PATCH 1/5] Revert "e2e/framework/test_context: add "control-plane" + to non-blocking-taints" + +This reverts commit 8641897057431d6c89a716d86c997f29049df0f7. +--- + test/e2e/framework/test_context.go | 7 +------ + 1 file changed, 1 insertion(+), 6 deletions(-) + +diff --git a/test/e2e/framework/test_context.go b/test/e2e/framework/test_context.go +index b4c4743b6ea..8bc5827d89a 100644 +--- a/test/e2e/framework/test_context.go ++++ b/test/e2e/framework/test_context.go +@@ -311,12 +311,7 @@ func RegisterCommonFlags(flags *flag.FlagSet) { + flags.StringVar(&TestContext.SystemdServices, "systemd-services", "docker", "The comma separated list of systemd services the framework will dump logs for.") + flags.BoolVar(&TestContext.DumpSystemdJournal, "dump-systemd-journal", false, "Whether to dump the full systemd journal.") + flags.StringVar(&TestContext.ImageServiceEndpoint, "image-service-endpoint", "", "The image service endpoint of cluster VM instances.") +- // TODO: remove the node-role.kubernetes.io/master taint in 1.25 or later. +- // The change will likely require an action for some users that do not +- // use k8s originated tools like kubeadm or kOps for creating clusters +- // and taint their control plane nodes with "master", expecting the test +- // suite to work with this legacy non-blocking taint. +- flags.StringVar(&TestContext.NonblockingTaints, "non-blocking-taints", `node-role.kubernetes.io/control-plane,node-role.kubernetes.io/master`, "Nodes with taints in this comma-delimited list will not block the test framework from starting tests. The default taint 'node-role.kubernetes.io/master' is DEPRECATED and will be removed from the list in a future release.") ++ flags.StringVar(&TestContext.NonblockingTaints, "non-blocking-taints", `node-role.kubernetes.io/master`, "Nodes with taints in this comma-delimited list will not block the test framework from starting tests.") + + flags.BoolVar(&TestContext.ListImages, "list-images", false, "If true, will show list of images used for runnning tests.") + flags.BoolVar(&TestContext.ListConformanceTests, "list-conformance-tests", false, "If true, will show list of conformance tests.") +-- +2.25.1 + diff --git a/kubernetes/kubernetes-1.24.4/debian/deb_folder/patches/Revert-kubeadm-apply-the-new-control-plane-taint-dur.patch b/kubernetes/kubernetes-1.24.4/debian/deb_folder/patches/Revert-kubeadm-apply-the-new-control-plane-taint-dur.patch new file mode 100644 index 000000000..5de979383 --- /dev/null +++ b/kubernetes/kubernetes-1.24.4/debian/deb_folder/patches/Revert-kubeadm-apply-the-new-control-plane-taint-dur.patch @@ -0,0 +1,86 @@ +From 2c9b24f312305a29de2c861a35c3ec18b4ad3994 Mon Sep 17 00:00:00 2001 +From: Jim Gauld +Date: Wed, 12 Oct 2022 13:59:48 -0400 +Subject: [PATCH 2/5] Revert "kubeadm: apply the new "control-plane" taint + during upgrade" + +This reverts commit db6061f5a6e6eb0002d572fa5ab1142e9b60bc1b. +--- + cmd/kubeadm/app/cmd/upgrade/apply.go | 9 ----- + cmd/kubeadm/app/phases/upgrade/postupgrade.go | 40 ------------------- + 2 files changed, 49 deletions(-) + +diff --git a/cmd/kubeadm/app/cmd/upgrade/apply.go b/cmd/kubeadm/app/cmd/upgrade/apply.go +index 042f10796b6..3644146d804 100644 +--- a/cmd/kubeadm/app/cmd/upgrade/apply.go ++++ b/cmd/kubeadm/app/cmd/upgrade/apply.go +@@ -163,15 +163,6 @@ func runApply(flags *applyFlags, args []string) error { + return err + } + +- // TODO: https://github.com/kubernetes/kubeadm/issues/2200 +- fmt.Printf("[upgrade/postupgrade] Adding the new taint %s to all control plane Nodes. "+ +- "After this step both taints %s and %s should be present on control plane Nodes.\n", +- kubeadmconstants.ControlPlaneTaint.String(), kubeadmconstants.ControlPlaneTaint.String(), +- kubeadmconstants.OldControlPlaneTaint.String()) +- if err := upgrade.AddNewControlPlaneTaint(client); err != nil { +- return err +- } +- + // Upgrade RBAC rules and addons. + klog.V(1).Infoln("[upgrade/postupgrade] upgrading RBAC rules and addons") + if err := upgrade.PerformPostUpgradeTasks(client, cfg, flags.dryRun); err != nil { +diff --git a/cmd/kubeadm/app/phases/upgrade/postupgrade.go b/cmd/kubeadm/app/phases/upgrade/postupgrade.go +index 55828597310..fa215bfedf8 100644 +--- a/cmd/kubeadm/app/phases/upgrade/postupgrade.go ++++ b/cmd/kubeadm/app/phases/upgrade/postupgrade.go +@@ -238,46 +238,6 @@ func RemoveOldControlPlaneLabel(client clientset.Interface) error { + return nil + } + +-// AddNewControlPlaneTaint finds all nodes with the new "control-plane" node-role label +-// and adds the new "control-plane" taint to them. +-// TODO: https://github.com/kubernetes/kubeadm/issues/2200 +-func AddNewControlPlaneTaint(client clientset.Interface) error { +- selectorControlPlane := labels.SelectorFromSet(labels.Set(map[string]string{ +- kubeadmconstants.LabelNodeRoleControlPlane: "", +- })) +- nodes, err := client.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{ +- LabelSelector: selectorControlPlane.String(), +- }) +- if err != nil { +- return errors.Wrapf(err, "could not list nodes labeled with %q", kubeadmconstants.LabelNodeRoleControlPlane) +- } +- +- for _, n := range nodes.Items { +- // Check if the node has the old / new taints +- hasOldTaint := false +- hasNewTaint := false +- for _, t := range n.Spec.Taints { +- switch t.String() { +- case kubeadmconstants.OldControlPlaneTaint.String(): +- hasOldTaint = true +- case kubeadmconstants.ControlPlaneTaint.String(): +- hasNewTaint = true +- } +- } +- // If the old taint is present and the new taint is missing, patch the node with the new taint. +- // When the old taint is missing, assume the user has manually untainted the node and take no action. +- if !hasNewTaint && hasOldTaint { +- err = apiclient.PatchNode(client, n.Name, func(n *v1.Node) { +- n.Spec.Taints = append(n.Spec.Taints, kubeadmconstants.ControlPlaneTaint) +- }) +- if err != nil { +- return err +- } +- } +- } +- return nil +-} +- + // UpdateKubeletDynamicEnvFileWithURLScheme reads the kubelet dynamic environment file + // from disk, ensure that the CRI endpoint flag has a scheme prefix and writes it + // back to disk. +-- +2.25.1 + diff --git a/kubernetes/kubernetes-1.24.4/debian/deb_folder/patches/Revert-kubeadm-apply-the-new-control-plane-taint-on-.patch b/kubernetes/kubernetes-1.24.4/debian/deb_folder/patches/Revert-kubeadm-apply-the-new-control-plane-taint-on-.patch new file mode 100644 index 000000000..86ea74c54 --- /dev/null +++ b/kubernetes/kubernetes-1.24.4/debian/deb_folder/patches/Revert-kubeadm-apply-the-new-control-plane-taint-on-.patch @@ -0,0 +1,144 @@ +From 7d898d974f2353b5984deb4ad42347726205696a Mon Sep 17 00:00:00 2001 +From: Jim Gauld +Date: Wed, 12 Oct 2022 14:00:19 -0400 +Subject: [PATCH 3/5] Revert "kubeadm: apply the new "control-plane" taint on + CP nodes" + +This reverts commit 370031cadac6240e49e7b30a644d19735b7d3338. +--- + cmd/kubeadm/app/apis/kubeadm/types.go | 6 +++--- + cmd/kubeadm/app/apis/kubeadm/v1beta2/doc.go | 2 +- + cmd/kubeadm/app/apis/kubeadm/v1beta2/types.go | 6 +++--- + cmd/kubeadm/app/apis/kubeadm/v1beta3/doc.go | 2 +- + cmd/kubeadm/app/apis/kubeadm/v1beta3/types.go | 6 +++--- + cmd/kubeadm/app/util/config/initconfiguration.go | 2 +- + cmd/kubeadm/app/util/config/initconfiguration_test.go | 8 ++++---- + 7 files changed, 16 insertions(+), 16 deletions(-) + +diff --git a/cmd/kubeadm/app/apis/kubeadm/types.go b/cmd/kubeadm/app/apis/kubeadm/types.go +index d49256908e7..55d1fd9a06f 100644 +--- a/cmd/kubeadm/app/apis/kubeadm/types.go ++++ b/cmd/kubeadm/app/apis/kubeadm/types.go +@@ -218,9 +218,9 @@ type NodeRegistrationOptions struct { + // CRISocket is used to retrieve container runtime info. This information will be annotated to the Node API object, for later re-use + CRISocket string + +- // Taints specifies the taints the Node API object should be registered with. If this field is unset, i.e. nil, +- // it will be defaulted with a control-plane taint for control-plane nodes. If you don't want to taint your control-plane +- // node, set this field to an empty slice, i.e. `taints: []` in the YAML file. This field is solely used for Node registration. ++ // Taints specifies the taints the Node API object should be registered with. If this field is unset, i.e. nil, in the `kubeadm init` process ++ // it will be defaulted to []v1.Taint{'node-role.kubernetes.io/master=""'}. If you don't want to taint your control-plane node, set this field to an ++ // empty slice, i.e. `taints: []` in the YAML file. This field is solely used for Node registration. + Taints []v1.Taint + + // KubeletExtraArgs passes through extra arguments to the kubelet. The arguments here are passed to the kubelet command line via the environment file +diff --git a/cmd/kubeadm/app/apis/kubeadm/v1beta2/doc.go b/cmd/kubeadm/app/apis/kubeadm/v1beta2/doc.go +index 64ae2bb8f7c..24e58868dbe 100644 +--- a/cmd/kubeadm/app/apis/kubeadm/v1beta2/doc.go ++++ b/cmd/kubeadm/app/apis/kubeadm/v1beta2/doc.go +@@ -172,7 +172,7 @@ limitations under the License. + // criSocket: "unix:///var/run/containerd/containerd.sock" + // taints: + // - key: "kubeadmNode" +-// value: "someValue" ++// value: "master" + // effect: "NoSchedule" + // kubeletExtraArgs: + // v: 4 +diff --git a/cmd/kubeadm/app/apis/kubeadm/v1beta2/types.go b/cmd/kubeadm/app/apis/kubeadm/v1beta2/types.go +index a5cf40c513a..30037e30d40 100644 +--- a/cmd/kubeadm/app/apis/kubeadm/v1beta2/types.go ++++ b/cmd/kubeadm/app/apis/kubeadm/v1beta2/types.go +@@ -201,9 +201,9 @@ type NodeRegistrationOptions struct { + // CRISocket is used to retrieve container runtime info. This information will be annotated to the Node API object, for later re-use + CRISocket string `json:"criSocket,omitempty"` + +- // Taints specifies the taints the Node API object should be registered with. If this field is unset, i.e. nil, +- // it will be defaulted with a control-plane taint for control-plane nodes. If you don't want to taint your control-plane +- // node, set this field to an empty slice, i.e. `taints: []` in the YAML file. This field is solely used for Node registration. ++ // Taints specifies the taints the Node API object should be registered with. If this field is unset, i.e. nil, in the `kubeadm init` process ++ // it will be defaulted to []v1.Taint{'node-role.kubernetes.io/master=""'}. If you don't want to taint your control-plane node, set this field to an ++ // empty slice, i.e. `taints: []` in the YAML file. This field is solely used for Node registration. + Taints []v1.Taint `json:"taints"` + + // KubeletExtraArgs passes through extra arguments to the kubelet. The arguments here are passed to the kubelet command line via the environment file +diff --git a/cmd/kubeadm/app/apis/kubeadm/v1beta3/doc.go b/cmd/kubeadm/app/apis/kubeadm/v1beta3/doc.go +index 8353359f91a..f7f34a7cffa 100644 +--- a/cmd/kubeadm/app/apis/kubeadm/v1beta3/doc.go ++++ b/cmd/kubeadm/app/apis/kubeadm/v1beta3/doc.go +@@ -176,7 +176,7 @@ limitations under the License. + // criSocket: "unix:///var/run/containerd/containerd.sock" + // taints: + // - key: "kubeadmNode" +-// value: "someValue" ++// value: "master" + // effect: "NoSchedule" + // kubeletExtraArgs: + // v: 4 +diff --git a/cmd/kubeadm/app/apis/kubeadm/v1beta3/types.go b/cmd/kubeadm/app/apis/kubeadm/v1beta3/types.go +index 5a5151bf64b..82ae10cc271 100644 +--- a/cmd/kubeadm/app/apis/kubeadm/v1beta3/types.go ++++ b/cmd/kubeadm/app/apis/kubeadm/v1beta3/types.go +@@ -215,9 +215,9 @@ type NodeRegistrationOptions struct { + // +optional + CRISocket string `json:"criSocket,omitempty"` + +- // Taints specifies the taints the Node API object should be registered with. If this field is unset, i.e. nil, +- // it will be defaulted with a control-plane taint for control-plane nodes. If you don't want to taint your control-plane +- // node, set this field to an empty slice, i.e. `taints: []` in the YAML file. This field is solely used for Node registration. ++ // Taints specifies the taints the Node API object should be registered with. If this field is unset, i.e. nil, in the `kubeadm init` process ++ // it will be defaulted to []v1.Taint{'node-role.kubernetes.io/master=""'}. If you don't want to taint your control-plane node, set this field to an ++ // empty slice, i.e. `taints: []` in the YAML file. This field is solely used for Node registration. + Taints []corev1.Taint `json:"taints"` + + // KubeletExtraArgs passes through extra arguments to the kubelet. The arguments here are passed to the kubelet command line via the environment file +diff --git a/cmd/kubeadm/app/util/config/initconfiguration.go b/cmd/kubeadm/app/util/config/initconfiguration.go +index 7d1bb67fc02..71c3c514bf0 100644 +--- a/cmd/kubeadm/app/util/config/initconfiguration.go ++++ b/cmd/kubeadm/app/util/config/initconfiguration.go +@@ -106,7 +106,7 @@ func SetNodeRegistrationDynamicDefaults(cfg *kubeadmapi.NodeRegistrationOptions, + // Only if the slice is nil, we should append the control-plane taint. This allows the user to specify an empty slice for no default control-plane taint + if controlPlaneTaint && cfg.Taints == nil { + // TODO: https://github.com/kubernetes/kubeadm/issues/2200 +- cfg.Taints = []v1.Taint{kubeadmconstants.OldControlPlaneTaint, kubeadmconstants.ControlPlaneTaint} ++ cfg.Taints = []v1.Taint{kubeadmconstants.OldControlPlaneTaint} + } + + if cfg.CRISocket == "" { +diff --git a/cmd/kubeadm/app/util/config/initconfiguration_test.go b/cmd/kubeadm/app/util/config/initconfiguration_test.go +index 074a1d821f4..93d7817a232 100644 +--- a/cmd/kubeadm/app/util/config/initconfiguration_test.go ++++ b/cmd/kubeadm/app/util/config/initconfiguration_test.go +@@ -115,17 +115,17 @@ func TestDefaultTaintsMarshaling(t *testing.T) { + expectedTaintCnt int + }{ + { +- desc: "Uninitialized nodeRegistration field produces expected taints", ++ desc: "Uninitialized nodeRegistration field produces a single taint (the master one)", + cfg: kubeadmapiv1.InitConfiguration{ + TypeMeta: metav1.TypeMeta{ + APIVersion: kubeadmapiv1.SchemeGroupVersion.String(), + Kind: constants.InitConfigurationKind, + }, + }, +- expectedTaintCnt: 2, ++ expectedTaintCnt: 1, + }, + { +- desc: "Uninitialized taints field produces expected taints", ++ desc: "Uninitialized taints field produces a single taint (the master one)", + cfg: kubeadmapiv1.InitConfiguration{ + TypeMeta: metav1.TypeMeta{ + APIVersion: kubeadmapiv1.SchemeGroupVersion.String(), +@@ -133,7 +133,7 @@ func TestDefaultTaintsMarshaling(t *testing.T) { + }, + NodeRegistration: kubeadmapiv1.NodeRegistrationOptions{}, + }, +- expectedTaintCnt: 2, ++ expectedTaintCnt: 1, + }, + { + desc: "Forsing taints to an empty slice produces no taints", +-- +2.25.1 + diff --git a/kubernetes/kubernetes-1.24.4/debian/deb_folder/patches/Revert-kubeadm-delete-the-old-master-label-during-up.patch b/kubernetes/kubernetes-1.24.4/debian/deb_folder/patches/Revert-kubeadm-delete-the-old-master-label-during-up.patch new file mode 100644 index 000000000..b3d33aad3 --- /dev/null +++ b/kubernetes/kubernetes-1.24.4/debian/deb_folder/patches/Revert-kubeadm-delete-the-old-master-label-during-up.patch @@ -0,0 +1,119 @@ +From 42c1abc4763c795b0c9bfb65b1ceba1fd2fa3427 Mon Sep 17 00:00:00 2001 +From: Jim Gauld +Date: Wed, 12 Oct 2022 14:00:47 -0400 +Subject: [PATCH 5/5] Revert "kubeadm: delete the old "master" label during + upgrade" + +This reverts commit c0871b4433783a30c97e204b2011cf17d0457a62. +--- + cmd/kubeadm/app/cmd/join.go | 2 +- + cmd/kubeadm/app/cmd/upgrade/apply.go | 7 +++---- + cmd/kubeadm/app/phases/upgrade/health.go | 21 +++++++++++++++++-- + cmd/kubeadm/app/phases/upgrade/postupgrade.go | 10 ++++++--- + 4 files changed, 30 insertions(+), 10 deletions(-) + +diff --git a/cmd/kubeadm/app/cmd/join.go b/cmd/kubeadm/app/cmd/join.go +index fa0b07e3631..934a0951ff3 100644 +--- a/cmd/kubeadm/app/cmd/join.go ++++ b/cmd/kubeadm/app/cmd/join.go +@@ -64,7 +64,7 @@ var ( + + * Certificate signing request was sent to apiserver and approval was received. + * The Kubelet was informed of the new secure connection details. +- * Control plane label and taint were applied to the new node. ++ * Control plane (master) label and taint were applied to the new node. + * The Kubernetes control plane instances scaled up. + {{.etcdMessage}} + +diff --git a/cmd/kubeadm/app/cmd/upgrade/apply.go b/cmd/kubeadm/app/cmd/upgrade/apply.go +index 3644146d804..d49859aab9d 100644 +--- a/cmd/kubeadm/app/cmd/upgrade/apply.go ++++ b/cmd/kubeadm/app/cmd/upgrade/apply.go +@@ -156,10 +156,9 @@ func runApply(flags *applyFlags, args []string) error { + } + + // TODO: https://github.com/kubernetes/kubeadm/issues/2200 +- fmt.Printf("[upgrade/postupgrade] Removing the deprecated label %s='' from all control plane Nodes. "+ +- "After this step only the label %s='' will be present on control plane Nodes.\n", +- kubeadmconstants.LabelNodeRoleOldControlPlane, kubeadmconstants.LabelNodeRoleControlPlane) +- if err := upgrade.RemoveOldControlPlaneLabel(client); err != nil { ++ fmt.Printf("[upgrade/postupgrade] Applying label %s='' to Nodes with label %s='' (deprecated)\n", ++ kubeadmconstants.LabelNodeRoleControlPlane, kubeadmconstants.LabelNodeRoleOldControlPlane) ++ if err := upgrade.LabelOldControlPlaneNodes(client); err != nil { + return err + } + +diff --git a/cmd/kubeadm/app/phases/upgrade/health.go b/cmd/kubeadm/app/phases/upgrade/health.go +index 55acdf865a6..b14bc6f3bbd 100644 +--- a/cmd/kubeadm/app/phases/upgrade/health.go ++++ b/cmd/kubeadm/app/phases/upgrade/health.go +@@ -212,17 +212,34 @@ func deleteHealthCheckJob(client clientset.Interface, ns, jobName string) error + + // controlPlaneNodesReady checks whether all control-plane Nodes in the cluster are in the Running state + func controlPlaneNodesReady(client clientset.Interface, _ *kubeadmapi.ClusterConfiguration) error { ++ // list nodes labeled with a "master" node-role ++ selectorOldControlPlane := labels.SelectorFromSet(labels.Set(map[string]string{ ++ constants.LabelNodeRoleOldControlPlane: "", ++ })) ++ nodesWithOldLabel, err := client.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{ ++ LabelSelector: selectorOldControlPlane.String(), ++ }) ++ if err != nil { ++ return errors.Wrapf(err, "could not list nodes labeled with %q", constants.LabelNodeRoleOldControlPlane) ++ } ++ ++ // list nodes labeled with a "control-plane" node-role + selectorControlPlane := labels.SelectorFromSet(labels.Set(map[string]string{ + constants.LabelNodeRoleControlPlane: "", + })) +- nodes, err := client.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{ ++ nodesControlPlane, err := client.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{ + LabelSelector: selectorControlPlane.String(), + }) + if err != nil { + return errors.Wrapf(err, "could not list nodes labeled with %q", constants.LabelNodeRoleControlPlane) + } + +- notReadyControlPlanes := getNotReadyNodes(nodes.Items) ++ nodes := append(nodesWithOldLabel.Items, nodesControlPlane.Items...) ++ if len(nodes) == 0 { ++ return errors.New("failed to find any nodes with a control-plane role") ++ } ++ ++ notReadyControlPlanes := getNotReadyNodes(nodes) + if len(notReadyControlPlanes) != 0 { + return errors.Errorf("there are NotReady control-planes in the cluster: %v", notReadyControlPlanes) + } +diff --git a/cmd/kubeadm/app/phases/upgrade/postupgrade.go b/cmd/kubeadm/app/phases/upgrade/postupgrade.go +index fa215bfedf8..36e884195bc 100644 +--- a/cmd/kubeadm/app/phases/upgrade/postupgrade.go ++++ b/cmd/kubeadm/app/phases/upgrade/postupgrade.go +@@ -214,9 +214,10 @@ func rollbackFiles(files map[string]string, originalErr error) error { + return errors.Errorf("couldn't move these files: %v. Got errors: %v", files, errorsutil.NewAggregate(errs)) + } + +-// RemoveOldControlPlaneLabel finds all nodes with the legacy node-role label and removes it ++// LabelOldControlPlaneNodes finds all nodes with the legacy node-role label and also applies ++// the "control-plane" node-role label to them. + // TODO: https://github.com/kubernetes/kubeadm/issues/2200 +-func RemoveOldControlPlaneLabel(client clientset.Interface) error { ++func LabelOldControlPlaneNodes(client clientset.Interface) error { + selectorOldControlPlane := labels.SelectorFromSet(labels.Set(map[string]string{ + kubeadmconstants.LabelNodeRoleOldControlPlane: "", + })) +@@ -228,8 +229,11 @@ func RemoveOldControlPlaneLabel(client clientset.Interface) error { + } + + for _, n := range nodesWithOldLabel.Items { ++ if _, hasNewLabel := n.ObjectMeta.Labels[kubeadmconstants.LabelNodeRoleControlPlane]; hasNewLabel { ++ continue ++ } + err = apiclient.PatchNode(client, n.Name, func(n *v1.Node) { +- delete(n.ObjectMeta.Labels, kubeadmconstants.LabelNodeRoleOldControlPlane) ++ n.ObjectMeta.Labels[kubeadmconstants.LabelNodeRoleControlPlane] = "" + }) + if err != nil { + return err +-- +2.25.1 + diff --git a/kubernetes/kubernetes-1.24.4/debian/deb_folder/patches/Revert-kubeadm-only-apply-the-new-control-plane-labe.patch b/kubernetes/kubernetes-1.24.4/debian/deb_folder/patches/Revert-kubeadm-only-apply-the-new-control-plane-labe.patch new file mode 100644 index 000000000..acfee1d4a --- /dev/null +++ b/kubernetes/kubernetes-1.24.4/debian/deb_folder/patches/Revert-kubeadm-only-apply-the-new-control-plane-labe.patch @@ -0,0 +1,123 @@ +From 0119bd093db1d72ebf4fa00c4869979da2f9e2a1 Mon Sep 17 00:00:00 2001 +From: Jim Gauld +Date: Wed, 12 Oct 2022 14:00:33 -0400 +Subject: [PATCH 4/5] Revert "kubeadm: only apply the new "control-plane" label + during init/join" + +This reverts commit a3d5e5598290df09f4ffd5cf6813653a346c8f4c. +--- + .../phases/markcontrolplane/markcontrolplane.go | 14 +++++++++++--- + .../markcontrolplane/markcontrolplane_test.go | 9 ++++++--- + cmd/kubeadm/app/util/staticpod/utils.go | 7 +++++++ + 3 files changed, 24 insertions(+), 6 deletions(-) + +diff --git a/cmd/kubeadm/app/phases/markcontrolplane/markcontrolplane.go b/cmd/kubeadm/app/phases/markcontrolplane/markcontrolplane.go +index dd4c89eca6d..c68f80f7fd6 100644 +--- a/cmd/kubeadm/app/phases/markcontrolplane/markcontrolplane.go ++++ b/cmd/kubeadm/app/phases/markcontrolplane/markcontrolplane.go +@@ -19,23 +19,31 @@ package markcontrolplane + import ( + "fmt" + +- v1 "k8s.io/api/core/v1" ++ "k8s.io/api/core/v1" + clientset "k8s.io/client-go/kubernetes" + + "k8s.io/kubernetes/cmd/kubeadm/app/constants" + "k8s.io/kubernetes/cmd/kubeadm/app/util/apiclient" + ) + +-// labelsToAdd holds a list of labels that are applied on kubeadm managed control plane nodes + var labelsToAdd = []string{ ++ // TODO: remove this label: ++ // https://github.com/kubernetes/kubeadm/issues/2200 ++ constants.LabelNodeRoleOldControlPlane, + constants.LabelNodeRoleControlPlane, + constants.LabelExcludeFromExternalLB, + } + + // MarkControlPlane taints the control-plane and sets the control-plane label + func MarkControlPlane(client clientset.Interface, controlPlaneName string, taints []v1.Taint) error { ++ // TODO: remove this "deprecated" amend and pass "labelsToAdd" directly: ++ // https://github.com/kubernetes/kubeadm/issues/2200 ++ labels := make([]string, len(labelsToAdd)) ++ copy(labels, labelsToAdd) ++ labels[0] = constants.LabelNodeRoleOldControlPlane + "(deprecated)" ++ + fmt.Printf("[mark-control-plane] Marking the node %s as control-plane by adding the labels: %v\n", +- controlPlaneName, labelsToAdd) ++ controlPlaneName, labels) + + if len(taints) > 0 { + taintStrs := []string{} +diff --git a/cmd/kubeadm/app/phases/markcontrolplane/markcontrolplane_test.go b/cmd/kubeadm/app/phases/markcontrolplane/markcontrolplane_test.go +index b171072f80a..cb9275124ab 100644 +--- a/cmd/kubeadm/app/phases/markcontrolplane/markcontrolplane_test.go ++++ b/cmd/kubeadm/app/phases/markcontrolplane/markcontrolplane_test.go +@@ -49,25 +49,26 @@ func TestMarkControlPlane(t *testing.T) { + existingLabels: []string{""}, + existingTaints: nil, + newTaints: []v1.Taint{kubeadmconstants.OldControlPlaneTaint}, +- expectedPatch: `{"metadata":{"labels":{"node-role.kubernetes.io/control-plane":"","node.kubernetes.io/exclude-from-external-load-balancers":""}},"spec":{"taints":[{"effect":"NoSchedule","key":"node-role.kubernetes.io/master"}]}}`, ++ expectedPatch: `{"metadata":{"labels":{"node-role.kubernetes.io/control-plane":"","node-role.kubernetes.io/master":"","node.kubernetes.io/exclude-from-external-load-balancers":""}},"spec":{"taints":[{"effect":"NoSchedule","key":"node-role.kubernetes.io/master"}]}}`, + }, + { + name: "control-plane label and taint missing but taint not wanted", + existingLabels: []string{""}, + existingTaints: nil, + newTaints: nil, +- expectedPatch: `{"metadata":{"labels":{"node-role.kubernetes.io/control-plane":"","node.kubernetes.io/exclude-from-external-load-balancers":""}}}`, ++ expectedPatch: `{"metadata":{"labels":{"node-role.kubernetes.io/control-plane":"","node-role.kubernetes.io/master":"","node.kubernetes.io/exclude-from-external-load-balancers":""}}}`, + }, + { + name: "control-plane label missing", + existingLabels: []string{""}, + existingTaints: []v1.Taint{kubeadmconstants.OldControlPlaneTaint}, + newTaints: []v1.Taint{kubeadmconstants.OldControlPlaneTaint}, +- expectedPatch: `{"metadata":{"labels":{"node-role.kubernetes.io/control-plane":"","node.kubernetes.io/exclude-from-external-load-balancers":""}}}`, ++ expectedPatch: `{"metadata":{"labels":{"node-role.kubernetes.io/control-plane":"","node-role.kubernetes.io/master":"","node.kubernetes.io/exclude-from-external-load-balancers":""}}}`, + }, + { + name: "control-plane taint missing", + existingLabels: []string{ ++ kubeadmconstants.LabelNodeRoleOldControlPlane, + kubeadmconstants.LabelNodeRoleControlPlane, + kubeadmconstants.LabelExcludeFromExternalLB, + }, +@@ -78,6 +79,7 @@ func TestMarkControlPlane(t *testing.T) { + { + name: "nothing missing", + existingLabels: []string{ ++ kubeadmconstants.LabelNodeRoleOldControlPlane, + kubeadmconstants.LabelNodeRoleControlPlane, + kubeadmconstants.LabelExcludeFromExternalLB, + }, +@@ -88,6 +90,7 @@ func TestMarkControlPlane(t *testing.T) { + { + name: "has taint and no new taints wanted", + existingLabels: []string{ ++ kubeadmconstants.LabelNodeRoleOldControlPlane, + kubeadmconstants.LabelNodeRoleControlPlane, + kubeadmconstants.LabelExcludeFromExternalLB, + }, +diff --git a/cmd/kubeadm/app/util/staticpod/utils.go b/cmd/kubeadm/app/util/staticpod/utils.go +index 75efc4f0f98..f0d8fed0d46 100644 +--- a/cmd/kubeadm/app/util/staticpod/utils.go ++++ b/cmd/kubeadm/app/util/staticpod/utils.go +@@ -287,6 +287,13 @@ func createHTTPProbe(host, path string, port int, scheme v1.URIScheme, initialDe + + // GetAPIServerProbeAddress returns the probe address for the API server + func GetAPIServerProbeAddress(endpoint *kubeadmapi.APIEndpoint) string { ++ // In the case of a self-hosted deployment, the initial host on which kubeadm --init is run, ++ // will generate a DaemonSet with a nodeSelector such that all nodes with the label ++ // node-role.kubernetes.io/master='' will have the API server deployed to it. Since the init ++ // is run only once on an initial host, the API advertise address will be invalid for any ++ // future hosts that do not have the same address. Furthermore, since liveness and readiness ++ // probes do not support the Downward API we cannot dynamically set the advertise address to ++ // the node's IP. The only option then is to use localhost. + if endpoint != nil && endpoint.AdvertiseAddress != "" { + return getProbeAddress(endpoint.AdvertiseAddress) + } +-- +2.25.1 + diff --git a/kubernetes/kubernetes-1.24.4/debian/deb_folder/patches/series b/kubernetes/kubernetes-1.24.4/debian/deb_folder/patches/series index 4edf50dba..2ed957c64 100644 --- a/kubernetes/kubernetes-1.24.4/debian/deb_folder/patches/series +++ b/kubernetes/kubernetes-1.24.4/debian/deb_folder/patches/series @@ -1,3 +1,8 @@ +Revert-e2e-framework-test_context-add-control-plane-.patch +Revert-kubeadm-apply-the-new-control-plane-taint-dur.patch +Revert-kubeadm-apply-the-new-control-plane-taint-on-.patch +Revert-kubeadm-only-apply-the-new-control-plane-labe.patch +Revert-kubeadm-delete-the-old-master-label-during-up.patch kubeadm-create-platform-pods-with-zero-CPU-resources.patch Revert-use-subpath-for-coredns-only-for-default-repo.patch kubernetes-make-isolcpus-allocation-SMT-aware.patch