From 22903011b263f97865af74933714259755647db1 Mon Sep 17 00:00:00 2001 From: Danil Grigorev Date: Tue, 31 Oct 2023 12:18:30 +0100 Subject: [PATCH] Fix nil machine occurence during upgrade Signed-off-by: Danil Grigorev --- .../rke2controlplane_controller_test.go | 170 ++++++++++++++++++ .../internal/controllers/suite_test.go | 81 ++++----- pkg/rke2/workload_cluster.go | 4 + 3 files changed, 213 insertions(+), 42 deletions(-) create mode 100644 controlplane/internal/controllers/rke2controlplane_controller_test.go diff --git a/controlplane/internal/controllers/rke2controlplane_controller_test.go b/controlplane/internal/controllers/rke2controlplane_controller_test.go new file mode 100644 index 00000000..6094409d --- /dev/null +++ b/controlplane/internal/controllers/rke2controlplane_controller_test.go @@ -0,0 +1,170 @@ +package controllers + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + bootstrapv1 "github.com/rancher-sandbox/cluster-api-provider-rke2/bootstrap/api/v1alpha1" + controlplanev1 "github.com/rancher-sandbox/cluster-api-provider-rke2/controlplane/api/v1alpha1" + + // "github.com/rancher-sandbox/cluster-api-provider-rke2/pkg/kubeconfig" + "github.com/rancher-sandbox/cluster-api-provider-rke2/pkg/rke2" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + "sigs.k8s.io/cluster-api/util/collections" + "sigs.k8s.io/cluster-api/util/conditions" + "sigs.k8s.io/cluster-api/util/kubeconfig" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +var _ = Describe("Reconclie control plane conditions", func() { + var ( + err error + cp *rke2.ControlPlane + rcp *controlplanev1.RKE2ControlPlane + ns *corev1.Namespace + nodeName = "node1" + node *corev1.Node + orphanedNode *corev1.Node + machine *clusterv1.Machine + config *bootstrapv1.RKE2Config + ) + + BeforeEach(func() { + ns, err = testEnv.CreateNamespace(ctx, "ns") + Expect(err).ToNot(HaveOccurred()) + + annotations := map[string]string{ + "test": "true", + } + node = &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: nodeName, + Labels: map[string]string{ + "node-role.kubernetes.io/master": "true", + }, + Annotations: map[string]string{ + clusterv1.MachineAnnotation: nodeName, + }, + }, + Status: corev1.NodeStatus{ + Conditions: []corev1.NodeCondition{{ + Type: corev1.NodeReady, + Status: corev1.ConditionTrue, + }}, + }, + } + Expect(testEnv.Create(ctx, node.DeepCopy())).To(Succeed()) + Expect(testEnv.Status().Update(ctx, node.DeepCopy())).To(Succeed()) + Expect(testEnv.Get(ctx, client.ObjectKeyFromObject(node), node)).To(Succeed()) + + orphanedNode = &corev1.Node{ObjectMeta: metav1.ObjectMeta{ + Name: "missing-machine", + Labels: map[string]string{ + "node-role.kubernetes.io/master": "true", + }, + }} + Expect(testEnv.Create(ctx, orphanedNode)).To(Succeed()) + + config = &bootstrapv1.RKE2Config{ObjectMeta: metav1.ObjectMeta{ + Name: "config", + Namespace: ns.Name, + }, Spec: bootstrapv1.RKE2ConfigSpec{ + AgentConfig: bootstrapv1.RKE2AgentConfig{ + NodeAnnotations: annotations, + }, + }} + Expect(testEnv.Create(ctx, config)).To(Succeed()) + + machine = &clusterv1.Machine{ + ObjectMeta: metav1.ObjectMeta{ + Name: nodeName, + Namespace: ns.Name, + }, + Spec: clusterv1.MachineSpec{ + ClusterName: "cluster", + Bootstrap: clusterv1.Bootstrap{ + ConfigRef: &corev1.ObjectReference{ + Kind: "RKE2Config", + APIVersion: bootstrapv1.GroupVersion.String(), + Name: config.Name, + Namespace: config.Namespace, + }, + }, + InfrastructureRef: corev1.ObjectReference{ + Kind: "Pod", + APIVersion: "v1", + Name: "stub", + Namespace: ns.Name, + }, + }, + Status: clusterv1.MachineStatus{ + NodeRef: &corev1.ObjectReference{ + Kind: "Node", + Name: nodeName, + UID: node.GetUID(), + Namespace: "", + }, + Conditions: clusterv1.Conditions{ + clusterv1.Condition{ + Type: clusterv1.ReadyCondition, + Status: corev1.ConditionTrue, + }, + }, + }, + } + ml := clusterv1.MachineList{Items: []clusterv1.Machine{*machine.DeepCopy()}} + Expect(testEnv.Create(ctx, machine.DeepCopy())).To(Succeed()) + + cluster := &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: ns.Name, + }, + } + Expect(testEnv.Client.Create(ctx, cluster)).To(Succeed()) + + rcp = &controlplanev1.RKE2ControlPlane{ + Status: controlplanev1.RKE2ControlPlaneStatus{ + Initialized: true, + }, + } + + cp, err = rke2.NewControlPlane(ctx, testEnv.GetClient(), cluster, rcp, collections.FromMachineList(&ml)) + Expect(err).ToNot(HaveOccurred()) + + ref := metav1.OwnerReference{ + APIVersion: clusterv1.GroupVersion.String(), + Kind: clusterv1.ClusterKind, + UID: cp.Cluster.GetUID(), + Name: cp.Cluster.GetName(), + } + Expect(testEnv.Client.Create(ctx, kubeconfig.GenerateSecretWithOwner( + client.ObjectKeyFromObject(cp.Cluster), + kubeconfig.FromEnvTestConfig(testEnv.Config, cp.Cluster), + ref))).To(Succeed()) + }) + + AfterEach(func() { + Expect(testEnv.DeleteAllOf(ctx, node)).To(Succeed()) + testEnv.Cleanup(ctx, node, ns) + }) + + It("should reconcile cp and machine conditions successfully", func() { + r := &RKE2ControlPlaneReconciler{ + Client: testEnv.GetClient(), + Scheme: testEnv.GetScheme(), + managementCluster: &rke2.Management{Client: testEnv.GetClient()}, + managementClusterUncached: &rke2.Management{Client: testEnv.GetClient()}, + } + _, err := r.reconcileControlPlaneConditions(ctx, cp) + Expect(err).ToNot(HaveOccurred()) + Expect(testEnv.Get(ctx, client.ObjectKeyFromObject(machine), machine)).To(Succeed()) + Expect(conditions.IsTrue(machine, controlplanev1.NodeMetadataUpToDate)).To(BeTrue()) + Expect(testEnv.Get(ctx, client.ObjectKeyFromObject(node), node)).To(Succeed()) + Expect(node.GetAnnotations()).To(HaveKeyWithValue("test", "true")) + Expect(conditions.IsFalse(rcp, controlplanev1.ControlPlaneComponentsHealthyCondition)).To(BeTrue()) + Expect(conditions.GetMessage(rcp, controlplanev1.ControlPlaneComponentsHealthyCondition)).To(Equal( + "Control plane node missing-machine does not have a corresponding machine")) + }) +}) diff --git a/controlplane/internal/controllers/suite_test.go b/controlplane/internal/controllers/suite_test.go index 13aa0b1b..d2dcf3c9 100644 --- a/controlplane/internal/controllers/suite_test.go +++ b/controlplane/internal/controllers/suite_test.go @@ -17,64 +17,61 @@ limitations under the License. package controllers import ( - "path/filepath" + "fmt" + "path" "testing" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/client-go/kubernetes/scheme" + ctrl "sigs.k8s.io/controller-runtime" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "k8s.io/client-go/kubernetes/scheme" - "k8s.io/client-go/rest" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/envtest" - logf "sigs.k8s.io/controller-runtime/pkg/log" - "sigs.k8s.io/controller-runtime/pkg/log/zap" - + // +kubebuilder:scaffold:imports + bootstrapv1 "github.com/rancher-sandbox/cluster-api-provider-rke2/bootstrap/api/v1alpha1" controlplanev1 "github.com/rancher-sandbox/cluster-api-provider-rke2/controlplane/api/v1alpha1" + "github.com/rancher-sandbox/cluster-api-provider-rke2/test/helpers" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" ) -// These tests use Ginkgo (BDD-style Go testing framework). Refer to -// http://onsi.github.io/ginkgo/ to learn more about Ginkgo. - var ( - cfg *rest.Config - k8sClient client.Client - testEnv *envtest.Environment + testEnv *helpers.TestEnvironment + ctx = ctrl.SetupSignalHandler() ) func TestAPIs(t *testing.T) { RegisterFailHandler(Fail) - + setup() + defer teardown() RunSpecs(t, "Controller Suite") } -var _ = BeforeSuite(func() { - logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) - - By("bootstrapping test environment") - testEnv = &envtest.Environment{ - CRDDirectoryPaths: []string{filepath.Join("..", "..", "config", "crd", "bases")}, - ErrorIfCRDPathMissing: true, - } +func setup() { + utilruntime.Must(clusterv1.AddToScheme(scheme.Scheme)) + utilruntime.Must(bootstrapv1.AddToScheme(scheme.Scheme)) + utilruntime.Must(controlplanev1.AddToScheme(scheme.Scheme)) + testEnvConfig := helpers.NewTestEnvironmentConfiguration([]string{ + path.Join("bootstrap", "config", "crd", "bases"), + path.Join("controlplane", "config", "crd", "bases"), + }, + ) var err error - // cfg is defined in this file globally. - cfg, err = testEnv.Start() - Expect(err).NotTo(HaveOccurred()) - Expect(cfg).NotTo(BeNil()) - - err = controlplanev1.AddToScheme(scheme.Scheme) - Expect(err).NotTo(HaveOccurred()) - - //+kubebuilder:scaffold:scheme - - k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) - Expect(err).NotTo(HaveOccurred()) - Expect(k8sClient).NotTo(BeNil()) -}) + testEnv, err = testEnvConfig.Build() + if err != nil { + panic(err) + } + go func() { + fmt.Println("Starting the manager") + if err := testEnv.StartManager(ctx); err != nil { + panic(fmt.Sprintf("Failed to start the envtest manager: %v", err)) + } + }() +} -var _ = AfterSuite(func() { - By("tearing down the test environment") - err := testEnv.Stop() - Expect(err).NotTo(HaveOccurred()) -}) +func teardown() { + if err := testEnv.Stop(); err != nil { + panic(fmt.Sprintf("Failed to stop envtest: %v", err)) + } +} diff --git a/pkg/rke2/workload_cluster.go b/pkg/rke2/workload_cluster.go index 5e939770..cb5aecc2 100644 --- a/pkg/rke2/workload_cluster.go +++ b/pkg/rke2/workload_cluster.go @@ -142,6 +142,10 @@ func (w *Workload) PatchNodes(ctx context.Context, cp *ControlPlane) error { for i := range w.Nodes { node := w.Nodes[i] + if _, found := cp.Machines[node.Name]; !found { + continue + } + if helper, ok := w.nodePatchHelpers[node.Name]; ok { if err := helper.Patch(ctx, node); err != nil { conditions.MarkUnknown(