From 1a556376828bf57c98339b736d0a0f5ff491ba2e Mon Sep 17 00:00:00 2001 From: drivebyer Date: Fri, 29 Mar 2024 15:28:42 +0800 Subject: [PATCH 1/4] feat: add redisreplication status masterNode Signed-off-by: drivebyer --- api/v1beta1/redisreplication_types.go | 4 ++- api/v1beta2/redisreplication_types.go | 4 ++- ...edis.opstreelabs.in_redisreplications.yaml | 6 ++++ controllers/redisreplication_controller.go | 26 +++++++++++++-- k8sutils/redis.go | 32 ++++++++----------- .../redis-replication/chainsaw-test.yaml | 22 ++++++------- 6 files changed, 58 insertions(+), 36 deletions(-) diff --git a/api/v1beta1/redisreplication_types.go b/api/v1beta1/redisreplication_types.go index 4cca92403..d4be20456 100644 --- a/api/v1beta1/redisreplication_types.go +++ b/api/v1beta1/redisreplication_types.go @@ -31,7 +31,9 @@ func (cr *RedisReplicationSpec) GetReplicationCounts(t string) int32 { } // RedisStatus defines the observed state of Redis -type RedisReplicationStatus struct{} +type RedisReplicationStatus struct { + MasterNode string `json:"masterNode,omitempty"` +} // +kubebuilder:object:root=true // +kubebuilder:subresource:status diff --git a/api/v1beta2/redisreplication_types.go b/api/v1beta2/redisreplication_types.go index 59248fb55..90bf70796 100644 --- a/api/v1beta2/redisreplication_types.go +++ b/api/v1beta2/redisreplication_types.go @@ -36,7 +36,9 @@ func (cr *RedisReplicationSpec) GetReplicationCounts(t string) int32 { } // RedisStatus defines the observed state of Redis -type RedisReplicationStatus struct{} +type RedisReplicationStatus struct { + MasterNode string `json:"masterNode,omitempty"` +} // +kubebuilder:object:root=true // +kubebuilder:subresource:status diff --git a/config/crd/bases/redis.redis.opstreelabs.in_redisreplications.yaml b/config/crd/bases/redis.redis.opstreelabs.in_redisreplications.yaml index c52f9a10a..e1c111432 100644 --- a/config/crd/bases/redis.redis.opstreelabs.in_redisreplications.yaml +++ b/config/crd/bases/redis.redis.opstreelabs.in_redisreplications.yaml @@ -3834,6 +3834,9 @@ spec: type: object status: description: RedisStatus defines the observed state of Redis + properties: + masterNode: + type: string type: object required: - spec @@ -8255,6 +8258,9 @@ spec: type: object status: description: RedisStatus defines the observed state of Redis + properties: + masterNode: + type: string type: object required: - spec diff --git a/controllers/redisreplication_controller.go b/controllers/redisreplication_controller.go index a57308e80..98a2b9fac 100644 --- a/controllers/redisreplication_controller.go +++ b/controllers/redisreplication_controller.go @@ -77,19 +77,39 @@ func (r *RedisReplicationReconciler) Reconcile(ctx context.Context, req ctrl.Req return ctrl.Result{RequeueAfter: time.Second * 60}, nil } - if len(k8sutils.GetRedisNodesByRole(ctx, r.K8sClient, r.Log, instance, "master")) > int(leaderReplicas) { + var realMaster string + masterNodes := k8sutils.GetRedisNodesByRole(ctx, r.K8sClient, r.Log, instance, "master") + if len(masterNodes) > int(leaderReplicas) { reqLogger.Info("Creating redis replication by executing replication creation commands", "Replication.Ready", strconv.Itoa(int(redisReplicationInfo.Status.ReadyReplicas))) - masterNodes := k8sutils.GetRedisNodesByRole(ctx, r.K8sClient, r.Log, instance, "master") slaveNodes := k8sutils.GetRedisNodesByRole(ctx, r.K8sClient, r.Log, instance, "slave") - err := k8sutils.CreateMasterSlaveReplication(ctx, r.K8sClient, r.Log, instance, masterNodes, slaveNodes) + realMaster = k8sutils.GetRedisReplicationRealMaster(ctx, r.K8sClient, r.Log, instance, masterNodes) + if len(slaveNodes) == 0 { + realMaster = masterNodes[0] + } + err := k8sutils.CreateMasterSlaveReplication(ctx, r.K8sClient, r.Log, instance, masterNodes, realMaster) if err != nil { return ctrl.Result{RequeueAfter: time.Second * 60}, err } } + realMaster = k8sutils.GetRedisReplicationRealMaster(ctx, r.K8sClient, r.Log, instance, masterNodes) + if err := r.UpdateRedisReplicationMaster(ctx, instance, realMaster); err != nil { + return ctrl.Result{}, err + } reqLogger.Info("Will reconcile redis operator in again 10 seconds") return ctrl.Result{RequeueAfter: time.Second * 10}, nil } +func (r *RedisReplicationReconciler) UpdateRedisReplicationMaster(ctx context.Context, instance *redisv1beta2.RedisReplication, masterNode string) error { + if instance.Status.MasterNode == masterNode { + return nil + } + instance.Status.MasterNode = masterNode + if err := r.Client.Status().Update(ctx, instance); err != nil { + return err + } + return nil +} + // SetupWithManager sets up the controller with the Manager. func (r *RedisReplicationReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). diff --git a/k8sutils/redis.go b/k8sutils/redis.go index b7f74ef05..bb983084c 100644 --- a/k8sutils/redis.go +++ b/k8sutils/redis.go @@ -554,25 +554,7 @@ func checkAttachedSlave(ctx context.Context, redisClient *redis.Client, logger l return 0 } -func CreateMasterSlaveReplication(ctx context.Context, client kubernetes.Interface, logger logr.Logger, cr *redisv1beta2.RedisReplication, masterPods []string, slavePods []string) error { - var realMasterPod string - - for _, podName := range masterPods { - redisClient := configureRedisReplicationClient(client, logger, cr, podName) - defer redisClient.Close() - - if checkAttachedSlave(ctx, redisClient, logger, podName) > 0 { - realMasterPod = podName - break - } - } - // realMasterPod = checkAttachedSlave(ctx, client, logger, cr, masterPods) - - if len(slavePods) < 1 { - realMasterPod = masterPods[0] - logger.V(1).Info("No Master Node Found with attached slave promoting the following pod to master", "pod", masterPods[0]) - } - +func CreateMasterSlaveReplication(ctx context.Context, client kubernetes.Interface, logger logr.Logger, cr *redisv1beta2.RedisReplication, masterPods []string, realMasterPod string) error { logger.V(1).Info("Redis Master Node is set to", "pod", realMasterPod) realMasterInfo := RedisDetails{ PodName: realMasterPod, @@ -596,3 +578,15 @@ func CreateMasterSlaveReplication(ctx context.Context, client kubernetes.Interfa return nil } + +func GetRedisReplicationRealMaster(ctx context.Context, client kubernetes.Interface, logger logr.Logger, cr *redisv1beta2.RedisReplication, masterPods []string) string { + for _, podName := range masterPods { + redisClient := configureRedisReplicationClient(client, logger, cr, podName) + defer redisClient.Close() + + if checkAttachedSlave(ctx, redisClient, logger, podName) > 0 { + return podName + } + } + return "" +} diff --git a/tests/e2e-chainsaw/v1beta2/setup/redis-replication/chainsaw-test.yaml b/tests/e2e-chainsaw/v1beta2/setup/redis-replication/chainsaw-test.yaml index ff7822ce7..fc913dd3e 100644 --- a/tests/e2e-chainsaw/v1beta2/setup/redis-replication/chainsaw-test.yaml +++ b/tests/e2e-chainsaw/v1beta2/setup/redis-replication/chainsaw-test.yaml @@ -52,15 +52,13 @@ spec: kubectl exec --namespace ${NAMESPACE} redis-replication-0 -- redis-cli -p 6379 set foo-0 bar-0 check: ($stdout=='OK'): true - # - script: - # timeout: 10s - # content: | - # kubectl exec --namespace ${NAMESPACE} redis-replication-1 -- redis-cli -p 6379 set foo-1 bar-1 - # check: - # ($stdout==`READONLY You can't write against a read only replica.`): true - # - script: - # timeout: 10s - # content: | - # kubectl exec --namespace ${NAMESPACE} redis-replication-2 -- redis-cli -p 6379 set foo-2 bar-2 - # check: - # ($stdout==`READONLY You can't write against a read only replica.`): true \ No newline at end of file + + - name: Check Status + try: + - script: + timeout: 10s + content: | + kubectl --namespace ${NAMESPACE} get redisreplications.v1beta2.redis.redis.opstreelabs.in redis-replication -o jsonpath='{.status.masterNode}' + check: + # by default, we select the first node as master + ($stdout=='redis-replication-0'): true \ No newline at end of file From 151117d962b4f47a347767194319689cddcad62f Mon Sep 17 00:00:00 2001 From: drivebyer Date: Fri, 29 Mar 2024 15:49:56 +0800 Subject: [PATCH 2/4] fix e2e Signed-off-by: drivebyer --- controllers/redissentinel_controller.go | 12 ++++++++++++ .../setup/redis-replication/chainsaw-test.yaml | 14 +++----------- .../setup/redis-replication/ready-replication.yaml | 8 ++++++++ 3 files changed, 23 insertions(+), 11 deletions(-) create mode 100644 tests/e2e-chainsaw/v1beta2/setup/redis-replication/ready-replication.yaml diff --git a/controllers/redissentinel_controller.go b/controllers/redissentinel_controller.go index ff2292f29..374a97ae8 100644 --- a/controllers/redissentinel_controller.go +++ b/controllers/redissentinel_controller.go @@ -2,6 +2,9 @@ package controllers import ( "context" + "k8s.io/client-go/util/workqueue" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" "time" redisv1beta2 "github.com/OT-CONTAINER-KIT/redis-operator/api/v1beta2" @@ -84,5 +87,14 @@ func (r *RedisSentinelReconciler) Reconcile(ctx context.Context, req ctrl.Reques func (r *RedisSentinelReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&redisv1beta2.RedisSentinel{}). + Watches(&redisv1beta2.RedisReplication{}, &handler.Funcs{ + CreateFunc: nil, + UpdateFunc: func(ctx context.Context, event event.UpdateEvent, limitingInterface workqueue.RateLimitingInterface) { + _ = event.ObjectNew.GetName() + _ = event.ObjectNew.GetNamespace() + }, + DeleteFunc: nil, + GenericFunc: nil, + }). Complete(r) } diff --git a/tests/e2e-chainsaw/v1beta2/setup/redis-replication/chainsaw-test.yaml b/tests/e2e-chainsaw/v1beta2/setup/redis-replication/chainsaw-test.yaml index fc913dd3e..0b536b458 100644 --- a/tests/e2e-chainsaw/v1beta2/setup/redis-replication/chainsaw-test.yaml +++ b/tests/e2e-chainsaw/v1beta2/setup/redis-replication/chainsaw-test.yaml @@ -15,6 +15,8 @@ spec: file: ready-svc.yaml - assert: file: ready-pvc.yaml + - assert: + file: ready-replication.yaml catch: - description: Redis Operator Logs podLogs: @@ -51,14 +53,4 @@ spec: content: | kubectl exec --namespace ${NAMESPACE} redis-replication-0 -- redis-cli -p 6379 set foo-0 bar-0 check: - ($stdout=='OK'): true - - - name: Check Status - try: - - script: - timeout: 10s - content: | - kubectl --namespace ${NAMESPACE} get redisreplications.v1beta2.redis.redis.opstreelabs.in redis-replication -o jsonpath='{.status.masterNode}' - check: - # by default, we select the first node as master - ($stdout=='redis-replication-0'): true \ No newline at end of file + ($stdout=='OK'): true \ No newline at end of file diff --git a/tests/e2e-chainsaw/v1beta2/setup/redis-replication/ready-replication.yaml b/tests/e2e-chainsaw/v1beta2/setup/redis-replication/ready-replication.yaml new file mode 100644 index 000000000..f3a13d647 --- /dev/null +++ b/tests/e2e-chainsaw/v1beta2/setup/redis-replication/ready-replication.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: redis.redis.opstreelabs.in/v1beta2 +kind: RedisReplication +metadata: + name: redis-replication +status: + # by default, the first pod is being selected as master + masterNode: redis-replication-0 \ No newline at end of file From 648a78d62b57e7b9c3d3afba1f443f69bf0a0a73 Mon Sep 17 00:00:00 2001 From: drivebyer Date: Fri, 29 Mar 2024 15:56:01 +0800 Subject: [PATCH 3/4] fix lint Signed-off-by: drivebyer --- controllers/redissentinel_controller.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/controllers/redissentinel_controller.go b/controllers/redissentinel_controller.go index 374a97ae8..bcbd58616 100644 --- a/controllers/redissentinel_controller.go +++ b/controllers/redissentinel_controller.go @@ -2,10 +2,11 @@ package controllers import ( "context" + "time" + "k8s.io/client-go/util/workqueue" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" - "time" redisv1beta2 "github.com/OT-CONTAINER-KIT/redis-operator/api/v1beta2" "github.com/OT-CONTAINER-KIT/redis-operator/k8sutils" From f923e05f4473c218ef8f2172ab85e0ac53374b0d Mon Sep 17 00:00:00 2001 From: drivebyer Date: Fri, 29 Mar 2024 16:07:02 +0800 Subject: [PATCH 4/4] fix lint Signed-off-by: drivebyer --- controllers/redissentinel_controller.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/controllers/redissentinel_controller.go b/controllers/redissentinel_controller.go index bcbd58616..5c9cf142f 100644 --- a/controllers/redissentinel_controller.go +++ b/controllers/redissentinel_controller.go @@ -4,10 +4,6 @@ import ( "context" "time" - "k8s.io/client-go/util/workqueue" - "sigs.k8s.io/controller-runtime/pkg/event" - "sigs.k8s.io/controller-runtime/pkg/handler" - redisv1beta2 "github.com/OT-CONTAINER-KIT/redis-operator/api/v1beta2" "github.com/OT-CONTAINER-KIT/redis-operator/k8sutils" "github.com/go-logr/logr" @@ -15,8 +11,11 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" + "k8s.io/client-go/util/workqueue" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" ) // RedisSentinelReconciler reconciles a RedisSentinel object