feat: support exporter custom port (OT-CONTAINER-KIT#728)

* support exporter custom port Signed-off-by: drivebyer <[email protected]> * add e2e Signed-off-by: drivebyer <[email protected]> --------- Signed-off-by: drivebyer <[email protected]> Signed-off-by: Matt Robinson <[email protected]>
mattrobinsonsre · Jul 11, 2024 · 0c4fd0a · 0c4fd0a
1 parent 87ff0ef
commit 0c4fd0a
Show file tree

Hide file tree

Showing 22 changed files with 298 additions and 86 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -22,6 +22,7 @@ COPY main.go main.go
 COPY api/ api/
 COPY controllers/ controllers/
 COPY k8sutils/ k8sutils/
+COPY pkg/ pkg/
 COPY mocks/ mocks/
 
 # Build

diff --git a/api/common_types.go b/api/common_types.go
@@ -37,7 +37,9 @@ type ExistingPasswordSecret struct {
 // RedisExporter interface will have the information for redis exporter related stuff
 // +k8s:deepcopy-gen=true
 type RedisExporter struct {
-	Enabled         bool                         `json:"enabled,omitempty"`
+	Enabled bool `json:"enabled,omitempty"`
+	// +kubebuilder:default:=9121
+	Port            *int                         `json:"port,omitempty"`
 	Image           string                       `json:"image"`
 	Resources       *corev1.ResourceRequirements `json:"resources,omitempty"`
 	ImagePullPolicy corev1.PullPolicy            `json:"imagePullPolicy,omitempty"`

diff --git a/api/v1beta2/rediscluster_default.go b/api/v1beta2/rediscluster_default.go
@@ -7,4 +7,7 @@ func (r *RedisCluster) SetDefault() {
 	if r.Spec.Port == nil {
 		r.Spec.Port = pointer.Int(6379)
 	}
+	if r.Spec.RedisExporter != nil && r.Spec.RedisExporter.Port == nil {
+		r.Spec.RedisExporter.Port = pointer.Int(9121)
+	}
 }
diff --git a/api/zz_generated.deepcopy.go b/api/zz_generated.deepcopy.go
diff --git a/config/crd/bases/redis.redis.opstreelabs.in_redis.yaml b/config/crd/bases/redis.redis.opstreelabs.in_redis.yaml
@@ -1274,6 +1274,9 @@ spec:
                     description: PullPolicy describes a policy for if/when to pull
                       a container image
                     type: string
+                  port:
+                    default: 9121
+                    type: integer
                   resources:
                     description: ResourceRequirements describes the compute resource
                       requirements.
@@ -5615,6 +5618,9 @@ spec:
                     description: PullPolicy describes a policy for if/when to pull
                       a container image
                     type: string
+                  port:
+                    default: 9121
+                    type: integer
                   resources:
                     description: ResourceRequirements describes the compute resource
                       requirements.

diff --git a/config/crd/bases/redis.redis.opstreelabs.in_redisclusters.yaml b/config/crd/bases/redis.redis.opstreelabs.in_redisclusters.yaml
@@ -402,6 +402,9 @@ spec:
                     description: PullPolicy describes a policy for if/when to pull
                       a container image
                     type: string
+                  port:
+                    default: 9121
+                    type: integer
                   resources:
                     description: ResourceRequirements describes the compute resource
                       requirements.
@@ -5920,6 +5923,9 @@ spec:
                     description: PullPolicy describes a policy for if/when to pull
                       a container image
                     type: string
+                  port:
+                    default: 9121
+                    type: integer
                   resources:
                     description: ResourceRequirements describes the compute resource
                       requirements.

diff --git a/config/crd/bases/redis.redis.opstreelabs.in_redisreplications.yaml b/config/crd/bases/redis.redis.opstreelabs.in_redisreplications.yaml
@@ -1276,6 +1276,9 @@ spec:
                     description: PullPolicy describes a policy for if/when to pull
                       a container image
                     type: string
+                  port:
+                    default: 9121
+                    type: integer
                   resources:
                     description: ResourceRequirements describes the compute resource
                       requirements.
@@ -5620,6 +5623,9 @@ spec:
                     description: PullPolicy describes a policy for if/when to pull
                       a container image
                     type: string
+                  port:
+                    default: 9121
+                    type: integer
                   resources:
                     description: ResourceRequirements describes the compute resource
                       requirements.

diff --git a/config/crd/bases/redis.redis.opstreelabs.in_redissentinels.yaml b/config/crd/bases/redis.redis.opstreelabs.in_redissentinels.yaml
@@ -3316,6 +3316,9 @@ spec:
                     description: PullPolicy describes a policy for if/when to pull
                       a container image
                     type: string
+                  port:
+                    default: 9121
+                    type: integer
                   resources:
                     description: ResourceRequirements describes the compute resource
                       requirements.

diff --git a/k8sutils/labels.go b/k8sutils/labels.go
@@ -1,6 +1,8 @@
 package k8sutils
 
 import (
+	"strconv"
+
 	redisv1beta2 "github.com/OT-CONTAINER-KIT/redis-operator/api/v1beta2"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
@@ -111,13 +113,16 @@ func filterAnnotations(anots map[string]string, ignoreAnnots ...string) map[stri
 }
 
 // generateServiceAnots generates and returns service annotations
-func generateServiceAnots(stsMeta metav1.ObjectMeta, additionalSvcAnnotations map[string]string) map[string]string {
+func generateServiceAnots(stsMeta metav1.ObjectMeta, additionalSvcAnnotations map[string]string, epp exporterPortProvider) map[string]string {
 	anots := map[string]string{
 		"redis.opstreelabs.in":       "true",
 		"redis.opstreelabs.instance": stsMeta.GetName(),
 		"prometheus.io/scrape":       "true",
 		"prometheus.io/port":         "9121",
 	}
+	if exporterPort, ok := epp(); ok {
+		anots["prometheus.io/port"] = strconv.Itoa(exporterPort)
+	}
 	for k, v := range stsMeta.GetAnnotations() {
 		anots[k] = v
 	}

diff --git a/k8sutils/labels_test.go b/k8sutils/labels_test.go
@@ -215,7 +215,7 @@ func TestGenerateServiceAnots(t *testing.T) {
 		"additional-annotation":      "additional-value",
 	}
 
-	resultAnnotations := generateServiceAnots(stsMeta, additionalSvcAnnotations)
+	resultAnnotations := generateServiceAnots(stsMeta, additionalSvcAnnotations, defaultExporterPortProvider)
 
 	if !reflect.DeepEqual(resultAnnotations, expectedAnnotations) {
 		t.Errorf("Expected annotations to be %v but got %v", expectedAnnotations, resultAnnotations)

diff --git a/k8sutils/redis-cluster.go b/k8sutils/redis-cluster.go
@@ -3,7 +3,9 @@ package k8sutils
 import (
 	commonapi "github.com/OT-CONTAINER-KIT/redis-operator/api"
 	redisv1beta2 "github.com/OT-CONTAINER-KIT/redis-operator/api/v1beta2"
+	"github.com/OT-CONTAINER-KIT/redis-operator/pkg/util"
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/utils/pointer"
 )
 
 // RedisClusterSTS is a interface to call Redis Statefulset function
@@ -129,11 +131,12 @@ func generateRedisClusterContainerParams(cr *redisv1beta2.RedisCluster, security
 		if cr.Spec.RedisExporter.Resources != nil {
 			containerProp.RedisExporterResources = cr.Spec.RedisExporter.Resources
 		}
-
 		if cr.Spec.RedisExporter.EnvVars != nil {
 			containerProp.RedisExporterEnv = cr.Spec.RedisExporter.EnvVars
 		}
-
+		if cr.Spec.RedisExporter.Port != nil {
+			containerProp.RedisExporterPort = cr.Spec.RedisExporter.Port
+		}
 	}
 	if readinessProbeDef != nil {
 		containerProp.ReadinessProbe = readinessProbeDef
@@ -240,25 +243,29 @@ func (service RedisClusterService) CreateRedisClusterService(cr *redisv1beta2.Re
 	serviceName := cr.ObjectMeta.Name + "-" + service.RedisServiceRole
 	logger := serviceLogger(cr.Namespace, serviceName)
 	labels := getRedisLabels(serviceName, cluster, service.RedisServiceRole, cr.ObjectMeta.Labels)
-	annotations := generateServiceAnots(cr.ObjectMeta, nil)
-	if cr.Spec.RedisExporter != nil && cr.Spec.RedisExporter.Enabled {
-		enableMetrics = true
+	var epp exporterPortProvider
+	if cr.Spec.RedisExporter != nil {
+		epp = func() (port int, enable bool) {
+			defaultP := pointer.Int(redisExporterPort)
+			return *util.Coalesce(cr.Spec.RedisExporter.Port, defaultP), cr.Spec.RedisExporter.Enabled
+		}
 	} else {
-		enableMetrics = false
+		epp = disableMetrics
 	}
+	annotations := generateServiceAnots(cr.ObjectMeta, nil, epp)
 	additionalServiceAnnotations := map[string]string{}
 	if cr.Spec.KubernetesConfig.Service != nil {
 		additionalServiceAnnotations = cr.Spec.KubernetesConfig.Service.ServiceAnnotations
 	}
 	objectMetaInfo := generateObjectMetaInformation(serviceName, cr.Namespace, labels, annotations)
 	headlessObjectMetaInfo := generateObjectMetaInformation(serviceName+"-headless", cr.Namespace, labels, annotations)
-	additionalObjectMetaInfo := generateObjectMetaInformation(serviceName+"-additional", cr.Namespace, labels, generateServiceAnots(cr.ObjectMeta, additionalServiceAnnotations))
-	err := CreateOrUpdateService(cr.Namespace, headlessObjectMetaInfo, redisClusterAsOwner(cr), false, true, "ClusterIP", *cr.Spec.Port)
+	additionalObjectMetaInfo := generateObjectMetaInformation(serviceName+"-additional", cr.Namespace, labels, generateServiceAnots(cr.ObjectMeta, additionalServiceAnnotations, epp))
+	err := CreateOrUpdateService(cr.Namespace, headlessObjectMetaInfo, redisClusterAsOwner(cr), disableMetrics, true, "ClusterIP", *cr.Spec.Port)
 	if err != nil {
 		logger.Error(err, "Cannot create headless service for Redis", "Setup.Type", service.RedisServiceRole)
 		return err
 	}
-	err = CreateOrUpdateService(cr.Namespace, objectMetaInfo, redisClusterAsOwner(cr), enableMetrics, false, "ClusterIP", *cr.Spec.Port)
+	err = CreateOrUpdateService(cr.Namespace, objectMetaInfo, redisClusterAsOwner(cr), epp, false, "ClusterIP", *cr.Spec.Port)
 	if err != nil {
 		logger.Error(err, "Cannot create service for Redis", "Setup.Type", service.RedisServiceRole)
 		return err
@@ -267,7 +274,7 @@ func (service RedisClusterService) CreateRedisClusterService(cr *redisv1beta2.Re
 	if cr.Spec.KubernetesConfig.Service != nil {
 		additionalServiceType = cr.Spec.KubernetesConfig.Service.ServiceType
 	}
-	err = CreateOrUpdateService(cr.Namespace, additionalObjectMetaInfo, redisClusterAsOwner(cr), false, false, additionalServiceType, *cr.Spec.Port)
+	err = CreateOrUpdateService(cr.Namespace, additionalObjectMetaInfo, redisClusterAsOwner(cr), disableMetrics, false, additionalServiceType, *cr.Spec.Port)
 	if err != nil {
 		logger.Error(err, "Cannot create additional service for Redis", "Setup.Type", service.RedisServiceRole)
 		return err

diff --git a/k8sutils/redis-replication.go b/k8sutils/redis-replication.go
@@ -1,30 +1,38 @@
 package k8sutils
 
-import redisv1beta2 "github.com/OT-CONTAINER-KIT/redis-operator/api/v1beta2"
+import (
+	redisv1beta2 "github.com/OT-CONTAINER-KIT/redis-operator/api/v1beta2"
+	"github.com/OT-CONTAINER-KIT/redis-operator/pkg/util"
+	"k8s.io/utils/pointer"
+)
 
 // CreateReplicationService method will create replication service for Redis
 func CreateReplicationService(cr *redisv1beta2.RedisReplication) error {
 	logger := serviceLogger(cr.Namespace, cr.ObjectMeta.Name)
 	labels := getRedisLabels(cr.ObjectMeta.Name, replication, "replication", cr.ObjectMeta.Labels)
-	annotations := generateServiceAnots(cr.ObjectMeta, nil)
-	if cr.Spec.RedisExporter != nil && cr.Spec.RedisExporter.Enabled {
-		enableMetrics = true
+	var epp exporterPortProvider
+	if cr.Spec.RedisExporter != nil {
+		epp = func() (port int, enable bool) {
+			defaultP := pointer.Int(redisExporterPort)
+			return *util.Coalesce(cr.Spec.RedisExporter.Port, defaultP), cr.Spec.RedisExporter.Enabled
+		}
 	} else {
-		enableMetrics = false
+		epp = disableMetrics
 	}
+	annotations := generateServiceAnots(cr.ObjectMeta, nil, epp)
 	additionalServiceAnnotations := map[string]string{}
 	if cr.Spec.KubernetesConfig.Service != nil {
 		additionalServiceAnnotations = cr.Spec.KubernetesConfig.Service.ServiceAnnotations
 	}
 	objectMetaInfo := generateObjectMetaInformation(cr.ObjectMeta.Name, cr.Namespace, labels, annotations)
 	headlessObjectMetaInfo := generateObjectMetaInformation(cr.ObjectMeta.Name+"-headless", cr.Namespace, labels, annotations)
-	additionalObjectMetaInfo := generateObjectMetaInformation(cr.ObjectMeta.Name+"-additional", cr.Namespace, labels, generateServiceAnots(cr.ObjectMeta, additionalServiceAnnotations))
-	err := CreateOrUpdateService(cr.Namespace, headlessObjectMetaInfo, redisReplicationAsOwner(cr), false, true, "ClusterIP", redisPort)
+	additionalObjectMetaInfo := generateObjectMetaInformation(cr.ObjectMeta.Name+"-additional", cr.Namespace, labels, generateServiceAnots(cr.ObjectMeta, additionalServiceAnnotations, epp))
+	err := CreateOrUpdateService(cr.Namespace, headlessObjectMetaInfo, redisReplicationAsOwner(cr), disableMetrics, true, "ClusterIP", redisPort)
 	if err != nil {
 		logger.Error(err, "Cannot create replication headless service for Redis")
 		return err
 	}
-	err = CreateOrUpdateService(cr.Namespace, objectMetaInfo, redisReplicationAsOwner(cr), enableMetrics, false, "ClusterIP", redisPort)
+	err = CreateOrUpdateService(cr.Namespace, objectMetaInfo, redisReplicationAsOwner(cr), epp, false, "ClusterIP", redisPort)
 	if err != nil {
 		logger.Error(err, "Cannot create replication service for Redis")
 		return err
@@ -33,7 +41,7 @@ func CreateReplicationService(cr *redisv1beta2.RedisReplication) error {
 	if cr.Spec.KubernetesConfig.Service != nil {
 		additionalServiceType = cr.Spec.KubernetesConfig.Service.ServiceType
 	}
-	err = CreateOrUpdateService(cr.Namespace, additionalObjectMetaInfo, redisReplicationAsOwner(cr), false, false, additionalServiceType, redisPort)
+	err = CreateOrUpdateService(cr.Namespace, additionalObjectMetaInfo, redisReplicationAsOwner(cr), disableMetrics, false, additionalServiceType, redisPort)
 	if err != nil {
 		logger.Error(err, "Cannot create additional service for Redis Replication")
 		return err

diff --git a/k8sutils/redis-sentinel.go b/k8sutils/redis-sentinel.go
@@ -4,6 +4,8 @@ import (
 	"context"
 	"encoding/json"
 	"errors"
+	"github.com/OT-CONTAINER-KIT/redis-operator/pkg/util"
+	"k8s.io/utils/pointer"
 
 	commonapi "github.com/OT-CONTAINER-KIT/redis-operator/api"
 	redisv1beta2 "github.com/OT-CONTAINER-KIT/redis-operator/api/v1beta2"
@@ -198,29 +200,31 @@ func (service RedisSentinelService) CreateRedisSentinelService(cr *redisv1beta2.
 	serviceName := cr.ObjectMeta.Name + "-" + service.RedisServiceRole
 	logger := serviceLogger(cr.Namespace, serviceName)
 	labels := getRedisLabels(serviceName, sentinel, service.RedisServiceRole, cr.ObjectMeta.Labels)
-	annotations := generateServiceAnots(cr.ObjectMeta, nil)
-
-	if cr.Spec.RedisExporter != nil && cr.Spec.RedisExporter.Enabled {
-		enableMetrics = true
+	var epp exporterPortProvider
+	if cr.Spec.RedisExporter != nil {
+		epp = func() (port int, enable bool) {
+			defaultP := pointer.Int(redisExporterPort)
+			return *util.Coalesce(cr.Spec.RedisExporter.Port, defaultP), cr.Spec.RedisExporter.Enabled
+		}
 	} else {
-		enableMetrics = false
+		epp = disableMetrics
 	}
-
+	annotations := generateServiceAnots(cr.ObjectMeta, nil, epp)
 	additionalServiceAnnotations := map[string]string{}
 	if cr.Spec.KubernetesConfig.Service != nil {
 		additionalServiceAnnotations = cr.Spec.KubernetesConfig.Service.ServiceAnnotations
 	}
 
 	objectMetaInfo := generateObjectMetaInformation(serviceName, cr.Namespace, labels, annotations)
 	headlessObjectMetaInfo := generateObjectMetaInformation(serviceName+"-headless", cr.Namespace, labels, annotations)
-	additionalObjectMetaInfo := generateObjectMetaInformation(serviceName+"-additional", cr.Namespace, labels, generateServiceAnots(cr.ObjectMeta, additionalServiceAnnotations))
+	additionalObjectMetaInfo := generateObjectMetaInformation(serviceName+"-additional", cr.Namespace, labels, generateServiceAnots(cr.ObjectMeta, additionalServiceAnnotations, epp))
 
-	err := CreateOrUpdateService(cr.Namespace, headlessObjectMetaInfo, redisSentinelAsOwner(cr), false, true, "ClusterIP", sentinelPort)
+	err := CreateOrUpdateService(cr.Namespace, headlessObjectMetaInfo, redisSentinelAsOwner(cr), disableMetrics, true, "ClusterIP", sentinelPort)
 	if err != nil {
 		logger.Error(err, "Cannot create headless service for Redis", "Setup.Type", service.RedisServiceRole)
 		return err
 	}
-	err = CreateOrUpdateService(cr.Namespace, objectMetaInfo, redisSentinelAsOwner(cr), enableMetrics, false, "ClusterIP", sentinelPort)
+	err = CreateOrUpdateService(cr.Namespace, objectMetaInfo, redisSentinelAsOwner(cr), epp, false, "ClusterIP", sentinelPort)
 	if err != nil {
 		logger.Error(err, "Cannot create service for Redis", "Setup.Type", service.RedisServiceRole)
 		return err
@@ -230,7 +234,7 @@ func (service RedisSentinelService) CreateRedisSentinelService(cr *redisv1beta2.
 	if cr.Spec.KubernetesConfig.Service != nil {
 		additionalServiceType = cr.Spec.KubernetesConfig.Service.ServiceType
 	}
-	err = CreateOrUpdateService(cr.Namespace, additionalObjectMetaInfo, redisSentinelAsOwner(cr), false, false, additionalServiceType, sentinelPort)
+	err = CreateOrUpdateService(cr.Namespace, additionalObjectMetaInfo, redisSentinelAsOwner(cr), disableMetrics, false, additionalServiceType, sentinelPort)
 	if err != nil {
 		logger.Error(err, "Cannot create additional service for Redis", "Setup.Type", service.RedisServiceRole)
 		return err

diff --git a/k8sutils/redis-standalone.go b/k8sutils/redis-standalone.go
@@ -1,34 +1,42 @@
 package k8sutils
 
-import redisv1beta2 "github.com/OT-CONTAINER-KIT/redis-operator/api/v1beta2"
-
-var (
-	enableMetrics bool
+import (
+	redisv1beta2 "github.com/OT-CONTAINER-KIT/redis-operator/api/v1beta2"
+	"github.com/OT-CONTAINER-KIT/redis-operator/pkg/util"
+	"k8s.io/utils/pointer"
 )
 
+//var (
+//	enableMetrics bool
+//)
+
 // CreateStandaloneService method will create standalone service for Redis
 func CreateStandaloneService(cr *redisv1beta2.Redis) error {
 	logger := serviceLogger(cr.Namespace, cr.ObjectMeta.Name)
 	labels := getRedisLabels(cr.ObjectMeta.Name, standalone, "standalone", cr.ObjectMeta.Labels)
-	annotations := generateServiceAnots(cr.ObjectMeta, nil)
-	if cr.Spec.RedisExporter != nil && cr.Spec.RedisExporter.Enabled {
-		enableMetrics = true
+	var epp exporterPortProvider
+	if cr.Spec.RedisExporter != nil {
+		epp = func() (port int, enable bool) {
+			defaultP := pointer.Int(redisExporterPort)
+			return *util.Coalesce(cr.Spec.RedisExporter.Port, defaultP), cr.Spec.RedisExporter.Enabled
+		}
 	} else {
-		enableMetrics = false
+		epp = disableMetrics
 	}
+	annotations := generateServiceAnots(cr.ObjectMeta, nil, epp)
 	additionalServiceAnnotations := map[string]string{}
 	if cr.Spec.KubernetesConfig.Service != nil {
 		additionalServiceAnnotations = cr.Spec.KubernetesConfig.Service.ServiceAnnotations
 	}
 	objectMetaInfo := generateObjectMetaInformation(cr.ObjectMeta.Name, cr.Namespace, labels, annotations)
 	headlessObjectMetaInfo := generateObjectMetaInformation(cr.ObjectMeta.Name+"-headless", cr.Namespace, labels, annotations)
-	additionalObjectMetaInfo := generateObjectMetaInformation(cr.ObjectMeta.Name+"-additional", cr.Namespace, labels, generateServiceAnots(cr.ObjectMeta, additionalServiceAnnotations))
-	err := CreateOrUpdateService(cr.Namespace, headlessObjectMetaInfo, redisAsOwner(cr), false, true, "ClusterIP", redisPort)
+	additionalObjectMetaInfo := generateObjectMetaInformation(cr.ObjectMeta.Name+"-additional", cr.Namespace, labels, generateServiceAnots(cr.ObjectMeta, additionalServiceAnnotations, epp))
+	err := CreateOrUpdateService(cr.Namespace, headlessObjectMetaInfo, redisAsOwner(cr), disableMetrics, true, "ClusterIP", redisPort)
 	if err != nil {
 		logger.Error(err, "Cannot create standalone headless service for Redis")
 		return err
 	}
-	err = CreateOrUpdateService(cr.Namespace, objectMetaInfo, redisAsOwner(cr), enableMetrics, false, "ClusterIP", redisPort)
+	err = CreateOrUpdateService(cr.Namespace, objectMetaInfo, redisAsOwner(cr), epp, false, "ClusterIP", redisPort)
 	if err != nil {
 		logger.Error(err, "Cannot create standalone service for Redis")
 		return err
@@ -37,7 +45,7 @@ func CreateStandaloneService(cr *redisv1beta2.Redis) error {
 	if cr.Spec.KubernetesConfig.Service != nil {
 		additionalServiceType = cr.Spec.KubernetesConfig.Service.ServiceType
 	}
-	err = CreateOrUpdateService(cr.Namespace, additionalObjectMetaInfo, redisAsOwner(cr), false, false, additionalServiceType, redisPort)
+	err = CreateOrUpdateService(cr.Namespace, additionalObjectMetaInfo, redisAsOwner(cr), disableMetrics, false, additionalServiceType, redisPort)
 	if err != nil {
 		logger.Error(err, "Cannot create additional service for Redis")
 		return err