From 78fb9178c1bed663c095a7d6506d891279370084 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 3 Jan 2025 09:59:36 +0100 Subject: [PATCH 01/11] chore(deps): bump the go-deps group across 1 directory with 2 updates (#5403) Bumps the go-deps group with 2 updates in the / directory: [cloud.google.com/go/storage](https://github.com/googleapis/google-cloud-go) and [github.com/onsi/ginkgo/v2](https://github.com/onsi/ginkgo). Updates `cloud.google.com/go/storage` from 1.48.0 to 1.49.0 - [Release notes](https://github.com/googleapis/google-cloud-go/releases) - [Changelog](https://github.com/googleapis/google-cloud-go/blob/main/CHANGES.md) - [Commits](https://github.com/googleapis/google-cloud-go/compare/spanner/v1.48.0...spanner/v1.49.0) Updates `github.com/onsi/ginkgo/v2` from 2.22.1 to 2.22.2 - [Release notes](https://github.com/onsi/ginkgo/releases) - [Changelog](https://github.com/onsi/ginkgo/blob/master/CHANGELOG.md) - [Commits](https://github.com/onsi/ginkgo/compare/v2.22.1...v2.22.2) --- updated-dependencies: - dependency-name: cloud.google.com/go/storage dependency-type: direct:production update-type: version-update:semver-minor dependency-group: go-deps - dependency-name: github.com/onsi/ginkgo/v2 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: go-deps ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 8 ++++---- go.sum | 16 ++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index fda714b35b..0aaea90474 100644 --- a/go.mod +++ b/go.mod @@ -26,7 +26,7 @@ replace ( require ( cloud.google.com/go/bigquery v1.65.0 cloud.google.com/go/pubsub v1.45.3 - cloud.google.com/go/storage v1.48.0 + cloud.google.com/go/storage v1.49.0 github.com/Azure/azure-storage-blob-go v0.15.0 github.com/ClickHouse/clickhouse-go v1.5.4 github.com/DATA-DOG/go-sqlmock v1.5.2 @@ -65,7 +65,7 @@ require ( github.com/minio/minio-go/v7 v7.0.82 github.com/mitchellh/mapstructure v1.5.0 github.com/olekukonko/tablewriter v0.0.5 - github.com/onsi/ginkgo/v2 v2.22.1 + github.com/onsi/ginkgo/v2 v2.22.2 github.com/onsi/gomega v1.36.2 github.com/ory/dockertest/v3 v3.11.0 github.com/oschwald/maxminddb-golang v1.13.1 @@ -140,7 +140,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.4.0 // indirect github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect github.com/DataDog/zstd v1.5.6 // indirect - github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.24.1 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.25.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.48.1 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.48.1 // indirect github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c // indirect @@ -199,7 +199,7 @@ require ( github.com/docker/go-units v0.5.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/dvsekhvalnov/jose2go v1.7.0 // indirect - github.com/envoyproxy/go-control-plane v0.13.0 // indirect + github.com/envoyproxy/go-control-plane v0.13.1 // indirect github.com/envoyproxy/protoc-gen-validate v1.1.0 // indirect github.com/evanphx/json-patch v5.9.0+incompatible github.com/fatih/color v1.17.0 // indirect diff --git a/go.sum b/go.sum index 9acae8f910..1c2fcd6143 100644 --- a/go.sum +++ b/go.sum @@ -89,8 +89,8 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.12.0/go.mod h1:fFLk2dp2oAhDz8QFKwqrjdJvxSp/W2g7nillojlL5Ho= cloud.google.com/go/storage v1.21.0/go.mod h1:XmRlxkgPjlBONznT2dDUU/5XlpU2OjMnKuqnZI01LAA= -cloud.google.com/go/storage v1.48.0 h1:FhBDHACbVtdPx7S/AbcKujPWiHvfO6F8OXGgCEbB2+o= -cloud.google.com/go/storage v1.48.0/go.mod h1:aFoDYNMAjv67lp+xcuZqjUKv/ctmplzQ3wJgodA7b+M= +cloud.google.com/go/storage v1.49.0 h1:zenOPBOWHCnojRd9aJZAyQXBYqkJkdQS42dxL55CIMw= +cloud.google.com/go/storage v1.49.0/go.mod h1:k1eHhhpLvrPjVGfo0mOUPEJ4Y2+a/Hv5PiwehZI9qGU= cloud.google.com/go/trace v1.0.0/go.mod h1:4iErSByzxkyHWzzlAj63/Gmjz0NH1ASqhJguHpGcr6A= cloud.google.com/go/trace v1.2.0/go.mod h1:Wc8y/uYyOhPy12KEnXG9XGrvfMz5F5SrYecQlbW1rwM= cloud.google.com/go/trace v1.11.2 h1:4ZmaBdL8Ng/ajrgKqY5jfvzqMXbrDcBsUGXOT9aqTtI= @@ -191,8 +191,8 @@ github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ github.com/DataDog/zstd v1.5.6 h1:LbEglqepa/ipmmQJUDnSsfvA8e8IStVcGaFWDuxvGOY= github.com/DataDog/zstd v1.5.6/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/GoogleCloudPlatform/cloudsql-proxy v1.29.0/go.mod h1:spvB9eLJH9dutlbPSRmHvSXXHOwGRyeXh1jVdquA2G8= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.24.1 h1:pB2F2JKCj1Znmp2rwxxt1J0Fg0wezTMgWYk5Mpbi1kg= -github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.24.1/go.mod h1:itPGVDKf9cC/ov4MdvJ2QZ0khw4bfoo9jzwTJlaxy2k= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.25.0 h1:3c8yed4lgqTt+oTQ+JNMDo+F4xprBf+O/il4ZC0nRLw= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.25.0/go.mod h1:obipzmGjfSjam60XLwGfqUkJsfiheAl+TUjG+4yzyPM= github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.48.1 h1:UQ0AhxogsIRZDkElkblfnwjc3IaltCm2HUMvezQaL7s= github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.48.1/go.mod h1:jyqM3eLpJ3IbIFDTKVz2rF9T/xWGW0rIriGwnz8l9Tk= github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.48.1 h1:oTX4vsorBZo/Zdum6OKPA4o7544hm6smoRv1QjpTwGo= @@ -505,8 +505,8 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= -github.com/envoyproxy/go-control-plane v0.13.0 h1:HzkeUz1Knt+3bK+8LG1bxOO/jzWZmdxpwC51i202les= -github.com/envoyproxy/go-control-plane v0.13.0/go.mod h1:GRaKG3dwvFoTg4nj7aXdZnvMg4d7nvT/wl9WgVXn3Q8= +github.com/envoyproxy/go-control-plane v0.13.1 h1:vPfJZCkob6yTMEgS+0TwfTUfbHjfy/6vOJ8hUWX/uXE= +github.com/envoyproxy/go-control-plane v0.13.1/go.mod h1:X45hY0mufo6Fd0KW3rqsGvQMw58jvjymeCzBU3mWyHw= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v1.1.0 h1:tntQDh69XqOCOZsDz0lVJQez/2L6Uu2PdjCQwWCJ3bM= github.com/envoyproxy/protoc-gen-validate v1.1.0/go.mod h1:sXRDRVmzEbkM7CVcM06s9shE/m23dg3wzjl0UWqJ2q4= @@ -1067,8 +1067,8 @@ github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108 github.com/onsi/ginkgo v1.14.2/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= -github.com/onsi/ginkgo/v2 v2.22.1 h1:QW7tbJAUDyVDVOM5dFa7qaybo+CRfR7bemlQUN6Z8aM= -github.com/onsi/ginkgo/v2 v2.22.1/go.mod h1:S6aTpoRsSq2cZOd+pssHAlKW/Q/jZt6cPrPlnj4a1xM= +github.com/onsi/ginkgo/v2 v2.22.2 h1:/3X8Panh8/WwhU/3Ssa6rCKqPLuAkVY2I0RoyDLySlU= +github.com/onsi/ginkgo/v2 v2.22.2/go.mod h1:oeMosUL+8LtarXBHu/c0bx2D/K9zyQ6uX3cTyztHwsk= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc= From e504d75e4bf1d086596c6360788abef28671d551 Mon Sep 17 00:00:00 2001 From: Vamsi Krishna Kandi Date: Mon, 6 Jan 2025 14:13:26 +0530 Subject: [PATCH 02/11] chore: collect stats for reporting event sampler (#5357) --- enterprise/reporting/error_reporting.go | 2 +- .../event_sampler/badger_event_sampler.go | 24 ++++- .../reporting/event_sampler/event_sampler.go | 18 ++-- .../event_sampler/event_sampler_test.go | 89 +++++++++++++++++-- .../in_memory_cache_event_sampler.go | 19 +++- .../event_sampler/stats_collector.go | 53 +++++++++++ enterprise/reporting/reporting.go | 2 +- runner/buckets.go | 4 + 8 files changed, 192 insertions(+), 19 deletions(-) create mode 100644 enterprise/reporting/event_sampler/stats_collector.go diff --git a/enterprise/reporting/error_reporting.go b/enterprise/reporting/error_reporting.go index 6a4c005443..39a478ed4e 100644 --- a/enterprise/reporting/error_reporting.go +++ b/enterprise/reporting/error_reporting.go @@ -124,7 +124,7 @@ func NewErrorDetailReporter( if eventSamplingEnabled.Load() { var err error - eventSampler, err = event_sampler.NewEventSampler(ctx, eventSamplingDuration, eventSamplerType, eventSamplingCardinality, event_sampler.BadgerEventSamplerErrorsPathName, conf, log) + eventSampler, err = event_sampler.NewEventSampler(ctx, eventSamplingDuration, eventSamplerType, eventSamplingCardinality, event_sampler.ErrorsReporting, conf, log, stats) if err != nil { panic(err) } diff --git a/enterprise/reporting/event_sampler/badger_event_sampler.go b/enterprise/reporting/event_sampler/badger_event_sampler.go index ac0e1ba79b..6c74537d95 100644 --- a/enterprise/reporting/event_sampler/badger_event_sampler.go +++ b/enterprise/reporting/event_sampler/badger_event_sampler.go @@ -12,6 +12,7 @@ import ( "github.com/rudderlabs/rudder-go-kit/config" "github.com/rudderlabs/rudder-go-kit/logger" + "github.com/rudderlabs/rudder-go-kit/stats" "github.com/rudderlabs/rudder-server/rruntime" "github.com/rudderlabs/rudder-server/utils/misc" ) @@ -23,6 +24,11 @@ type BadgerEventSampler struct { ctx context.Context cancel context.CancelFunc wg sync.WaitGroup + sc *StatsCollector +} + +func GetPathName(module string) string { + return "/" + module + "-badger" } func DefaultPath(pathName string) (string, error) { @@ -33,8 +39,15 @@ func DefaultPath(pathName string) (string, error) { return fmt.Sprintf(`%v%v`, tmpDirPath, pathName), nil } -func NewBadgerEventSampler(ctx context.Context, pathName string, ttl config.ValueLoader[time.Duration], conf *config.Config, log logger.Logger) (*BadgerEventSampler, error) { - dbPath, err := DefaultPath(pathName) +func NewBadgerEventSampler( + ctx context.Context, + module string, + ttl config.ValueLoader[time.Duration], + conf *config.Config, + log logger.Logger, + stats stats.Stats, +) (*BadgerEventSampler, error) { + dbPath, err := DefaultPath(GetPathName(module)) if err != nil || dbPath == "" { return nil, err } @@ -63,6 +76,7 @@ func NewBadgerEventSampler(ctx context.Context, pathName string, ttl config.Valu ctx: ctx, cancel: cancel, wg: sync.WaitGroup{}, + sc: NewStatsCollector(BadgerTypeEventSampler, module, stats), } if err != nil { @@ -81,6 +95,9 @@ func NewBadgerEventSampler(ctx context.Context, pathName string, ttl config.Valu func (es *BadgerEventSampler) Get(key string) (bool, error) { es.mu.Lock() defer es.mu.Unlock() + start := time.Now() + defer es.sc.RecordGetDuration(start) + es.sc.RecordGet() var found bool @@ -106,6 +123,9 @@ func (es *BadgerEventSampler) Get(key string) (bool, error) { func (es *BadgerEventSampler) Put(key string) error { es.mu.Lock() defer es.mu.Unlock() + start := time.Now() + defer es.sc.RecordPutDuration(start) + es.sc.RecordPut() return es.db.Update(func(txn *badger.Txn) error { entry := badger.NewEntry([]byte(key), []byte{1}).WithTTL(es.ttl.Load()) diff --git a/enterprise/reporting/event_sampler/event_sampler.go b/enterprise/reporting/event_sampler/event_sampler.go index 18bb1d2b2f..80cff6af2a 100644 --- a/enterprise/reporting/event_sampler/event_sampler.go +++ b/enterprise/reporting/event_sampler/event_sampler.go @@ -6,13 +6,14 @@ import ( "github.com/rudderlabs/rudder-go-kit/config" "github.com/rudderlabs/rudder-go-kit/logger" + "github.com/rudderlabs/rudder-go-kit/stats" ) const ( - BadgerTypeEventSampler = "badger" - InMemoryCacheTypeEventSampler = "in_memory_cache" - BadgerEventSamplerMetricsPathName = "/metrics-reporting-badger" - BadgerEventSamplerErrorsPathName = "/errors-reporting-badger" + BadgerTypeEventSampler = "badger" + InMemoryCacheTypeEventSampler = "in_memory_cache" + MetricsReporting = "metrics-reporting" + ErrorsReporting = "errors-reporting" ) //go:generate mockgen -destination=../../../mocks/enterprise/reporting/event_sampler/mock_event_sampler.go -package=mocks github.com/rudderlabs/rudder-server/enterprise/reporting/event_sampler EventSampler @@ -27,18 +28,19 @@ func NewEventSampler( ttl config.ValueLoader[time.Duration], eventSamplerType config.ValueLoader[string], eventSamplingCardinality config.ValueLoader[int], - badgerDBPath string, + module string, conf *config.Config, log logger.Logger, + stats stats.Stats, ) (es EventSampler, err error) { switch eventSamplerType.Load() { case BadgerTypeEventSampler: - es, err = NewBadgerEventSampler(ctx, badgerDBPath, ttl, conf, log) + es, err = NewBadgerEventSampler(ctx, module, ttl, conf, log, stats) case InMemoryCacheTypeEventSampler: - es, err = NewInMemoryCacheEventSampler(ctx, ttl, eventSamplingCardinality) + es, err = NewInMemoryCacheEventSampler(ctx, module, ttl, eventSamplingCardinality, stats) default: log.Warnf("invalid event sampler type: %s. Using default badger event sampler", eventSamplerType.Load()) - es, err = NewBadgerEventSampler(ctx, badgerDBPath, ttl, conf, log) + es, err = NewBadgerEventSampler(ctx, module, ttl, conf, log, stats) } if err != nil { diff --git a/enterprise/reporting/event_sampler/event_sampler_test.go b/enterprise/reporting/event_sampler/event_sampler_test.go index ae4dc22534..a8e27ddda6 100644 --- a/enterprise/reporting/event_sampler/event_sampler_test.go +++ b/enterprise/reporting/event_sampler/event_sampler_test.go @@ -11,6 +11,8 @@ import ( "github.com/rudderlabs/rudder-go-kit/config" "github.com/rudderlabs/rudder-go-kit/logger" + "github.com/rudderlabs/rudder-go-kit/stats" + "github.com/rudderlabs/rudder-go-kit/stats/memstats" ) func TestBadger(t *testing.T) { @@ -23,15 +25,40 @@ func TestBadger(t *testing.T) { t.Run("should put and get keys", func(t *testing.T) { assert.Equal(t, 3000*time.Millisecond, ttl.Load()) - es, _ := NewEventSampler(ctx, ttl, eventSamplerType, eventSamplingCardinality, BadgerEventSamplerMetricsPathName, conf, log) + statsStore, err := memstats.New() + require.NoError(t, err) + es, _ := NewEventSampler(ctx, ttl, eventSamplerType, eventSamplingCardinality, MetricsReporting, conf, log, statsStore) _ = es.Put("key1") _ = es.Put("key2") _ = es.Put("key3") + + require.Equal(t, statsStore.Get(StatReportingEventSamplerRequestsTotal, map[string]string{ + "type": BadgerTypeEventSampler, + "module": MetricsReporting, + "operation": "put", + }).LastValue(), float64(3)) + require.Equal(t, len(statsStore.Get(StatReportingEventSamplerRequestDuration, map[string]string{ + "type": BadgerTypeEventSampler, + "module": MetricsReporting, + "operation": "put", + }).Durations()), 3) + val1, _ := es.Get("key1") val2, _ := es.Get("key2") val3, _ := es.Get("key3") val4, _ := es.Get("key4") + require.Equal(t, statsStore.Get(StatReportingEventSamplerRequestsTotal, map[string]string{ + "type": BadgerTypeEventSampler, + "module": MetricsReporting, + "operation": "get", + }).LastValue(), float64(4)) + require.Equal(t, len(statsStore.Get(StatReportingEventSamplerRequestDuration, map[string]string{ + "type": BadgerTypeEventSampler, + "module": MetricsReporting, + "operation": "get", + }).Durations()), 4) + assert.True(t, val1, "Expected key1 to be present") assert.True(t, val2, "Expected key2 to be present") assert.True(t, val3, "Expected key3 to be present") @@ -43,7 +70,7 @@ func TestBadger(t *testing.T) { conf.Set("Reporting.eventSampling.durationInMinutes", 100) assert.Equal(t, 100*time.Millisecond, ttl.Load()) - es, _ := NewEventSampler(ctx, ttl, eventSamplerType, eventSamplingCardinality, BadgerEventSamplerMetricsPathName, conf, log) + es, _ := NewEventSampler(ctx, ttl, eventSamplerType, eventSamplingCardinality, MetricsReporting, conf, log, stats.NOP) defer es.Close() _ = es.Put("key1") @@ -65,15 +92,40 @@ func TestInMemoryCache(t *testing.T) { t.Run("should put and get keys", func(t *testing.T) { assert.Equal(t, 3000*time.Millisecond, ttl.Load()) - es, _ := NewEventSampler(ctx, ttl, eventSamplerType, eventSamplingCardinality, BadgerEventSamplerMetricsPathName, conf, log) + statsStore, err := memstats.New() + require.NoError(t, err) + es, _ := NewEventSampler(ctx, ttl, eventSamplerType, eventSamplingCardinality, MetricsReporting, conf, log, statsStore) _ = es.Put("key1") _ = es.Put("key2") _ = es.Put("key3") + + require.Equal(t, statsStore.Get(StatReportingEventSamplerRequestsTotal, map[string]string{ + "type": InMemoryCacheTypeEventSampler, + "module": MetricsReporting, + "operation": "put", + }).LastValue(), float64(3)) + require.Equal(t, len(statsStore.Get(StatReportingEventSamplerRequestDuration, map[string]string{ + "type": InMemoryCacheTypeEventSampler, + "module": MetricsReporting, + "operation": "put", + }).Durations()), 3) + val1, _ := es.Get("key1") val2, _ := es.Get("key2") val3, _ := es.Get("key3") val4, _ := es.Get("key4") + require.Equal(t, statsStore.Get(StatReportingEventSamplerRequestsTotal, map[string]string{ + "type": InMemoryCacheTypeEventSampler, + "module": MetricsReporting, + "operation": "get", + }).LastValue(), float64(4)) + require.Equal(t, len(statsStore.Get(StatReportingEventSamplerRequestDuration, map[string]string{ + "type": InMemoryCacheTypeEventSampler, + "module": MetricsReporting, + "operation": "get", + }).Durations()), 4) + assert.True(t, val1, "Expected key1 to be present") assert.True(t, val2, "Expected key2 to be present") assert.True(t, val3, "Expected key3 to be present") @@ -83,7 +135,7 @@ func TestInMemoryCache(t *testing.T) { t.Run("should not get evicted keys", func(t *testing.T) { conf.Set("Reporting.eventSampling.durationInMinutes", 100) assert.Equal(t, 100*time.Millisecond, ttl.Load()) - es, _ := NewEventSampler(ctx, ttl, eventSamplerType, eventSamplingCardinality, BadgerEventSamplerMetricsPathName, conf, log) + es, _ := NewEventSampler(ctx, ttl, eventSamplerType, eventSamplingCardinality, MetricsReporting, conf, log, stats.NOP) _ = es.Put("key1") require.Eventually(t, func() bool { @@ -95,19 +147,43 @@ func TestInMemoryCache(t *testing.T) { t.Run("should not add keys if length exceeds", func(t *testing.T) { conf.Set("Reporting.eventSampling.durationInMinutes", 3000) assert.Equal(t, 3000*time.Millisecond, ttl.Load()) - es, _ := NewEventSampler(ctx, ttl, eventSamplerType, eventSamplingCardinality, BadgerEventSamplerMetricsPathName, conf, log) + statsStore, err := memstats.New() + require.NoError(t, err) + es, _ := NewEventSampler(ctx, ttl, eventSamplerType, eventSamplingCardinality, MetricsReporting, conf, log, statsStore) _ = es.Put("key1") _ = es.Put("key2") _ = es.Put("key3") _ = es.Put("key4") _ = es.Put("key5") + require.Equal(t, statsStore.Get(StatReportingEventSamplerRequestsTotal, map[string]string{ + "type": InMemoryCacheTypeEventSampler, + "module": MetricsReporting, + "operation": "put", + }).LastValue(), float64(3)) + require.Equal(t, len(statsStore.Get(StatReportingEventSamplerRequestDuration, map[string]string{ + "type": InMemoryCacheTypeEventSampler, + "module": MetricsReporting, + "operation": "put", + }).Durations()), 3) + val1, _ := es.Get("key1") val2, _ := es.Get("key2") val3, _ := es.Get("key3") val4, _ := es.Get("key4") val5, _ := es.Get("key5") + require.Equal(t, statsStore.Get(StatReportingEventSamplerRequestsTotal, map[string]string{ + "type": InMemoryCacheTypeEventSampler, + "module": MetricsReporting, + "operation": "get", + }).LastValue(), float64(5)) + require.Equal(t, len(statsStore.Get(StatReportingEventSamplerRequestDuration, map[string]string{ + "type": InMemoryCacheTypeEventSampler, + "module": MetricsReporting, + "operation": "get", + }).Durations()), 5) + assert.True(t, val1, "Expected key1 to be present") assert.True(t, val2, "Expected key2 to be present") assert.True(t, val3, "Expected key3 to be present") @@ -147,9 +223,10 @@ func BenchmarkEventSampler(b *testing.B) { ttl, eventSamplerType, eventSamplingCardinality, - BadgerEventSamplerMetricsPathName, + MetricsReporting, conf, log, + stats.NOP, ) require.NoError(b, err) diff --git a/enterprise/reporting/event_sampler/in_memory_cache_event_sampler.go b/enterprise/reporting/event_sampler/in_memory_cache_event_sampler.go index a8e413fc8e..509ae6b632 100644 --- a/enterprise/reporting/event_sampler/in_memory_cache_event_sampler.go +++ b/enterprise/reporting/event_sampler/in_memory_cache_event_sampler.go @@ -6,6 +6,7 @@ import ( "github.com/rudderlabs/rudder-go-kit/cachettl" "github.com/rudderlabs/rudder-go-kit/config" + "github.com/rudderlabs/rudder-go-kit/stats" ) type InMemoryCacheEventSampler struct { @@ -15,9 +16,16 @@ type InMemoryCacheEventSampler struct { ttl config.ValueLoader[time.Duration] limit config.ValueLoader[int] length int + sc *StatsCollector } -func NewInMemoryCacheEventSampler(ctx context.Context, ttl config.ValueLoader[time.Duration], limit config.ValueLoader[int]) (*InMemoryCacheEventSampler, error) { +func NewInMemoryCacheEventSampler( + ctx context.Context, + module string, + ttl config.ValueLoader[time.Duration], + limit config.ValueLoader[int], + stats stats.Stats, +) (*InMemoryCacheEventSampler, error) { c := cachettl.New[string, bool](cachettl.WithNoRefreshTTL) ctx, cancel := context.WithCancel(ctx) @@ -28,6 +36,7 @@ func NewInMemoryCacheEventSampler(ctx context.Context, ttl config.ValueLoader[ti ttl: ttl, limit: limit, length: 0, + sc: NewStatsCollector(InMemoryCacheTypeEventSampler, module, stats), } es.cache.OnEvicted(func(key string, value bool) { @@ -38,6 +47,10 @@ func NewInMemoryCacheEventSampler(ctx context.Context, ttl config.ValueLoader[ti } func (es *InMemoryCacheEventSampler) Get(key string) (bool, error) { + start := time.Now() + defer es.sc.RecordGetDuration(start) + es.sc.RecordGet() + value := es.cache.Get(key) return value, nil } @@ -47,6 +60,10 @@ func (es *InMemoryCacheEventSampler) Put(key string) error { return nil } + start := time.Now() + defer es.sc.RecordPutDuration(start) + es.sc.RecordPut() + es.cache.Put(key, true, es.ttl.Load()) es.length++ return nil diff --git a/enterprise/reporting/event_sampler/stats_collector.go b/enterprise/reporting/event_sampler/stats_collector.go new file mode 100644 index 0000000000..48ab49e853 --- /dev/null +++ b/enterprise/reporting/event_sampler/stats_collector.go @@ -0,0 +1,53 @@ +package event_sampler + +import ( + "time" + + "github.com/rudderlabs/rudder-go-kit/stats" +) + +const ( + StatReportingEventSamplerRequestsTotal = "reporting_event_sampler_requests_total" + StatReportingEventSamplerRequestDuration = "reporting_event_sampler_request_duration_seconds" +) + +type StatsCollector struct { + stats stats.Stats + getCounter stats.Measurement + putCounter stats.Measurement + getDuration stats.Measurement + putDuration stats.Measurement +} + +func NewStatsCollector(eventSamplerType, module string, statsFactory stats.Stats) *StatsCollector { + getRequestTags := getTags(eventSamplerType, module, "get") + putRequestTags := getTags(eventSamplerType, module, "put") + + return &StatsCollector{ + stats: statsFactory, + getCounter: statsFactory.NewTaggedStat(StatReportingEventSamplerRequestsTotal, stats.CountType, getRequestTags), + putCounter: statsFactory.NewTaggedStat(StatReportingEventSamplerRequestsTotal, stats.CountType, putRequestTags), + getDuration: statsFactory.NewTaggedStat(StatReportingEventSamplerRequestDuration, stats.TimerType, getRequestTags), + putDuration: statsFactory.NewTaggedStat(StatReportingEventSamplerRequestDuration, stats.TimerType, putRequestTags), + } +} + +func (sc *StatsCollector) RecordGet() { + sc.getCounter.Increment() +} + +func (sc *StatsCollector) RecordPut() { + sc.putCounter.Increment() +} + +func (sc *StatsCollector) RecordGetDuration(start time.Time) { + sc.getDuration.SendTiming(time.Since(start)) +} + +func (sc *StatsCollector) RecordPutDuration(start time.Time) { + sc.putDuration.SendTiming(time.Since(start)) +} + +func getTags(eventSamplerType, module, operation string) stats.Tags { + return stats.Tags{"type": eventSamplerType, "module": module, "operation": operation} +} diff --git a/enterprise/reporting/reporting.go b/enterprise/reporting/reporting.go index 983289e89f..a2a1e7e5ed 100644 --- a/enterprise/reporting/reporting.go +++ b/enterprise/reporting/reporting.go @@ -108,7 +108,7 @@ func NewDefaultReporter(ctx context.Context, conf *config.Config, log logger.Log if eventSamplingEnabled.Load() { var err error - eventSampler, err = event_sampler.NewEventSampler(ctx, eventSamplingDuration, eventSamplerType, eventSamplingCardinality, event_sampler.BadgerEventSamplerMetricsPathName, conf, log) + eventSampler, err = event_sampler.NewEventSampler(ctx, eventSamplingDuration, eventSamplerType, eventSamplingCardinality, event_sampler.MetricsReporting, conf, log, stats) if err != nil { panic(err) } diff --git a/runner/buckets.go b/runner/buckets.go index 7e2272c24f..70609719f1 100644 --- a/runner/buckets.go +++ b/runner/buckets.go @@ -50,6 +50,10 @@ var ( "reporting_client_get_aggregated_reports_count": { 1, 5, 10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10000, }, + "reporting_event_sampler_request_duration_seconds": { + // 1ms, 5ms, 10ms, 25ms, 50ms, 100ms + 0.001, 0.005, 0.01, 0.025, 0.05, 0.1, + }, "csv_file_size": { float64(10 * bytesize.B), float64(100 * bytesize.B), float64(1 * bytesize.KB), float64(10 * bytesize.KB), float64(100 * bytesize.KB), From d29eb141dd2fe46e1c3f66c39433649fd44353bd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Jan 2025 15:25:28 +0530 Subject: [PATCH 03/11] chore(deps): bump the go-deps group with 2 updates (#5412) --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 0aaea90474..bdb6d3ce9f 100644 --- a/go.mod +++ b/go.mod @@ -105,7 +105,7 @@ require ( go.uber.org/goleak v1.3.0 go.uber.org/mock v0.5.0 golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 - golang.org/x/oauth2 v0.24.0 + golang.org/x/oauth2 v0.25.0 golang.org/x/sync v0.10.0 google.golang.org/api v0.214.0 google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576 @@ -344,7 +344,7 @@ require ( golang.org/x/sys v0.28.0 // indirect golang.org/x/term v0.27.0 // indirect golang.org/x/text v0.21.0 // indirect - golang.org/x/time v0.8.0 + golang.org/x/time v0.9.0 golang.org/x/tools v0.28.0 // indirect golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect google.golang.org/genproto v0.0.0-20241118233622-e639e219e697 // indirect diff --git a/go.sum b/go.sum index 1c2fcd6143..a761025f52 100644 --- a/go.sum +++ b/go.sum @@ -1518,8 +1518,8 @@ golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/oauth2 v0.24.0 h1:KTBBxWqUa0ykRPLtV69rRto9TLXcqYkeswu48x/gvNE= -golang.org/x/oauth2 v0.24.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/oauth2 v0.25.0 h1:CY4y7XT9v0cRI9oupztF8AgiIu99L/ksR/Xp/6jrZ70= +golang.org/x/oauth2 v0.25.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1637,8 +1637,8 @@ golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20220224211638-0e9765cccd65/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.8.0 h1:9i3RxcPv3PZnitoVGMPDKZSq1xW1gK1Xy3ArNOGZfEg= -golang.org/x/time v0.8.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= +golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= From a801e3dabf65da618b8e18ee575d6e1c8d1f503b Mon Sep 17 00:00:00 2001 From: Sankeerth Date: Mon, 6 Jan 2025 17:16:37 +0530 Subject: [PATCH 04/11] fix: invalid error response handling during oauth refresh flow (#5401) * fix: invalid error response handling during oauth refresh flow * chore: adding more tests to increase coverage --------- Co-authored-by: Sai Sankeerth --- services/oauth/v2/common/constants.go | 9 +- services/oauth/v2/oauth.go | 31 ++++-- services/oauth/v2/oauth_test.go | 153 ++++++++++++++++++++++++++ 3 files changed, 179 insertions(+), 14 deletions(-) diff --git a/services/oauth/v2/common/constants.go b/services/oauth/v2/common/constants.go index a655a46b44..38c680923f 100644 --- a/services/oauth/v2/common/constants.go +++ b/services/oauth/v2/common/constants.go @@ -5,10 +5,11 @@ const ( // CategoryAuthStatusInactive Identifier to be sent from destination(during transformation/delivery) CategoryAuthStatusInactive = "AUTH_STATUS_INACTIVE" // RefTokenInvalidGrant Identifier for invalid_grant or access_denied errors(during refreshing the token) - RefTokenInvalidGrant = "ref_token_invalid_grant" - TimeOutError = "timeout" - NetworkError = "network_error" - None = "none" + RefTokenInvalidGrant = "ref_token_invalid_grant" + RefTokenInvalidResponse = "INVALID_REFRESH_RESPONSE" + TimeOutError = "timeout" + NetworkError = "network_error" + None = "none" DestKey ContextKey = "destination" SecretKey ContextKey = "secret" diff --git a/services/oauth/v2/oauth.go b/services/oauth/v2/oauth.go index 1be55950a0..d06c42baad 100644 --- a/services/oauth/v2/oauth.go +++ b/services/oauth/v2/oauth.go @@ -330,17 +330,28 @@ func (h *OAuthHandler) GetRefreshTokenErrResp(response string, accountSecret *Ac h.Logger.Debugn("Failed with error response", logger.NewErrorField(err)) message = fmt.Sprintf("Unmarshal of response unsuccessful: %v", response) errorType = "unmarshallableResponse" - } else if gjson.Get(response, "body.code").String() == common.RefTokenInvalidGrant { - // User (or) AccessToken (or) RefreshToken has been revoked - bodyMsg := gjson.Get(response, "body.message").String() - if bodyMsg == "" { - // Default message - h.Logger.Debugn("Unable to get body.message", logger.NewStringField("Response", response)) - message = ErrPermissionOrTokenRevoked.Error() - } else { - message = bodyMsg + } else { + code := gjson.Get(response, "body.code").String() + switch code { + case common.RefTokenInvalidGrant: + // User (or) AccessToken (or) RefreshToken has been revoked + bodyMsg := gjson.Get(response, "body.message").String() + if bodyMsg == "" { + // Default message + h.Logger.Debugn("Unable to get body.message", logger.NewStringField("Response", response)) + message = ErrPermissionOrTokenRevoked.Error() + } else { + message = bodyMsg + } + errorType = common.RefTokenInvalidGrant + case common.RefTokenInvalidResponse: + errorType = code + msg := gjson.Get(response, "body.message").String() + if msg == "" { + msg = "invalid refresh response" + } + message = msg } - errorType = common.RefTokenInvalidGrant } return errorType, message } diff --git a/services/oauth/v2/oauth_test.go b/services/oauth/v2/oauth_test.go index 26182f3133..604724f9ee 100644 --- a/services/oauth/v2/oauth_test.go +++ b/services/oauth/v2/oauth_test.go @@ -1,7 +1,9 @@ package v2_test import ( + "bytes" "fmt" + "io" "math/rand" "net" "net/http" @@ -576,6 +578,157 @@ var _ = Describe("Oauth", func() { Expect(response).To(Equal(expectedResponse)) Expect(err).To(MatchError(fmt.Errorf("error occurred while fetching/refreshing account info from CP: mock mock 127.0.0.1:1234->127.0.0.1:12340: read: connection timed out"))) }) + + It("refreshToken function call when stored cache is same as provided secret and cpApiCall returns a failed response because of faulty implementation in some downstream service", func() { + refreshTokenParams := &v2.RefreshTokenParams{ + AccountID: "123", + WorkspaceID: "456", + DestDefName: "testDest", + Destination: Destination, + Secret: []byte(`{"access_token":"storedAccessToken","refresh_token":"dummyRefreshToken","developer_token":"dummyDeveloperToken"}`), + } + + ctrl := gomock.NewController(GinkgoT()) + mockHttpClient := mockhttpclient.NewMockHttpClient(ctrl) + cpResponseString := `{ + "status": 500, + "code": "INVALID_REFRESH_RESPONSE", + "body": { + "code": "INVALID_REFRESH_RESPONSE", + "message": "Missing required token fields in refresh response" + }, + "access_token":"storedAccessToken", + "refresh_token":"dummyRefreshToken", + "developer_token":"dummyDeveloperToken" + }` + mockHttpClient.EXPECT().Do(gomock.Any()).Return(&http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(bytes.NewBufferString(cpResponseString)), + }, nil) + cpConnector := controlplane.NewConnector( + config.Default, + controlplane.WithClient(mockHttpClient), + controlplane.WithStats(stats.Default), + ) + + mockTokenProvider := mock_oauthV2.NewMockTokenProvider(ctrl) + mockTokenProvider.EXPECT().Identity().Return(&testutils.BasicAuthMock{}) + oauthHandler := v2.NewOAuthHandler(mockTokenProvider, + v2.WithCache(v2.NewCache()), + v2.WithLocker(kitsync.NewPartitionRWLocker()), + v2.WithStats(stats.Default), + v2.WithLogger(logger.NewLogger().Child("MockOAuthHandler")), + v2.WithCpConnector(cpConnector), + ) + statusCode, response, err := oauthHandler.RefreshToken(refreshTokenParams) + Expect(statusCode).To(Equal(http.StatusInternalServerError)) + expectedResponse := &v2.AuthResponse{ + Err: "INVALID_REFRESH_RESPONSE", + ErrorMessage: "Missing required token fields in refresh response", + } + Expect(response).To(Equal(expectedResponse)) + Expect(err).To(MatchError(fmt.Errorf("error occurred while fetching/refreshing account info from CP: Missing required token fields in refresh response"))) + }) + + It("refreshToken function call when stored cache is same as provided secret and cpApiCall returns a failed response because of invalid refresh response without message", func() { + refreshTokenParams := &v2.RefreshTokenParams{ + AccountID: "123", + WorkspaceID: "456", + DestDefName: "testDest", + Destination: Destination, + Secret: []byte(`{"access_token":"storedAccessToken","refresh_token":"dummyRefreshToken","developer_token":"dummyDeveloperToken"}`), + } + + ctrl := gomock.NewController(GinkgoT()) + mockHttpClient := mockhttpclient.NewMockHttpClient(ctrl) + cpResponseString := `{ + "status": 500, + "code": "INVALID_REFRESH_RESPONSE", + "body": { + "code": "INVALID_REFRESH_RESPONSE" + }, + "access_token":"storedAccessToken", + "refresh_token":"dummyRefreshToken", + "developer_token":"dummyDeveloperToken" + }` + mockHttpClient.EXPECT().Do(gomock.Any()).Return(&http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(bytes.NewBufferString(cpResponseString)), + }, nil) + cpConnector := controlplane.NewConnector( + config.Default, + controlplane.WithClient(mockHttpClient), + controlplane.WithStats(stats.Default), + ) + + mockTokenProvider := mock_oauthV2.NewMockTokenProvider(ctrl) + mockTokenProvider.EXPECT().Identity().Return(&testutils.BasicAuthMock{}) + oauthHandler := v2.NewOAuthHandler(mockTokenProvider, + v2.WithCache(v2.NewCache()), + v2.WithLocker(kitsync.NewPartitionRWLocker()), + v2.WithStats(stats.Default), + v2.WithLogger(logger.NewLogger().Child("MockOAuthHandler")), + v2.WithCpConnector(cpConnector), + ) + statusCode, response, err := oauthHandler.RefreshToken(refreshTokenParams) + Expect(statusCode).To(Equal(http.StatusInternalServerError)) + expectedResponse := &v2.AuthResponse{ + Err: "INVALID_REFRESH_RESPONSE", + ErrorMessage: "invalid refresh response", + } + Expect(response).To(Equal(expectedResponse)) + Expect(err).To(MatchError(fmt.Errorf("error occurred while fetching/refreshing account info from CP: invalid refresh response"))) + }) + + It("refreshToken function call when stored cache is same as provided secret and cpApiCall returns a failed response because of invalid refresh response without message", func() { + refreshTokenParams := &v2.RefreshTokenParams{ + AccountID: "123", + WorkspaceID: "456", + DestDefName: "testDest", + Destination: Destination, + Secret: []byte(`{"access_token":"storedAccessToken","refresh_token":"dummyRefreshToken","developer_token":"dummyDeveloperToken"}`), + } + + ctrl := gomock.NewController(GinkgoT()) + mockHttpClient := mockhttpclient.NewMockHttpClient(ctrl) + cpResponseString := fmt.Sprintf(`{ + "status": 500, + "code": "%[1]s", + "body": { + "code": "%[1]s" + }, + "access_token":"storedAccessToken", + "refresh_token":"dummyRefreshToken", + "developer_token":"dummyDeveloperToken" + }`, common.RefTokenInvalidGrant) + mockHttpClient.EXPECT().Do(gomock.Any()).Return(&http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(bytes.NewBufferString(cpResponseString)), + }, nil) + cpConnector := controlplane.NewConnector( + config.Default, + controlplane.WithClient(mockHttpClient), + controlplane.WithStats(stats.Default), + ) + + mockTokenProvider := mock_oauthV2.NewMockTokenProvider(ctrl) + mockTokenProvider.EXPECT().Identity().Return(&testutils.BasicAuthMock{}) + oauthHandler := v2.NewOAuthHandler(mockTokenProvider, + v2.WithCache(v2.NewCache()), + v2.WithLocker(kitsync.NewPartitionRWLocker()), + v2.WithStats(stats.Default), + v2.WithLogger(logger.NewLogger().Child("MockOAuthHandler")), + v2.WithCpConnector(cpConnector), + ) + statusCode, response, err := oauthHandler.RefreshToken(refreshTokenParams) + Expect(statusCode).To(Equal(http.StatusBadRequest)) + expectedResponse := &v2.AuthResponse{ + Err: common.RefTokenInvalidGrant, + ErrorMessage: v2.ErrPermissionOrTokenRevoked.Error(), + } + Expect(response).To(Equal(expectedResponse)) + Expect(err).To(MatchError(fmt.Errorf("invalid grant"))) + }) }) Describe("Test AuthStatusToggle function", func() { From de3e87c83f4cb0adf6fd4dff0437fbcd7bcd2855 Mon Sep 17 00:00:00 2001 From: Leonidas Vrachnis Date: Mon, 6 Jan 2025 13:26:42 +0100 Subject: [PATCH 05/11] feat: client side load balancing for user transformations (#5375) --- go.mod | 1 + go.sum | 2 + processor/transformer/transformer.go | 70 +- processor/transformer/transformer_test.go | 1216 +++++++++++---------- utils/sysUtils/httpclient.go | 35 + 5 files changed, 705 insertions(+), 619 deletions(-) diff --git a/go.mod b/go.mod index 0aaea90474..5383cac5d5 100644 --- a/go.mod +++ b/go.mod @@ -35,6 +35,7 @@ require ( github.com/apache/pulsar-client-go v0.14.0 github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de github.com/aws/aws-sdk-go v1.55.5 + github.com/bufbuild/httplb v0.3.0 github.com/cenkalti/backoff v2.2.1+incompatible github.com/cenkalti/backoff/v4 v4.3.0 github.com/confluentinc/confluent-kafka-go/v2 v2.6.1 diff --git a/go.sum b/go.sum index 1c2fcd6143..2281cc09b3 100644 --- a/go.sum +++ b/go.sum @@ -357,6 +357,8 @@ github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdb github.com/bsm/gomega v1.26.0/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= +github.com/bufbuild/httplb v0.3.0 h1:sCMPD+89ydD3atcVareDsiv/kUT+pLHolENMoCGZJV8= +github.com/bufbuild/httplb v0.3.0/go.mod h1:qDNs7dSFxIhKi/DA/rCCPVzbQfHs1JVxPMl9EvrbL4Q= github.com/buger/goterm v1.0.4 h1:Z9YvGmOih81P0FbVtEYTFF6YsSgxSUKEhf/f9bTMXbY= github.com/buger/goterm v1.0.4/go.mod h1:HiFWV3xnkolgrBV3mY8m0X0Pumt4zg4QhbdOzQtB8tE= github.com/bytedance/sonic v1.11.6/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4= diff --git a/processor/transformer/transformer.go b/processor/transformer/transformer.go index 8af9ec60e1..57d51d977f 100644 --- a/processor/transformer/transformer.go +++ b/processor/transformer/transformer.go @@ -7,6 +7,7 @@ import ( "context" "fmt" "io" + "net" "net/http" "os" "runtime/trace" @@ -15,6 +16,8 @@ import ( "sync" "time" + "github.com/bufbuild/httplb" + "github.com/bufbuild/httplb/resolver" "github.com/cenkalti/backoff" jsoniter "github.com/json-iterator/go" "github.com/samber/lo" @@ -26,6 +29,7 @@ import ( backendconfig "github.com/rudderlabs/rudder-server/backend-config" "github.com/rudderlabs/rudder-server/processor/integrations" "github.com/rudderlabs/rudder-server/utils/httputil" + "github.com/rudderlabs/rudder-server/utils/sysUtils" "github.com/rudderlabs/rudder-server/utils/types" warehouseutils "github.com/rudderlabs/rudder-server/warehouse/utils" ) @@ -136,9 +140,9 @@ type Response struct { type Opt func(*handle) -func WithClient(client *http.Client) Opt { +func WithClient(client HTTPDoer) Opt { return func(s *handle) { - s.client = client + s.httpClient = client } } @@ -149,6 +153,10 @@ type Transformer interface { Validate(ctx context.Context, clientEvents []TransformerEvent, batchSize int) Response } +type HTTPDoer interface { + Do(req *http.Request) (*http.Response, error) +} + // handle is the handle for this class type handle struct { sentStat stats.Measurement @@ -159,7 +167,7 @@ type handle struct { logger logger.Logger stat stats.Stats - client *http.Client + httpClient HTTPDoer guardConcurrency chan struct{} @@ -211,16 +219,41 @@ func NewTransformer(conf *config.Config, log logger.Logger, stat stats.Stats, op trans.guardConcurrency = make(chan struct{}, trans.config.maxConcurrency) - if trans.client == nil { - trans.client = &http.Client{ - Transport: &http.Transport{ - DisableKeepAlives: trans.config.disableKeepAlives, - MaxConnsPerHost: trans.config.maxHTTPConnections, - MaxIdleConnsPerHost: trans.config.maxHTTPIdleConnections, - IdleConnTimeout: trans.config.maxIdleConnDuration, - }, - Timeout: trans.config.timeoutDuration, - } + clientType := conf.GetString("Transformer.Client.type", "stdlib") + + transport := &http.Transport{ + DisableKeepAlives: trans.config.disableKeepAlives, + MaxConnsPerHost: trans.config.maxHTTPConnections, + MaxIdleConnsPerHost: trans.config.maxHTTPIdleConnections, + IdleConnTimeout: trans.config.maxIdleConnDuration, + } + client := &http.Client{ + Transport: transport, + Timeout: trans.config.timeoutDuration, + } + + switch clientType { + case "stdlib": + trans.httpClient = client + case "recycled": + trans.httpClient = sysUtils.NewRecycledHTTPClient(func() *http.Client { + return client + }, config.GetDuration("Transformer.Client.ttl", 120, time.Second)) + case "httplb": + trans.httpClient = httplb.NewClient( + httplb.WithTransport("http", &HTTPLBTransport{ + Transport: transport, + }), + httplb.WithResolver( + resolver.NewDNSResolver( + net.DefaultResolver, + resolver.PreferIPv6, + config.GetDuration("Transformer.Client.ttl", 120, time.Second), // TTL value + ), + ), + ) + default: + panic(fmt.Sprintf("unknown transformer client type: %s", clientType)) } for _, opt := range opts { @@ -245,6 +278,14 @@ func (trans *handle) Validate(ctx context.Context, clientEvents []TransformerEve return trans.transform(ctx, clientEvents, trans.trackingPlanValidationURL(), batchSize, trackingPlanValidationStage) } +type HTTPLBTransport struct { + *http.Transport +} + +func (t *HTTPLBTransport) NewRoundTripper(scheme, target string, config httplb.TransportConfig) httplb.RoundTripperResult { + return httplb.RoundTripperResult{RoundTripper: t.Transport, Close: t.CloseIdleConnections} +} + func (trans *handle) transform( ctx context.Context, clientEvents []TransformerEvent, @@ -474,7 +515,7 @@ func (trans *handle) doPost(ctx context.Context, rawJSON []byte, url, stage stri // Header to let transformer know that the client understands event filter code req.Header.Set("X-Feature-Filter-Code", "?1") - resp, reqErr = trans.client.Do(req) + resp, reqErr = trans.httpClient.Do(req) }) trans.stat.NewTaggedStat("processor.transformer_request_time", stats.TimerType, tags).SendTiming(time.Since(requestStartTime)) if reqErr != nil { @@ -495,7 +536,6 @@ func (trans *handle) doPost(ctx context.Context, rawJSON []byte, url, stage stri retryCount++ trans.logger.Warnn( "JS HTTP connection error", - logger.NewStringField("URL", url), logger.NewErrorField(err), logger.NewIntField("attempts", int64(retryCount)), ) diff --git a/processor/transformer/transformer_test.go b/processor/transformer/transformer_test.go index ceef9540c9..a920831a99 100644 --- a/processor/transformer/transformer_test.go +++ b/processor/transformer/transformer_test.go @@ -142,85 +142,151 @@ func (et *endpointTransformer) ServeHTTP(w http.ResponseWriter, r *http.Request) } func TestTransformer(t *testing.T) { - t.Run("success", func(t *testing.T) { - ft := &fakeTransformer{ - t: t, - } - - srv := httptest.NewServer(ft) - defer srv.Close() - - tc := []struct { - batchSize int - eventsCount int - failEvery int - }{ - {batchSize: 0, eventsCount: 0}, - {batchSize: 10, eventsCount: 100}, - {batchSize: 10, eventsCount: 9}, - {batchSize: 10, eventsCount: 91}, - {batchSize: 10, eventsCount: 99}, - {batchSize: 10, eventsCount: 1}, - {batchSize: 10, eventsCount: 80, failEvery: 4}, - {batchSize: 10, eventsCount: 80, failEvery: 1}, - } + clientTypes := []string{"stdlib", "recycled", "httplb"} + for _, clientType := range clientTypes { + t.Run(fmt.Sprintf("with %s client", clientType), func(t *testing.T) { + conf := config.New() + conf.Set("Transformer.Client.type", clientType) + + t.Run("success", func(t *testing.T) { + ft := &fakeTransformer{ + t: t, + } - for _, tt := range tc { - statsStore, err := memstats.New() - require.NoError(t, err) + srv := httptest.NewServer(ft) + defer srv.Close() - tr := handle{} - tr.stat = statsStore - tr.logger = logger.NOP - tr.conf = config.Default - tr.client = srv.Client() - tr.guardConcurrency = make(chan struct{}, 200) - tr.sentStat = tr.stat.NewStat("transformer_sent", stats.CountType) - tr.receivedStat = tr.stat.NewStat("transformer_received", stats.CountType) - tr.cpDownGauge = tr.stat.NewStat("control_plane_down", stats.GaugeType) - tr.config.timeoutDuration = 1 * time.Second - tr.config.failOnUserTransformTimeout = config.SingleValueLoader(true) - tr.config.failOnError = config.SingleValueLoader(true) - tr.config.maxRetryBackoffInterval = config.SingleValueLoader(1 * time.Second) - tr.config.maxRetry = config.SingleValueLoader(1) - - batchSize := tt.batchSize - eventsCount := tt.eventsCount - failEvery := tt.failEvery - - events := make([]TransformerEvent, eventsCount) - expectedResponse := Response{} - - transformationID := rand.String(10) - - destinationConfig := backendconfigtest.NewDestinationBuilder("WEBHOOK"). - WithUserTransformation(transformationID, rand.String(10)).Build() - - metadata := Metadata{ - DestinationType: destinationConfig.DestinationDefinition.Name, - SourceID: rand.String(10), - DestinationID: destinationConfig.ID, - TransformationID: destinationConfig.Transformations[0].ID, - } - - for i := range events { - msgID := fmt.Sprintf("messageID-%d", i) - statusCode := http.StatusOK - - if failEvery != 0 && i%failEvery == 0 { - statusCode = http.StatusBadRequest + tc := []struct { + batchSize int + eventsCount int + failEvery int + }{ + {batchSize: 0, eventsCount: 0}, + {batchSize: 10, eventsCount: 100}, + {batchSize: 10, eventsCount: 9}, + {batchSize: 10, eventsCount: 91}, + {batchSize: 10, eventsCount: 99}, + {batchSize: 10, eventsCount: 1}, + {batchSize: 10, eventsCount: 80, failEvery: 4}, + {batchSize: 10, eventsCount: 80, failEvery: 1}, } - metadata := metadata - metadata.MessageID = msgID + for _, tt := range tc { + statsStore, err := memstats.New() + require.NoError(t, err) + + tr := handle{} + tr.stat = statsStore + tr.logger = logger.NOP + tr.conf = conf + tr.httpClient = srv.Client() + tr.guardConcurrency = make(chan struct{}, 200) + tr.sentStat = tr.stat.NewStat("transformer_sent", stats.CountType) + tr.receivedStat = tr.stat.NewStat("transformer_received", stats.CountType) + tr.cpDownGauge = tr.stat.NewStat("control_plane_down", stats.GaugeType) + tr.config.timeoutDuration = 1 * time.Second + tr.config.failOnUserTransformTimeout = config.SingleValueLoader(true) + tr.config.failOnError = config.SingleValueLoader(true) + tr.config.maxRetryBackoffInterval = config.SingleValueLoader(1 * time.Second) + tr.config.maxRetry = config.SingleValueLoader(1) + + batchSize := tt.batchSize + eventsCount := tt.eventsCount + failEvery := tt.failEvery + + events := make([]TransformerEvent, eventsCount) + expectedResponse := Response{} + + transformationID := rand.String(10) + + destinationConfig := backendconfigtest.NewDestinationBuilder("WEBHOOK"). + WithUserTransformation(transformationID, rand.String(10)).Build() + + metadata := Metadata{ + DestinationType: destinationConfig.DestinationDefinition.Name, + SourceID: rand.String(10), + DestinationID: destinationConfig.ID, + TransformationID: destinationConfig.Transformations[0].ID, + } + + for i := range events { + msgID := fmt.Sprintf("messageID-%d", i) + statusCode := http.StatusOK + + if failEvery != 0 && i%failEvery == 0 { + statusCode = http.StatusBadRequest + } + + metadata := metadata + metadata.MessageID = msgID - events[i] = TransformerEvent{ - Metadata: metadata, + events[i] = TransformerEvent{ + Metadata: metadata, + Message: map[string]interface{}{ + "src-key-1": msgID, + "forceStatusCode": statusCode, + }, + Destination: destinationConfig, + Credentials: []Credential{ + { + ID: "test-credential", + Key: "test-key", + Value: "test-value", + IsSecret: false, + }, + }, + } + + tResp := TransformerResponse{ + Metadata: metadata, + StatusCode: statusCode, + Output: map[string]interface{}{ + "src-key-1": msgID, + "echo-key-1": msgID, + }, + } + + if statusCode < http.StatusBadRequest { + expectedResponse.Events = append(expectedResponse.Events, tResp) + } else { + tResp.Error = "error" + expectedResponse.FailedEvents = append(expectedResponse.FailedEvents, tResp) + } + } + + rsp := tr.transform(context.TODO(), events, srv.URL, batchSize, "test-stage") + require.Equal(t, expectedResponse, rsp) + + metrics := statsStore.GetByName("processor.transformer_request_time") + if tt.eventsCount > 0 { + require.NotEmpty(t, metrics) + for _, m := range metrics { + require.Equal(t, stats.Tags{ + "stage": "test-stage", + "sourceId": metadata.SourceID, + "destinationType": destinationConfig.DestinationDefinition.Name, + "destinationId": destinationConfig.ID, + "transformationId": destinationConfig.Transformations[0].ID, + + // Legacy tags: to be removed + "dest_type": destinationConfig.DestinationDefinition.Name, + "dest_id": destinationConfig.ID, + "src_id": metadata.SourceID, + }, m.Tags) + } + } + } + }) + + t.Run("timeout", func(t *testing.T) { + msgID := "messageID-0" + events := append([]TransformerEvent{}, TransformerEvent{ + Metadata: Metadata{ + MessageID: msgID, + }, Message: map[string]interface{}{ - "src-key-1": msgID, - "forceStatusCode": statusCode, + "src-key-1": msgID, }, - Destination: destinationConfig, Credentials: []Credential{ { ID: "test-credential", @@ -229,319 +295,128 @@ func TestTransformer(t *testing.T) { IsSecret: false, }, }, - } + }) - tResp := TransformerResponse{ - Metadata: metadata, - StatusCode: statusCode, - Output: map[string]interface{}{ - "src-key-1": msgID, - "echo-key-1": msgID, + testCases := []struct { + name string + retries int + expectedRetries int + expectPanic bool + stage string + expectedResponse []TransformerResponse + failOnUserTransformTimeout bool + }{ + { + name: "user transformation timeout", + retries: 3, + stage: userTransformerStage, + expectPanic: true, + failOnUserTransformTimeout: false, }, - } - - if statusCode < http.StatusBadRequest { - expectedResponse.Events = append(expectedResponse.Events, tResp) - } else { - tResp.Error = "error" - expectedResponse.FailedEvents = append(expectedResponse.FailedEvents, tResp) - } - } - - rsp := tr.transform(context.TODO(), events, srv.URL, batchSize, "test-stage") - require.Equal(t, expectedResponse, rsp) - - metrics := statsStore.GetByName("processor.transformer_request_time") - if tt.eventsCount > 0 { - require.NotEmpty(t, metrics) - for _, m := range metrics { - require.Equal(t, stats.Tags{ - "stage": "test-stage", - "sourceId": metadata.SourceID, - "destinationType": destinationConfig.DestinationDefinition.Name, - "destinationId": destinationConfig.ID, - "transformationId": destinationConfig.Transformations[0].ID, - - // Legacy tags: to be removed - "dest_type": destinationConfig.DestinationDefinition.Name, - "dest_id": destinationConfig.ID, - "src_id": metadata.SourceID, - }, m.Tags) - } - } - } - }) - - t.Run("timeout", func(t *testing.T) { - msgID := "messageID-0" - events := append([]TransformerEvent{}, TransformerEvent{ - Metadata: Metadata{ - MessageID: msgID, - }, - Message: map[string]interface{}{ - "src-key-1": msgID, - }, - Credentials: []Credential{ - { - ID: "test-credential", - Key: "test-key", - Value: "test-value", - IsSecret: false, - }, - }, - }) - - testCases := []struct { - name string - retries int - expectedRetries int - expectPanic bool - stage string - expectedResponse []TransformerResponse - failOnUserTransformTimeout bool - }{ - { - name: "user transformation timeout", - retries: 3, - stage: userTransformerStage, - expectPanic: true, - failOnUserTransformTimeout: false, - }, - { - name: "user transformation timeout with fail on timeout", - retries: 3, - stage: userTransformerStage, - expectPanic: false, - expectedResponse: []TransformerResponse{ { - Metadata: Metadata{ - MessageID: msgID, - }, - StatusCode: TransformerRequestTimeout, - Output: map[string]interface{}{ - "src-key-1": msgID, + name: "user transformation timeout with fail on timeout", + retries: 3, + stage: userTransformerStage, + expectPanic: false, + expectedResponse: []TransformerResponse{ + { + Metadata: Metadata{ + MessageID: msgID, + }, + StatusCode: TransformerRequestTimeout, + Output: map[string]interface{}{ + "src-key-1": msgID, + }, + }, }, + failOnUserTransformTimeout: true, }, - }, - failOnUserTransformTimeout: true, - }, - { - name: "destination transformation timeout", - retries: 3, - stage: destTransformerStage, - expectPanic: true, - failOnUserTransformTimeout: false, - }, - { - name: "destination transformation timeout with fail on timeout", - retries: 3, - stage: destTransformerStage, - expectPanic: true, - failOnUserTransformTimeout: true, - }, - } - - for _, tc := range testCases { - tc := tc - - t.Run(tc.name, func(t *testing.T) { - ch := make(chan struct{}) - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - <-ch - })) - defer srv.Close() - - client := srv.Client() - client.Timeout = 1 * time.Millisecond - - tr := handle{} - tr.config.timeoutDuration = 1 * time.Millisecond - tr.stat = stats.Default - tr.logger = logger.NOP - tr.conf = config.Default - tr.client = client - tr.config.maxRetry = config.SingleValueLoader(tc.retries) - tr.config.failOnUserTransformTimeout = config.SingleValueLoader(tc.failOnUserTransformTimeout) - tr.cpDownGauge = tr.stat.NewStat("control_plane_down", stats.GaugeType) - tr.config.maxRetryBackoffInterval = config.SingleValueLoader(1 * time.Second) + { + name: "destination transformation timeout", + retries: 3, + stage: destTransformerStage, + expectPanic: true, + failOnUserTransformTimeout: false, + }, + { + name: "destination transformation timeout with fail on timeout", + retries: 3, + stage: destTransformerStage, + expectPanic: true, + failOnUserTransformTimeout: true, + }, + } - if tc.expectPanic { - require.Panics(t, func() { - _ = tr.request(context.TODO(), srv.URL, tc.stage, events) + for _, tc := range testCases { + tc := tc + + t.Run(tc.name, func(t *testing.T) { + ch := make(chan struct{}) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + <-ch + })) + defer srv.Close() + + client := srv.Client() + client.Timeout = 1 * time.Millisecond + + tr := handle{} + tr.config.timeoutDuration = 1 * time.Millisecond + tr.stat = stats.Default + tr.logger = logger.NOP + tr.conf = conf + tr.httpClient = client + tr.config.maxRetry = config.SingleValueLoader(tc.retries) + tr.config.failOnUserTransformTimeout = config.SingleValueLoader(tc.failOnUserTransformTimeout) + tr.cpDownGauge = tr.stat.NewStat("control_plane_down", stats.GaugeType) + tr.config.maxRetryBackoffInterval = config.SingleValueLoader(1 * time.Second) + + if tc.expectPanic { + require.Panics(t, func() { + _ = tr.request(context.TODO(), srv.URL, tc.stage, events) + }) + close(ch) + return + } + + _, err := url.Parse(srv.URL) + require.NoError(t, err) + + rsp := tr.request(context.TODO(), srv.URL, tc.stage, events) + require.Len(t, rsp, 1) + require.Equal(t, rsp[0].StatusCode, TransformerRequestTimeout) + require.Equal(t, rsp[0].Metadata, Metadata{ + MessageID: msgID, + }) + require.Contains(t, rsp[0].Error, "transformer request timed out:") + close(ch) }) - close(ch) - return } - - _, err := url.Parse(srv.URL) - require.NoError(t, err) - - rsp := tr.request(context.TODO(), srv.URL, tc.stage, events) - require.Len(t, rsp, 1) - require.Equal(t, rsp[0].StatusCode, TransformerRequestTimeout) - require.Equal(t, rsp[0].Metadata, Metadata{ - MessageID: msgID, - }) - require.Contains(t, rsp[0].Error, "transformer request timed out:") - close(ch) }) - } - }) - - t.Run("endless retries in case of control plane down", func(t *testing.T) { - msgID := "messageID-0" - events := append([]TransformerEvent{}, TransformerEvent{ - Metadata: Metadata{ - MessageID: msgID, - }, - Message: map[string]interface{}{ - "src-key-1": msgID, - }, - Credentials: []Credential{ - { - ID: "test-credential", - Key: "test-key", - Value: "test-value", - IsSecret: false, - }, - }, - }) - - elt := &endlessLoopTransformer{ - maxRetryCount: 3, - statusCode: StatusCPDown, - statusError: "control plane not reachable", - apiVersion: types.SupportedTransformerApiVersion, - t: t, - } - - srv := httptest.NewServer(elt) - defer srv.Close() - - tr := handle{} - tr.stat = stats.Default - tr.logger = logger.NOP - tr.conf = config.Default - tr.client = srv.Client() - tr.config.maxRetry = config.SingleValueLoader(1) - tr.config.maxRetryBackoffInterval = config.SingleValueLoader(1 * time.Second) - tr.config.timeoutDuration = 1 * time.Second - tr.config.failOnUserTransformTimeout = config.SingleValueLoader(false) - tr.cpDownGauge = tr.stat.NewStat("control_plane_down", stats.GaugeType) - - rsp := tr.request(context.TODO(), srv.URL, "test-stage", events) - require.Equal(t, rsp, []TransformerResponse{ - { - Metadata: Metadata{ - MessageID: msgID, - }, - StatusCode: http.StatusOK, - Output: map[string]interface{}{ - "src-key-1": msgID, - }, - }, - }) - require.Equal(t, elt.retryCount, 3) - }) - t.Run("retries", func(t *testing.T) { - msgID := "messageID-0" - events := append([]TransformerEvent{}, TransformerEvent{ - Metadata: Metadata{ - MessageID: msgID, - }, - Message: map[string]interface{}{ - "src-key-1": msgID, - }, - Destination: backendconfig.DestinationT{ - Transformations: []backendconfig.TransformationT{ - { - ID: "test-transformation", - VersionID: "test-version", + t.Run("endless retries in case of control plane down", func(t *testing.T) { + msgID := "messageID-0" + events := append([]TransformerEvent{}, TransformerEvent{ + Metadata: Metadata{ + MessageID: msgID, }, - }, - }, - Credentials: []Credential{ - { - ID: "test-credential", - Key: "test-key", - Value: "test-value", - IsSecret: false, - }, - }, - }) - - testCases := []struct { - name string - retries int - maxRetryCount int - statusCode int - statusError string - expectedRetries int - expectPanic bool - expectedResponse []TransformerResponse - failOnError bool - }{ - { - name: "too many requests", - retries: 3, - maxRetryCount: 10, - statusCode: http.StatusTooManyRequests, - statusError: "too many requests", - expectedRetries: 4, - expectPanic: true, - failOnError: false, - }, - { - name: "too many requests with fail on error", - retries: 3, - maxRetryCount: 10, - statusCode: http.StatusTooManyRequests, - statusError: "too many requests", - expectedRetries: 4, - expectPanic: false, - expectedResponse: []TransformerResponse{ - { - Metadata: Metadata{ - MessageID: msgID, - }, - StatusCode: TransformerRequestFailure, - Error: "transformer request failed: transformer returned status code: 429", + Message: map[string]interface{}{ + "src-key-1": msgID, }, - }, - failOnError: true, - }, - { - name: "transient control plane error", - retries: 30, - maxRetryCount: 3, - statusCode: StatusCPDown, - statusError: "control plane not reachable", - expectedRetries: 3, - expectPanic: false, - expectedResponse: []TransformerResponse{ - { - Metadata: Metadata{ - MessageID: msgID, - }, - StatusCode: http.StatusOK, - Output: map[string]interface{}{ - "src-key-1": msgID, + Credentials: []Credential{ + { + ID: "test-credential", + Key: "test-key", + Value: "test-value", + IsSecret: false, }, }, - }, - failOnError: false, - }, - } - - for _, tc := range testCases { - tc := tc + }) - t.Run(tc.name, func(t *testing.T) { elt := &endlessLoopTransformer{ - maxRetryCount: tc.maxRetryCount, - statusCode: tc.statusCode, - statusError: tc.statusError, + maxRetryCount: 3, + statusCode: StatusCPDown, + statusError: "control plane not reachable", apiVersion: types.SupportedTransformerApiVersion, t: t, } @@ -552,69 +427,16 @@ func TestTransformer(t *testing.T) { tr := handle{} tr.stat = stats.Default tr.logger = logger.NOP - tr.conf = config.Default - tr.client = srv.Client() + tr.conf = conf + tr.httpClient = srv.Client() + tr.config.maxRetry = config.SingleValueLoader(1) + tr.config.maxRetryBackoffInterval = config.SingleValueLoader(1 * time.Second) + tr.config.timeoutDuration = 1 * time.Second tr.config.failOnUserTransformTimeout = config.SingleValueLoader(false) - tr.config.maxRetry = config.SingleValueLoader(tc.retries) - tr.config.failOnError = config.SingleValueLoader(tc.failOnError) tr.cpDownGauge = tr.stat.NewStat("control_plane_down", stats.GaugeType) - tr.config.timeoutDuration = 1 * time.Second - tr.config.maxRetryBackoffInterval = config.SingleValueLoader(1 * time.Second) - - if tc.expectPanic { - require.Panics(t, func() { - _ = tr.request(context.TODO(), srv.URL, "test-stage", events) - }) - require.Equal(t, elt.retryCount, tc.expectedRetries) - return - } rsp := tr.request(context.TODO(), srv.URL, "test-stage", events) - require.Equal(t, tc.expectedResponse, rsp) - require.Equal(t, tc.expectedRetries, elt.retryCount) - }) - } - }) - - t.Run("version compatibility", func(t *testing.T) { - msgID := "messageID-0" - events := append([]TransformerEvent{}, TransformerEvent{ - Metadata: Metadata{ - MessageID: msgID, - }, - Message: map[string]interface{}{ - "src-key-1": msgID, - }, - Destination: backendconfig.DestinationT{ - Transformations: []backendconfig.TransformationT{ - { - ID: "test-transformation", - VersionID: "test-version", - }, - }, - }, - Credentials: []Credential{ - { - ID: "test-credential", - Key: "test-key", - Value: "test-value", - IsSecret: false, - }, - }, - }) - - testCases := []struct { - name string - apiVersion int - skipApiVersion bool - expectPanic bool - expectedResponse []TransformerResponse - }{ - { - name: "compatible api version", - apiVersion: types.SupportedTransformerApiVersion, - expectPanic: false, - expectedResponse: []TransformerResponse{ + require.Equal(t, rsp, []TransformerResponse{ { Metadata: Metadata{ MessageID: msgID, @@ -624,143 +446,287 @@ func TestTransformer(t *testing.T) { "src-key-1": msgID, }, }, - }, - }, - { - name: "incompatible api version", - apiVersion: 1, - expectPanic: true, - }, - { - name: "unexpected api version", - skipApiVersion: true, - expectPanic: true, - }, - } + }) + require.Equal(t, elt.retryCount, 3) + }) - for _, tc := range testCases { - tc := tc + t.Run("retries", func(t *testing.T) { + msgID := "messageID-0" + events := append([]TransformerEvent{}, TransformerEvent{ + Metadata: Metadata{ + MessageID: msgID, + }, + Message: map[string]interface{}{ + "src-key-1": msgID, + }, + Destination: backendconfig.DestinationT{ + Transformations: []backendconfig.TransformationT{ + { + ID: "test-transformation", + VersionID: "test-version", + }, + }, + }, + Credentials: []Credential{ + { + ID: "test-credential", + Key: "test-key", + Value: "test-value", + IsSecret: false, + }, + }, + }) - t.Run(tc.name, func(t *testing.T) { - elt := &endlessLoopTransformer{ - maxRetryCount: 0, - skipApiVersion: tc.skipApiVersion, - apiVersion: tc.apiVersion, - t: t, + testCases := []struct { + name string + retries int + maxRetryCount int + statusCode int + statusError string + expectedRetries int + expectPanic bool + expectedResponse []TransformerResponse + failOnError bool + }{ + { + name: "too many requests", + retries: 3, + maxRetryCount: 10, + statusCode: http.StatusTooManyRequests, + statusError: "too many requests", + expectedRetries: 4, + expectPanic: true, + failOnError: false, + }, + { + name: "too many requests with fail on error", + retries: 3, + maxRetryCount: 10, + statusCode: http.StatusTooManyRequests, + statusError: "too many requests", + expectedRetries: 4, + expectPanic: false, + expectedResponse: []TransformerResponse{ + { + Metadata: Metadata{ + MessageID: msgID, + }, + StatusCode: TransformerRequestFailure, + Error: "transformer request failed: transformer returned status code: 429", + }, + }, + failOnError: true, + }, + { + name: "transient control plane error", + retries: 30, + maxRetryCount: 3, + statusCode: StatusCPDown, + statusError: "control plane not reachable", + expectedRetries: 3, + expectPanic: false, + expectedResponse: []TransformerResponse{ + { + Metadata: Metadata{ + MessageID: msgID, + }, + StatusCode: http.StatusOK, + Output: map[string]interface{}{ + "src-key-1": msgID, + }, + }, + }, + failOnError: false, + }, } - srv := httptest.NewServer(elt) - defer srv.Close() - - tr := handle{} - tr.client = srv.Client() - tr.stat = stats.Default - tr.conf = config.Default - tr.logger = logger.NOP - tr.cpDownGauge = tr.stat.NewStat("control_plane_down", stats.GaugeType) - tr.config.maxRetry = config.SingleValueLoader(1) - tr.config.timeoutDuration = 1 * time.Second - tr.config.maxRetryBackoffInterval = config.SingleValueLoader(1 * time.Second) - - if tc.expectPanic { - require.Panics(t, func() { - _ = tr.request(context.TODO(), srv.URL, "test-stage", events) + for _, tc := range testCases { + tc := tc + + t.Run(tc.name, func(t *testing.T) { + elt := &endlessLoopTransformer{ + maxRetryCount: tc.maxRetryCount, + statusCode: tc.statusCode, + statusError: tc.statusError, + apiVersion: types.SupportedTransformerApiVersion, + t: t, + } + + srv := httptest.NewServer(elt) + defer srv.Close() + + tr := handle{} + tr.stat = stats.Default + tr.logger = logger.NOP + tr.conf = conf + tr.httpClient = srv.Client() + tr.config.failOnUserTransformTimeout = config.SingleValueLoader(false) + tr.config.maxRetry = config.SingleValueLoader(tc.retries) + tr.config.failOnError = config.SingleValueLoader(tc.failOnError) + tr.cpDownGauge = tr.stat.NewStat("control_plane_down", stats.GaugeType) + tr.config.timeoutDuration = 1 * time.Second + tr.config.maxRetryBackoffInterval = config.SingleValueLoader(1 * time.Second) + + if tc.expectPanic { + require.Panics(t, func() { + _ = tr.request(context.TODO(), srv.URL, "test-stage", events) + }) + require.Equal(t, elt.retryCount, tc.expectedRetries) + return + } + + rsp := tr.request(context.TODO(), srv.URL, "test-stage", events) + require.Equal(t, tc.expectedResponse, rsp) + require.Equal(t, tc.expectedRetries, elt.retryCount) }) - return } - - rsp := tr.request(context.TODO(), srv.URL, "test-stage", events) - require.Equal(t, tc.expectedResponse, rsp) }) - } - }) - t.Run("endpoints", func(t *testing.T) { - msgID := "messageID-0" - expectedResponse := Response{ - Events: []TransformerResponse{ - { - Output: map[string]interface{}{ - "src-key-1": msgID, - }, + t.Run("version compatibility", func(t *testing.T) { + msgID := "messageID-0" + events := append([]TransformerEvent{}, TransformerEvent{ Metadata: Metadata{ MessageID: msgID, }, - StatusCode: http.StatusOK, - }, - }, - } - events := append([]TransformerEvent{}, TransformerEvent{ - Metadata: Metadata{ - MessageID: msgID, - }, - Message: map[string]interface{}{ - "src-key-1": msgID, - }, - Destination: backendconfig.DestinationT{ - DestinationDefinition: backendconfig.DestinationDefinitionT{ - Name: "test-destination", - }, - Transformations: []backendconfig.TransformationT{ - { - ID: "test-transformation", - VersionID: "test-version", + Message: map[string]interface{}{ + "src-key-1": msgID, }, - }, - }, - Credentials: []Credential{ - { - ID: "test-credential", - Key: "test-key", - Value: "test-value", - IsSecret: false, - }, - }, - }) - - t.Run("Destination transformations", func(t *testing.T) { - et := &endpointTransformer{ - supportedPaths: []string{"/v0/destinations/test-destination"}, - t: t, - } + Destination: backendconfig.DestinationT{ + Transformations: []backendconfig.TransformationT{ + { + ID: "test-transformation", + VersionID: "test-version", + }, + }, + }, + Credentials: []Credential{ + { + ID: "test-credential", + Key: "test-key", + Value: "test-value", + IsSecret: false, + }, + }, + }) - srv := httptest.NewServer(et) - defer srv.Close() + testCases := []struct { + name string + apiVersion int + skipApiVersion bool + expectPanic bool + expectedResponse []TransformerResponse + }{ + { + name: "compatible api version", + apiVersion: types.SupportedTransformerApiVersion, + expectPanic: false, + expectedResponse: []TransformerResponse{ + { + Metadata: Metadata{ + MessageID: msgID, + }, + StatusCode: http.StatusOK, + Output: map[string]interface{}{ + "src-key-1": msgID, + }, + }, + }, + }, + { + name: "incompatible api version", + apiVersion: 1, + expectPanic: true, + }, + { + name: "unexpected api version", + skipApiVersion: true, + expectPanic: true, + }, + } - c := config.New() - c.Set("Processor.maxRetry", 1) - c.Set("DEST_TRANSFORM_URL", srv.URL) + for _, tc := range testCases { + tc := tc + + t.Run(tc.name, func(t *testing.T) { + elt := &endlessLoopTransformer{ + maxRetryCount: 0, + skipApiVersion: tc.skipApiVersion, + apiVersion: tc.apiVersion, + t: t, + } + + srv := httptest.NewServer(elt) + defer srv.Close() + + tr := handle{} + tr.httpClient = srv.Client() + tr.stat = stats.Default + tr.conf = conf + tr.logger = logger.NOP + tr.cpDownGauge = tr.stat.NewStat("control_plane_down", stats.GaugeType) + tr.config.maxRetry = config.SingleValueLoader(1) + tr.config.timeoutDuration = 1 * time.Second + tr.config.maxRetryBackoffInterval = config.SingleValueLoader(1 * time.Second) + + if tc.expectPanic { + require.Panics(t, func() { + _ = tr.request(context.TODO(), srv.URL, "test-stage", events) + }) + return + } + + rsp := tr.request(context.TODO(), srv.URL, "test-stage", events) + require.Equal(t, tc.expectedResponse, rsp) + }) + } + }) - tr := NewTransformer(c, logger.NOP, stats.Default, WithClient(srv.Client())) - rsp := tr.Transform(context.TODO(), events, 10) - require.Equal(t, rsp, expectedResponse) - }) + t.Run("endpoints", func(t *testing.T) { + msgID := "messageID-0" + expectedResponse := Response{ + Events: []TransformerResponse{ + { + Output: map[string]interface{}{ + "src-key-1": msgID, + }, + Metadata: Metadata{ + MessageID: msgID, + }, + StatusCode: http.StatusOK, + }, + }, + } + events := append([]TransformerEvent{}, TransformerEvent{ + Metadata: Metadata{ + MessageID: msgID, + }, + Message: map[string]interface{}{ + "src-key-1": msgID, + }, + Destination: backendconfig.DestinationT{ + DestinationDefinition: backendconfig.DestinationDefinitionT{ + Name: "test-destination", + }, + Transformations: []backendconfig.TransformationT{ + { + ID: "test-transformation", + VersionID: "test-version", + }, + }, + }, + Credentials: []Credential{ + { + ID: "test-credential", + Key: "test-key", + Value: "test-value", + IsSecret: false, + }, + }, + }) - t.Run("Destination warehouse transformations", func(t *testing.T) { - testCases := []struct { - name string - destinationType string - }{ - { - name: "rs", - destinationType: warehouseutils.RS, - }, - { - name: "clickhouse", - destinationType: warehouseutils.CLICKHOUSE, - }, - { - name: "snowflake", - destinationType: warehouseutils.SNOWFLAKE, - }, - } - - for _, tc := range testCases { - tc := tc - - t.Run(tc.name, func(t *testing.T) { + t.Run("Destination transformations", func(t *testing.T) { et := &endpointTransformer{ - supportedPaths: []string{`/v0/destinations/` + tc.name}, + supportedPaths: []string{"/v0/destinations/test-destination"}, t: t, } @@ -772,77 +738,119 @@ func TestTransformer(t *testing.T) { c.Set("DEST_TRANSFORM_URL", srv.URL) tr := NewTransformer(c, logger.NOP, stats.Default, WithClient(srv.Client())) + rsp := tr.Transform(context.TODO(), events, 10) + require.Equal(t, rsp, expectedResponse) + }) - events := append([]TransformerEvent{}, TransformerEvent{ - Metadata: Metadata{ - MessageID: msgID, - }, - Message: map[string]interface{}{ - "src-key-1": msgID, + t.Run("Destination warehouse transformations", func(t *testing.T) { + testCases := []struct { + name string + destinationType string + }{ + { + name: "rs", + destinationType: warehouseutils.RS, }, - Destination: backendconfig.DestinationT{ - DestinationDefinition: backendconfig.DestinationDefinitionT{ - Name: tc.destinationType, - }, - Transformations: []backendconfig.TransformationT{ - { - ID: "test-transformation", - VersionID: "test-version", - }, - }, + { + name: "clickhouse", + destinationType: warehouseutils.CLICKHOUSE, }, - Credentials: []Credential{ - { - ID: "test-credential", - Key: "test-key", - Value: "test-value", - IsSecret: false, - }, + { + name: "snowflake", + destinationType: warehouseutils.SNOWFLAKE, }, - }) + } - rsp := tr.Transform(context.TODO(), events, 10) - require.Equal(t, rsp, expectedResponse) + for _, tc := range testCases { + tc := tc + + t.Run(tc.name, func(t *testing.T) { + et := &endpointTransformer{ + supportedPaths: []string{`/v0/destinations/` + tc.name}, + t: t, + } + + srv := httptest.NewServer(et) + defer srv.Close() + + c := config.New() + c.Set("Processor.maxRetry", 1) + c.Set("DEST_TRANSFORM_URL", srv.URL) + + tr := NewTransformer(c, logger.NOP, stats.Default, WithClient(srv.Client())) + + events := append([]TransformerEvent{}, TransformerEvent{ + Metadata: Metadata{ + MessageID: msgID, + }, + Message: map[string]interface{}{ + "src-key-1": msgID, + }, + Destination: backendconfig.DestinationT{ + DestinationDefinition: backendconfig.DestinationDefinitionT{ + Name: tc.destinationType, + }, + Transformations: []backendconfig.TransformationT{ + { + ID: "test-transformation", + VersionID: "test-version", + }, + }, + }, + Credentials: []Credential{ + { + ID: "test-credential", + Key: "test-key", + Value: "test-value", + IsSecret: false, + }, + }, + }) + + rsp := tr.Transform(context.TODO(), events, 10) + require.Equal(t, rsp, expectedResponse) + }) + } }) - } - }) - t.Run("User transformations", func(t *testing.T) { - et := &endpointTransformer{ - supportedPaths: []string{"/customTransform"}, - t: t, - } + t.Run("User transformations", func(t *testing.T) { + et := &endpointTransformer{ + supportedPaths: []string{"/customTransform"}, + t: t, + } - srv := httptest.NewServer(et) - defer srv.Close() + srv := httptest.NewServer(et) + defer srv.Close() - c := config.New() - c.Set("Processor.maxRetry", 1) - c.Set("USER_TRANSFORM_URL", srv.URL) + c := config.New() + c.Set("Processor.maxRetry", 1) + c.Set("USER_TRANSFORM_URL", srv.URL) - tr := NewTransformer(c, logger.NOP, stats.Default, WithClient(srv.Client())) - rsp := tr.UserTransform(context.TODO(), events, 10) - require.Equal(t, rsp, expectedResponse) - }) + tr := NewTransformer(c, logger.NOP, stats.Default, WithClient(srv.Client())) + rsp := tr.UserTransform(context.TODO(), events, 10) + require.Equal(t, rsp, expectedResponse) + }) - t.Run("Tracking Plan Validations", func(t *testing.T) { - et := &endpointTransformer{ - supportedPaths: []string{"/v0/validate"}, - t: t, - } + t.Run("Tracking Plan Validations", func(t *testing.T) { + et := &endpointTransformer{ + supportedPaths: []string{"/v0/validate"}, + t: t, + } - srv := httptest.NewServer(et) - defer srv.Close() + srv := httptest.NewServer(et) + defer srv.Close() - c := config.New() - c.Set("Processor.maxRetry", 1) - c.Set("DEST_TRANSFORM_URL", srv.URL) + c := config.New() + c.Set("Processor.maxRetry", 1) + c.Set("DEST_TRANSFORM_URL", srv.URL) - tr := NewTransformer(c, logger.NOP, stats.Default, WithClient(srv.Client())) - rsp := tr.Validate(context.TODO(), events, 10) - require.Equal(t, rsp, expectedResponse) + tr := NewTransformer(c, logger.NOP, stats.Default, WithClient(srv.Client())) + rsp := tr.Validate(context.TODO(), events, 10) + require.Equal(t, rsp, expectedResponse) + }) + }) }) - }) + } } func TestLongRunningTransformation(t *testing.T) { diff --git a/utils/sysUtils/httpclient.go b/utils/sysUtils/httpclient.go index a7a81881ab..22bbb9ad94 100644 --- a/utils/sysUtils/httpclient.go +++ b/utils/sysUtils/httpclient.go @@ -3,9 +3,44 @@ package sysUtils import ( "net/http" + "sync" + "time" ) // HTTPClient interface type HTTPClientI interface { Do(req *http.Request) (*http.Response, error) } + +type RecycledHTTPClient struct { + client *http.Client + lastRefreshTime time.Time + ttl time.Duration + clientFunc func() *http.Client + lock sync.Mutex +} + +func NewRecycledHTTPClient(_clientFunc func() *http.Client, _ttl time.Duration) *RecycledHTTPClient { + return &RecycledHTTPClient{ + client: _clientFunc(), + clientFunc: _clientFunc, + ttl: _ttl, + lastRefreshTime: time.Now(), + } +} + +func (r *RecycledHTTPClient) GetClient() *http.Client { + r.lock.Lock() + defer r.lock.Unlock() + + if r.ttl > 0 && time.Since(r.lastRefreshTime) > r.ttl { + r.client.CloseIdleConnections() + r.client = r.clientFunc() + r.lastRefreshTime = time.Now() + } + return r.client +} + +func (r *RecycledHTTPClient) Do(req *http.Request) (*http.Response, error) { + return r.GetClient().Do(req) +} From 29b279d0b9409045d0cc89cc14129793126004e6 Mon Sep 17 00:00:00 2001 From: devops-github-rudderstack <88187154+devops-github-rudderstack@users.noreply.github.com> Date: Tue, 7 Jan 2025 13:21:17 +0530 Subject: [PATCH 06/11] chore: release 1.40.0 (#5410) --- CHANGELOG.md | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5822b2ae34..3ff73fe014 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,45 @@ # Changelog +## [1.40.0](https://github.com/rudderlabs/rudder-server/compare/v1.39.0...v1.40.0) (2025-01-06) + + +### Features + +* add support for hash to bingads offline conversions ([#5390](https://github.com/rudderlabs/rudder-server/issues/5390)) ([8a186e5](https://github.com/rudderlabs/rudder-server/commit/8a186e59e74772ebf10aca2ec352c6a51537c57a)) +* client side load balancing for user transformations ([#5375](https://github.com/rudderlabs/rudder-server/issues/5375)) ([de3e87c](https://github.com/rudderlabs/rudder-server/commit/de3e87c83f4cb0adf6fd4dff0437fbcd7bcd2855)) + + +### Bug Fixes + +* allow only enabled dest in backendSubscriber, fix klaviyo bulk ([#5309](https://github.com/rudderlabs/rudder-server/issues/5309)) ([98827e5](https://github.com/rudderlabs/rudder-server/commit/98827e57eae6949ea06bd7a0ca58e678360e66e6)) +* bq partitioning for additional columns ([#5293](https://github.com/rudderlabs/rudder-server/issues/5293)) ([98827e5](https://github.com/rudderlabs/rudder-server/commit/98827e57eae6949ea06bd7a0ca58e678360e66e6)) +* disable vacuum at startup for reporting ([#5325](https://github.com/rudderlabs/rudder-server/issues/5325)) ([98827e5](https://github.com/rudderlabs/rudder-server/commit/98827e57eae6949ea06bd7a0ca58e678360e66e6)) +* klaviyo bulk upload and BingAds OC ([#5305](https://github.com/rudderlabs/rudder-server/issues/5305)) ([98827e5](https://github.com/rudderlabs/rudder-server/commit/98827e57eae6949ea06bd7a0ca58e678360e66e6)) +* processing pickup race condition ([#5374](https://github.com/rudderlabs/rudder-server/issues/5374)) ([b417005](https://github.com/rudderlabs/rudder-server/commit/b4170057d7035303dac762588bba21e2bd6413c1)) +* processing pickup race condition ([#5374](https://github.com/rudderlabs/rudder-server/issues/5374)) ([a82aa47](https://github.com/rudderlabs/rudder-server/commit/a82aa4719922f0ff82a938df69b6a83b8979e911)) +* processing pickup race condition ([#5374](https://github.com/rudderlabs/rudder-server/issues/5374)) ([dd33fee](https://github.com/rudderlabs/rudder-server/commit/dd33feefa233457fb7667666ede870118d3fc125)) +* replay tracking plan bug ([#5389](https://github.com/rudderlabs/rudder-server/issues/5389)) ([b417005](https://github.com/rudderlabs/rudder-server/commit/b4170057d7035303dac762588bba21e2bd6413c1)) + + +### Miscellaneous + +* [Snyk] Security upgrade alpine from 3.17 to 3.21.0 ([#5366](https://github.com/rudderlabs/rudder-server/issues/5366)) ([f22e8f4](https://github.com/rudderlabs/rudder-server/commit/f22e8f4bb61893af132f3f4386155c83c695fd51)) +* add error msg in the logs when gw req fails ([#5369](https://github.com/rudderlabs/rudder-server/issues/5369)) ([b417005](https://github.com/rudderlabs/rudder-server/commit/b4170057d7035303dac762588bba21e2bd6413c1)) +* add error msg in the logs when gw req fails ([#5369](https://github.com/rudderlabs/rudder-server/issues/5369)) ([a82aa47](https://github.com/rudderlabs/rudder-server/commit/a82aa4719922f0ff82a938df69b6a83b8979e911)) +* cleanup archiver to use uploadID for filtering ([#5346](https://github.com/rudderlabs/rudder-server/issues/5346)) ([8fa4436](https://github.com/rudderlabs/rudder-server/commit/8fa4436346530817ee512aa0e116f2b1070eb25c)) +* collect stats for reporting event sampler ([#5357](https://github.com/rudderlabs/rudder-server/issues/5357)) ([e504d75](https://github.com/rudderlabs/rudder-server/commit/e504d75e4bf1d086596c6360788abef28671d551)) +* **deps:** bump google.golang.org/api from 0.211.0 to 0.212.0 in the frequent group ([#5378](https://github.com/rudderlabs/rudder-server/issues/5378)) ([7b46cd2](https://github.com/rudderlabs/rudder-server/commit/7b46cd2eec2f995aff9db6f73bc197b101d145d8)) +* **deps:** bump google.golang.org/protobuf from 1.35.2 to 1.36.0 in the go-deps group ([#5379](https://github.com/rudderlabs/rudder-server/issues/5379)) ([7425659](https://github.com/rudderlabs/rudder-server/commit/74256599bbbe460c0166470196704b9de9477c14)) +* **deps:** bump the go-deps group across 1 directory with 2 updates ([#5397](https://github.com/rudderlabs/rudder-server/issues/5397)) ([46d9b6a](https://github.com/rudderlabs/rudder-server/commit/46d9b6aa376a6f6c6611d9d7e3bfcbbbbc7fb478)) +* **deps:** bump the go-deps group across 1 directory with 2 updates ([#5403](https://github.com/rudderlabs/rudder-server/issues/5403)) ([78fb917](https://github.com/rudderlabs/rudder-server/commit/78fb9178c1bed663c095a7d6506d891279370084)) +* **deps:** bump the go-deps group across 1 directory with 4 updates ([#5368](https://github.com/rudderlabs/rudder-server/issues/5368)) ([33a1e30](https://github.com/rudderlabs/rudder-server/commit/33a1e3078dccdd071e555175410c1f461d67b1e6)) +* **deps:** bump the go-deps group across 1 directory with 5 updates ([#5386](https://github.com/rudderlabs/rudder-server/issues/5386)) ([471d492](https://github.com/rudderlabs/rudder-server/commit/471d492910cd2fb9722d5d1bf4f8640e6f4a2f0b)) +* **deps:** bump the go-deps group with 3 updates ([#5373](https://github.com/rudderlabs/rudder-server/issues/5373)) ([d0ce669](https://github.com/rudderlabs/rudder-server/commit/d0ce6697971851848ad2b0010a8e1aa0e541c513)) +* oauth v2 stats refactor ([#5262](https://github.com/rudderlabs/rudder-server/issues/5262)) ([18f4bdf](https://github.com/rudderlabs/rudder-server/commit/18f4bdf0e918f7b8d6792220a9abe6d227377585)) +* reduce the error report sample events ([#5371](https://github.com/rudderlabs/rudder-server/issues/5371)) ([989310c](https://github.com/rudderlabs/rudder-server/commit/989310cb109e6bfd675def135ca0974ecf8b52d6)) +* remove full vacuum at flusher startup ([#5332](https://github.com/rudderlabs/rudder-server/issues/5332)) ([98827e5](https://github.com/rudderlabs/rudder-server/commit/98827e57eae6949ea06bd7a0ca58e678360e66e6)) +* sync release v1.39.0 to main branch ([#5367](https://github.com/rudderlabs/rudder-server/issues/5367)) ([9f79eee](https://github.com/rudderlabs/rudder-server/commit/9f79eee2262b46ecf1fe6e4abce0dcac9cf25eec)) + ## [1.39.3](https://github.com/rudderlabs/rudder-server/compare/v1.39.2...v1.39.3) (2024-12-23) From b51605af528cda22c1158f4daeae2675d92ee50a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 8 Jan 2025 08:43:36 +0530 Subject: [PATCH 07/11] chore(deps): bump google.golang.org/api from 0.214.0 to 0.215.0 in the frequent group (#5415) --- go.mod | 8 ++++---- go.sum | 16 ++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index 8827a8870f..0750085140 100644 --- a/go.mod +++ b/go.mod @@ -108,8 +108,8 @@ require ( golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 golang.org/x/oauth2 v0.25.0 golang.org/x/sync v0.10.0 - google.golang.org/api v0.214.0 - google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576 + google.golang.org/api v0.215.0 + google.golang.org/genproto/googleapis/rpc v0.0.0-20241223144023-3abc09e42ca8 google.golang.org/grpc v1.68.1 google.golang.org/protobuf v1.36.1 ) @@ -236,7 +236,7 @@ require ( github.com/google/s2a-go v0.1.8 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect - github.com/googleapis/gax-go/v2 v2.14.0 // indirect + github.com/googleapis/gax-go/v2 v2.14.1 // indirect github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect github.com/hamba/avro/v2 v2.26.0 // indirect @@ -349,7 +349,7 @@ require ( golang.org/x/tools v0.28.0 // indirect golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect google.golang.org/genproto v0.0.0-20241118233622-e639e219e697 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20241118233622-e639e219e697 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576 // indirect google.golang.org/grpc/stats/opentelemetry v0.0.0-20240907200651-3ffb98b2c93a // indirect gopkg.in/alexcesaro/statsd.v2 v2.0.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect diff --git a/go.sum b/go.sum index 3cdfdbbfcf..85cea23eb8 100644 --- a/go.sum +++ b/go.sum @@ -769,8 +769,8 @@ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5m github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= github.com/googleapis/gax-go/v2 v2.2.0/go.mod h1:as02EH8zWkzwUoLbBaFeQ+arQaj/OthfcblKl4IGNaM= -github.com/googleapis/gax-go/v2 v2.14.0 h1:f+jMrjBPl+DL9nI4IQzLUxMq7XrAqFYB7hBPqMNIe8o= -github.com/googleapis/gax-go/v2 v2.14.0/go.mod h1:lhBCnjdLrWRaPvLWhmc8IS24m9mr07qSYnHncrgo+zk= +github.com/googleapis/gax-go/v2 v2.14.1 h1:hb0FFeiPaQskmvakKu5EbCbpntQn48jyHuvrkurSS/Q= +github.com/googleapis/gax-go/v2 v2.14.1/go.mod h1:Hb/NubMaVM88SrNkvl8X/o8XWwDJEPqouaLeN2IUxoA= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/handlers v1.5.2 h1:cLTUSsNkgcwhgRqvCNmdbRWG0A3N4F+M2nWKdScwyEE= @@ -1774,8 +1774,8 @@ google.golang.org/api v0.69.0/go.mod h1:boanBiw+h5c3s+tBPgEzLDRHfFLWV0qXxRHz3ws7 google.golang.org/api v0.70.0/go.mod h1:Bs4ZM2HGifEvXwd50TtW70ovgJffJYw2oRCOFU/SkfA= google.golang.org/api v0.71.0/go.mod h1:4PyU6e6JogV1f9eA4voyrTY2batOLdgZ5qZ5HOCc4j8= google.golang.org/api v0.74.0/go.mod h1:ZpfMZOVRMywNyvJFeqL9HRWBgAuRfSjJFpe9QtRRyDs= -google.golang.org/api v0.214.0 h1:h2Gkq07OYi6kusGOaT/9rnNljuXmqPnaig7WGPmKbwA= -google.golang.org/api v0.214.0/go.mod h1:bYPpLG8AyeMWwDU6NXoB00xC0DFkikVvd5MfwoxjLqE= +google.golang.org/api v0.215.0 h1:jdYF4qnyczlEz2ReWIsosNLDuzXyvFHJtI5gcr0J7t0= +google.golang.org/api v0.215.0/go.mod h1:fta3CVtuJYOEdugLNWm6WodzOS8KdFckABwN4I40hzY= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1871,10 +1871,10 @@ google.golang.org/genproto v0.0.0-20220324131243-acbaeb5b85eb/go.mod h1:hAL49I2I google.golang.org/genproto v0.0.0-20220401170504-314d38edb7de/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= google.golang.org/genproto v0.0.0-20241118233622-e639e219e697 h1:ToEetK57OidYuqD4Q5w+vfEnPvPpuTwedCNVohYJfNk= google.golang.org/genproto v0.0.0-20241118233622-e639e219e697/go.mod h1:JJrvXBWRZaFMxBufik1a4RpFw4HhgVtBBWQeQgUj2cc= -google.golang.org/genproto/googleapis/api v0.0.0-20241118233622-e639e219e697 h1:pgr/4QbFyktUv9CtQ/Fq4gzEE6/Xs7iCXbktaGzLHbQ= -google.golang.org/genproto/googleapis/api v0.0.0-20241118233622-e639e219e697/go.mod h1:+D9ySVjN8nY8YCVjc5O7PZDIdZporIDY3KaGfJunh88= -google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576 h1:8ZmaLZE4XWrtU3MyClkYqqtl6Oegr3235h7jxsDyqCY= -google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576/go.mod h1:5uTbfoYQed2U9p3KIj2/Zzm02PYhndfdmML0qC3q3FU= +google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576 h1:CkkIfIt50+lT6NHAVoRYEyAvQGFM7xEwXUUywFvEb3Q= +google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576/go.mod h1:1R3kvZ1dtP3+4p4d3G8uJ8rFk/fWlScl38vanWACI08= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241223144023-3abc09e42ca8 h1:TqExAhdPaB60Ux47Cn0oLV07rGnxZzIsaRhQaqS666A= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241223144023-3abc09e42ca8/go.mod h1:lcTa1sDdWEIHMWlITnIczmw5w60CF9ffkb8Z+DVmmjA= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= From 688262e8e85692ed0656679d6ad54dfa5fd3c258 Mon Sep 17 00:00:00 2001 From: Sankeerth Date: Wed, 8 Jan 2025 13:03:18 +0530 Subject: [PATCH 08/11] chore: send sample event to reporting in async destinations (#5396) * chore: async destination reporting * chore: removing redundant properties * chore: review comments --------- Co-authored-by: Sai Sankeerth --- router/batchrouter/handle_async.go | 145 ++++++++++++++++++----------- router/batchrouter/types.go | 17 ++++ router/batchrouter/worker.go | 5 +- 3 files changed, 114 insertions(+), 53 deletions(-) diff --git a/router/batchrouter/handle_async.go b/router/batchrouter/handle_async.go index c5dfdb89d7..d15146fcff 100644 --- a/router/batchrouter/handle_async.go +++ b/router/batchrouter/handle_async.go @@ -12,6 +12,7 @@ import ( "time" "github.com/google/uuid" + "github.com/samber/lo" "github.com/tidwall/gjson" "github.com/rudderlabs/rudder-go-kit/stats" @@ -43,7 +44,11 @@ func (brt *Handle) getImportingJobs(ctx context.Context, destinationID string, l } func (brt *Handle) updateJobStatuses(ctx context.Context, destinationID string, allJobs, completedJobs []*jobsdb.JobT, statusList []*jobsdb.JobStatusT) error { - reportMetrics := brt.getReportMetrics(statusList, brt.getParamertsFromJobs(allJobs)) + reportMetrics := brt.getReportMetrics(getReportMetricsParams{ + StatusList: statusList, + ParametersMap: brt.getParamertsFromJobs(allJobs), + JobsList: allJobs, + }) parameterFilters := []jobsdb.ParameterFilterT{{Name: "destination_id", Value: destinationID}} return misc.RetryWithNotify(ctx, brt.jobsDBCommandTimeout.Load(), brt.jobdDBMaxRetries.Load(), func(ctx context.Context) error { @@ -364,7 +369,13 @@ func (brt *Handle) asyncUploadWorker(ctx context.Context) { if uploadResponse.ImportingParameters != nil && len(uploadResponse.ImportingJobIDs) > 0 { brt.asyncDestinationStruct[destinationID].UploadInProgress = true } - brt.setMultipleJobStatus(uploadResponse, true, brt.asyncDestinationStruct[destinationID].AttemptNums, brt.asyncDestinationStruct[destinationID].FirstAttemptedAts, brt.asyncDestinationStruct[destinationID].OriginalJobParameters) + brt.setMultipleJobStatus(setMultipleJobStatusParams{ + AsyncOutput: uploadResponse, + Attempted: true, + AttemptNums: brt.asyncDestinationStruct[destinationID].AttemptNums, + FirstAttemptedAts: brt.asyncDestinationStruct[destinationID].FirstAttemptedAts, + OriginalJobParameters: brt.asyncDestinationStruct[destinationID].OriginalJobParameters, + }) brt.asyncStructCleanUp(destinationID) } brt.asyncDestinationStruct[destinationID].UploadMutex.Unlock() @@ -452,7 +463,13 @@ func (brt *Handle) sendJobsToStorage(batchJobs BatchedJobs) { out.SuccessResponse = fmt.Sprintf(`{"error":"%s"`, rterror.DisabledEgress.Error()) // skipcq: GO-R4002 } - brt.setMultipleJobStatus(out, false, getAttemptNumbers(batchJobs.Jobs), getFirstAttemptAts(batchJobs.Jobs), getOriginalJobParameters(batchJobs.Jobs)) + brt.setMultipleJobStatus(setMultipleJobStatusParams{ + AsyncOutput: out, + AttemptNums: getAttemptNumbers(batchJobs.Jobs), + FirstAttemptedAts: getFirstAttemptAts(batchJobs.Jobs), + OriginalJobParameters: getOriginalJobParameters(batchJobs.Jobs), + JobsList: batchJobs.Jobs, + }) return } @@ -469,7 +486,13 @@ func (brt *Handle) sendJobsToStorage(batchJobs BatchedJobs) { out.FailedReason = `Jobs flowed over the prescribed limit` } - brt.setMultipleJobStatus(out, false, getAttemptNumbers(batchJobs.Jobs), getFirstAttemptAts(batchJobs.Jobs), getOriginalJobParameters(batchJobs.Jobs)) + brt.setMultipleJobStatus(setMultipleJobStatusParams{ + AsyncOutput: out, + AttemptNums: getAttemptNumbers(batchJobs.Jobs), + FirstAttemptedAts: getFirstAttemptAts(batchJobs.Jobs), + OriginalJobParameters: getOriginalJobParameters(batchJobs.Jobs), + JobsList: batchJobs.Jobs, + }) return } } @@ -533,7 +556,15 @@ func (brt *Handle) sendJobsToStorage(batchJobs BatchedJobs) { out.FailedReason = `Jobs flowed over the prescribed limit` } - brt.setMultipleJobStatus(out, false, getAttemptNumbers(batchJobs.Jobs), getFirstAttemptAts(batchJobs.Jobs), getOriginalJobParameters(batchJobs.Jobs)) + brt.setMultipleJobStatus( + setMultipleJobStatusParams{ + AsyncOutput: out, + AttemptNums: getAttemptNumbers(batchJobs.Jobs), + FirstAttemptedAts: getFirstAttemptAts(batchJobs.Jobs), + OriginalJobParameters: getOriginalJobParameters(batchJobs.Jobs), + JobsList: batchJobs.Jobs, + }, + ) } newAttemptNums := getAttemptNumbers(batchJobs.Jobs) @@ -560,17 +591,20 @@ func (brt *Handle) createFakeJob(jobID int64, parameters stdjson.RawMessage) *jo } } -func (brt *Handle) getReportMetrics(statusList []*jobsdb.JobStatusT, parametersMap map[int64]stdjson.RawMessage) []*utilTypes.PUReportedMetric { +func (brt *Handle) getReportMetrics(params getReportMetricsParams) []*utilTypes.PUReportedMetric { reportMetrics := make([]*utilTypes.PUReportedMetric, 0) connectionDetailsMap := make(map[string]*utilTypes.ConnectionDetails) transformedAtMap := make(map[string]string) statusDetailsMap := make(map[string]*utilTypes.StatusDetail) routerWorkspaceJobStatusCount := make(map[string]int) - for _, status := range statusList { + jobsMap := lo.SliceToMap(params.JobsList, func(j *jobsdb.JobT) (int64, *jobsdb.JobT) { + return j.JobID, j + }) + for _, status := range params.StatusList { var parameters routerutils.JobParameters - err := json.Unmarshal(parametersMap[status.JobID], ¶meters) + err := json.Unmarshal(params.ParametersMap[status.JobID], ¶meters) if err != nil { - brt.logger.Error("Unmarshal of job parameters failed. ", string(parametersMap[status.JobID])) + brt.logger.Error("Unmarshal of job parameters failed. ", string(params.ParametersMap[status.JobID])) } workspaceID := status.WorkspaceId eventName := parameters.EventName @@ -598,6 +632,9 @@ func (brt *Handle) getReportMetrics(statusList []*jobsdb.JobStatusT, parametersM errorCode = 0 } sampleEvent := routerutils.EmptyPayload + if job, ok := jobsMap[status.JobID]; ok { + sampleEvent = job.EventPayload + } sd = &utilTypes.StatusDetail{ Status: status.JobState, StatusCode: errorCode, @@ -647,105 +684,105 @@ func (brt *Handle) getReportMetrics(statusList []*jobsdb.JobStatusT, parametersM return reportMetrics } -func (brt *Handle) setMultipleJobStatus(asyncOutput common.AsyncUploadOutput, attempted bool, attemptNums map[int64]int, firstAttemptedAts map[int64]time.Time, originalJobParameters map[int64]stdjson.RawMessage) { - workspaceID := brt.GetWorkspaceIDForDestID(asyncOutput.DestinationID) +func (brt *Handle) setMultipleJobStatus(params setMultipleJobStatusParams) { + workspaceID := brt.GetWorkspaceIDForDestID(params.AsyncOutput.DestinationID) var completedJobsList []*jobsdb.JobT var statusList []*jobsdb.JobStatusT jobIDConnectionDetailsMap := make(map[int64]jobsdb.ConnectionDetails) - if len(asyncOutput.ImportingJobIDs) > 0 { - for _, jobId := range asyncOutput.ImportingJobIDs { + if len(params.AsyncOutput.ImportingJobIDs) > 0 { + for _, jobId := range params.AsyncOutput.ImportingJobIDs { jobIDConnectionDetailsMap[jobId] = jobsdb.ConnectionDetails{ - DestinationID: asyncOutput.DestinationID, - SourceID: gjson.GetBytes(originalJobParameters[jobId], "source_id").String(), + DestinationID: params.AsyncOutput.DestinationID, + SourceID: gjson.GetBytes(params.OriginalJobParameters[jobId], "source_id").String(), } status := jobsdb.JobStatusT{ JobID: jobId, JobState: jobsdb.Importing.State, - AttemptNum: attemptNums[jobId] + 1, + AttemptNum: params.AttemptNums[jobId] + 1, ExecTime: time.Now(), RetryTime: time.Now(), ErrorCode: "200", - ErrorResponse: routerutils.EnhanceJsonWithTime(firstAttemptedAts[jobId], "firstAttemptedAt", routerutils.EmptyPayload), - Parameters: asyncOutput.ImportingParameters, - JobParameters: originalJobParameters[jobId], + ErrorResponse: routerutils.EnhanceJsonWithTime(params.FirstAttemptedAts[jobId], "firstAttemptedAt", routerutils.EmptyPayload), + Parameters: params.AsyncOutput.ImportingParameters, + JobParameters: params.OriginalJobParameters[jobId], WorkspaceId: workspaceID, } statusList = append(statusList, &status) } } - if len(asyncOutput.SucceededJobIDs) > 0 { - for _, jobId := range asyncOutput.SucceededJobIDs { + if len(params.AsyncOutput.SucceededJobIDs) > 0 { + for _, jobId := range params.AsyncOutput.SucceededJobIDs { jobIDConnectionDetailsMap[jobId] = jobsdb.ConnectionDetails{ - DestinationID: asyncOutput.DestinationID, - SourceID: gjson.GetBytes(originalJobParameters[jobId], "source_id").String(), + DestinationID: params.AsyncOutput.DestinationID, + SourceID: gjson.GetBytes(params.OriginalJobParameters[jobId], "source_id").String(), } status := jobsdb.JobStatusT{ JobID: jobId, JobState: jobsdb.Succeeded.State, - AttemptNum: attemptNums[jobId], + AttemptNum: params.AttemptNums[jobId], ExecTime: time.Now(), RetryTime: time.Now(), ErrorCode: "200", - ErrorResponse: routerutils.EnhanceJsonWithTime(firstAttemptedAts[jobId], "firstAttemptedAt", stdjson.RawMessage(asyncOutput.SuccessResponse)), + ErrorResponse: routerutils.EnhanceJsonWithTime(params.FirstAttemptedAts[jobId], "firstAttemptedAt", stdjson.RawMessage(params.AsyncOutput.SuccessResponse)), Parameters: routerutils.EmptyPayload, - JobParameters: originalJobParameters[jobId], + JobParameters: params.OriginalJobParameters[jobId], WorkspaceId: workspaceID, } statusList = append(statusList, &status) - completedJobsList = append(completedJobsList, brt.createFakeJob(jobId, originalJobParameters[jobId])) + completedJobsList = append(completedJobsList, brt.createFakeJob(jobId, params.OriginalJobParameters[jobId])) } } - if len(asyncOutput.FailedJobIDs) > 0 { - for _, jobId := range asyncOutput.FailedJobIDs { + if len(params.AsyncOutput.FailedJobIDs) > 0 { + for _, jobId := range params.AsyncOutput.FailedJobIDs { jobIDConnectionDetailsMap[jobId] = jobsdb.ConnectionDetails{ - DestinationID: asyncOutput.DestinationID, - SourceID: gjson.GetBytes(originalJobParameters[jobId], "source_id").String(), + DestinationID: params.AsyncOutput.DestinationID, + SourceID: gjson.GetBytes(params.OriginalJobParameters[jobId], "source_id").String(), } - resp := misc.UpdateJSONWithNewKeyVal(routerutils.EmptyPayload, "error", asyncOutput.FailedReason) + resp := misc.UpdateJSONWithNewKeyVal(routerutils.EmptyPayload, "error", params.AsyncOutput.FailedReason) status := jobsdb.JobStatusT{ JobID: jobId, JobState: jobsdb.Failed.State, - AttemptNum: attemptNums[jobId], + AttemptNum: params.AttemptNums[jobId], ExecTime: time.Now(), RetryTime: time.Now(), ErrorCode: "500", - ErrorResponse: routerutils.EnhanceJsonWithTime(firstAttemptedAts[jobId], "firstAttemptedAt", resp), + ErrorResponse: routerutils.EnhanceJsonWithTime(params.FirstAttemptedAts[jobId], "firstAttemptedAt", resp), Parameters: routerutils.EmptyPayload, - JobParameters: originalJobParameters[jobId], + JobParameters: params.OriginalJobParameters[jobId], WorkspaceId: workspaceID, } - if attempted { - status.AttemptNum = attemptNums[jobId] + 1 + if params.Attempted { + status.AttemptNum = params.AttemptNums[jobId] + 1 } if brt.retryLimitReached(&status) { status.JobState = jobsdb.Aborted.State - completedJobsList = append(completedJobsList, brt.createFakeJob(jobId, originalJobParameters[jobId])) + completedJobsList = append(completedJobsList, brt.createFakeJob(jobId, params.OriginalJobParameters[jobId])) } statusList = append(statusList, &status) } } - if len(asyncOutput.AbortJobIDs) > 0 { - for _, jobId := range asyncOutput.AbortJobIDs { + if len(params.AsyncOutput.AbortJobIDs) > 0 { + for _, jobId := range params.AsyncOutput.AbortJobIDs { jobIDConnectionDetailsMap[jobId] = jobsdb.ConnectionDetails{ - DestinationID: asyncOutput.DestinationID, - SourceID: gjson.GetBytes(originalJobParameters[jobId], "source_id").String(), + DestinationID: params.AsyncOutput.DestinationID, + SourceID: gjson.GetBytes(params.OriginalJobParameters[jobId], "source_id").String(), } - resp := misc.UpdateJSONWithNewKeyVal(routerutils.EmptyPayload, "error", asyncOutput.AbortReason) + resp := misc.UpdateJSONWithNewKeyVal(routerutils.EmptyPayload, "error", params.AsyncOutput.AbortReason) status := jobsdb.JobStatusT{ JobID: jobId, JobState: jobsdb.Aborted.State, - AttemptNum: attemptNums[jobId], + AttemptNum: params.AttemptNums[jobId], ExecTime: time.Now(), RetryTime: time.Now(), ErrorCode: "400", - ErrorResponse: routerutils.EnhanceJsonWithTime(firstAttemptedAts[jobId], "firstAttemptedAt", stdjson.RawMessage(resp)), + ErrorResponse: routerutils.EnhanceJsonWithTime(params.FirstAttemptedAts[jobId], "firstAttemptedAt", stdjson.RawMessage(resp)), Parameters: routerutils.EmptyPayload, - JobParameters: originalJobParameters[jobId], + JobParameters: params.OriginalJobParameters[jobId], WorkspaceId: workspaceID, } statusList = append(statusList, &status) - completedJobsList = append(completedJobsList, brt.createFakeJob(jobId, originalJobParameters[jobId])) + completedJobsList = append(completedJobsList, brt.createFakeJob(jobId, params.OriginalJobParameters[jobId])) } } @@ -756,11 +793,15 @@ func (brt *Handle) setMultipleJobStatus(asyncOutput common.AsyncUploadOutput, at parameterFilters := []jobsdb.ParameterFilterT{ { Name: "destination_id", - Value: asyncOutput.DestinationID, + Value: params.AsyncOutput.DestinationID, }, } - reportMetrics := brt.getReportMetrics(statusList, originalJobParameters) + reportMetrics := brt.getReportMetrics(getReportMetricsParams{ + StatusList: statusList, + ParametersMap: params.OriginalJobParameters, + JobsList: params.JobsList, + }) // Mark the status of the jobs err := misc.RetryWithNotify(context.Background(), brt.jobsDBCommandTimeout.Load(), brt.jobdDBMaxRetries.Load(), func(ctx context.Context) error { @@ -794,12 +835,12 @@ func (brt *Handle) setMultipleJobStatus(asyncOutput common.AsyncUploadOutput, at brt.destType, float64(len(completedJobsList)), ) - if attempted { + if params.Attempted { var sourceID string if len(statusList) > 0 { - sourceID = gjson.GetBytes(originalJobParameters[statusList[0].JobID], "source_id").String() + sourceID = gjson.GetBytes(params.OriginalJobParameters[statusList[0].JobID], "source_id").String() } - brt.recordAsyncDestinationDeliveryStatus(sourceID, asyncOutput.DestinationID, statusList) + brt.recordAsyncDestinationDeliveryStatus(sourceID, params.AsyncOutput.DestinationID, statusList) } } diff --git a/router/batchrouter/types.go b/router/batchrouter/types.go index 0df69805be..5635dbc5fc 100644 --- a/router/batchrouter/types.go +++ b/router/batchrouter/types.go @@ -1,10 +1,12 @@ package batchrouter import ( + stdjson "encoding/json" "time" backendconfig "github.com/rudderlabs/rudder-server/backend-config" "github.com/rudderlabs/rudder-server/jobsdb" + "github.com/rudderlabs/rudder-server/router/batchrouter/asyncdestinationmanager/common" router_utils "github.com/rudderlabs/rudder-server/router/utils" ) @@ -59,3 +61,18 @@ type BatchedJobs struct { TimeWindow time.Time JobState string // ENUM waiting, executing, succeeded, waiting_retry, filtered, failed, aborted, migrating, migrated, wont_migrate } + +type getReportMetricsParams struct { + StatusList []*jobsdb.JobStatusT + ParametersMap map[int64]stdjson.RawMessage + JobsList []*jobsdb.JobT +} + +type setMultipleJobStatusParams struct { + AsyncOutput common.AsyncUploadOutput + Attempted bool + AttemptNums map[int64]int + FirstAttemptedAts map[int64]time.Time + OriginalJobParameters map[int64]stdjson.RawMessage + JobsList []*jobsdb.JobT +} diff --git a/router/batchrouter/worker.go b/router/batchrouter/worker.go index c38cc5d9c2..fbe539cf4e 100644 --- a/router/batchrouter/worker.go +++ b/router/batchrouter/worker.go @@ -140,7 +140,10 @@ func (w *worker) processJobAsync(jobsWg *sync.WaitGroup, destinationJobs *Destin if err != nil { panic(fmt.Errorf("storing %s jobs into ErrorDB: %w", brt.destType, err)) } - reportMetrics := brt.getReportMetrics(drainList, brt.getParamertsFromJobs(drainJobList)) + reportMetrics := brt.getReportMetrics(getReportMetricsParams{ + StatusList: drainList, + ParametersMap: brt.getParamertsFromJobs(drainJobList), + }) err = misc.RetryWithNotify(context.Background(), brt.jobsDBCommandTimeout.Load(), brt.jobdDBMaxRetries.Load(), func(ctx context.Context) error { return brt.jobsDB.WithUpdateSafeTx(ctx, func(tx jobsdb.UpdateSafeTx) error { err := brt.jobsDB.UpdateJobStatusInTx(ctx, tx, drainList, []string{brt.destType}, parameterFilters) From 5ad01b1ebebd3c32db2abb95f4d08baa185b95e1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 8 Jan 2025 13:28:52 +0530 Subject: [PATCH 09/11] chore(deps): bump the go-deps group across 1 directory with 5 updates (#5417) --- go.mod | 42 +++++++++++++-------------- go.sum | 91 +++++++++++++++++++++++++++++----------------------------- 2 files changed, 67 insertions(+), 66 deletions(-) diff --git a/go.mod b/go.mod index 0750085140..4a014e9a7b 100644 --- a/go.mod +++ b/go.mod @@ -35,10 +35,10 @@ require ( github.com/apache/pulsar-client-go v0.14.0 github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de github.com/aws/aws-sdk-go v1.55.5 - github.com/bufbuild/httplb v0.3.0 + github.com/bufbuild/httplb v0.3.1 github.com/cenkalti/backoff v2.2.1+incompatible github.com/cenkalti/backoff/v4 v4.3.0 - github.com/confluentinc/confluent-kafka-go/v2 v2.6.1 + github.com/confluentinc/confluent-kafka-go/v2 v2.8.0 github.com/databricks/databricks-sql-go v1.6.1 github.com/denisenkom/go-mssqldb v0.12.3 github.com/dgraph-io/badger/v4 v4.5.0 @@ -63,7 +63,7 @@ require ( github.com/lib/pq v1.10.9 github.com/linkedin/goavro/v2 v2.13.0 github.com/marcboeker/go-duckdb v1.8.3 - github.com/minio/minio-go/v7 v7.0.82 + github.com/minio/minio-go/v7 v7.0.83 github.com/mitchellh/mapstructure v1.5.0 github.com/olekukonko/tablewriter v0.0.5 github.com/onsi/ginkgo/v2 v2.22.2 @@ -77,7 +77,7 @@ require ( github.com/rudderlabs/analytics-go v3.3.3+incompatible github.com/rudderlabs/bing-ads-go-sdk v0.2.3 github.com/rudderlabs/compose-test v0.1.3 - github.com/rudderlabs/rudder-go-kit v0.45.0 + github.com/rudderlabs/rudder-go-kit v0.45.1 github.com/rudderlabs/rudder-observability-kit v0.0.3 github.com/rudderlabs/rudder-schemas v0.5.4 github.com/rudderlabs/rudder-transformer/go v0.0.0-20240910055720-f77d2ab4125a @@ -111,7 +111,7 @@ require ( google.golang.org/api v0.215.0 google.golang.org/genproto/googleapis/rpc v0.0.0-20241223144023-3abc09e42ca8 google.golang.org/grpc v1.68.1 - google.golang.org/protobuf v1.36.1 + google.golang.org/protobuf v1.36.2 ) require ( @@ -205,7 +205,7 @@ require ( github.com/evanphx/json-patch v5.9.0+incompatible github.com/fatih/color v1.17.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect - github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/fsnotify/fsnotify v1.8.0 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect github.com/gabriel-vasile/mimetype v1.4.5 // indirect github.com/go-ini/ini v1.67.0 // indirect @@ -219,7 +219,7 @@ require ( github.com/go-sql-driver/mysql v1.8.1 // indirect github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/go-viper/mapstructure/v2 v2.1.0 // indirect - github.com/goccy/go-json v0.10.3 // indirect + github.com/goccy/go-json v0.10.4 // indirect github.com/goccy/go-reflect v1.2.0 // indirect github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 // indirect github.com/gogo/protobuf v1.3.2 // indirect @@ -255,7 +255,7 @@ require ( github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/klauspost/asmfmt v1.3.2 // indirect github.com/klauspost/compress v1.17.11 // indirect - github.com/klauspost/cpuid/v2 v2.2.8 // indirect + github.com/klauspost/cpuid/v2 v2.2.9 // indirect github.com/kr/fs v0.1.0 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/lufia/plan9stats v0.0.0-20240909124753-873cd0166683 // indirect @@ -283,12 +283,12 @@ require ( github.com/pierrec/lz4/v4 v4.1.21 // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/pkg/sftp v1.13.6 // indirect + github.com/pkg/sftp v1.13.7 // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect - github.com/prometheus/client_golang v1.20.4 // indirect - github.com/prometheus/common v0.59.1 // indirect + github.com/prometheus/client_golang v1.20.5 // indirect + github.com/prometheus/common v0.60.1 // indirect github.com/prometheus/procfs v0.15.1 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/rs/xid v1.6.0 // indirect @@ -326,16 +326,16 @@ require ( go.opentelemetry.io/contrib/detectors/gcp v1.29.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 // indirect - go.opentelemetry.io/otel v1.30.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.30.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.30.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.30.0 // indirect - go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.30.0 // indirect - go.opentelemetry.io/otel/exporters/zipkin v1.30.0 // indirect - go.opentelemetry.io/otel/metric v1.30.0 // indirect - go.opentelemetry.io/otel/sdk v1.30.0 // indirect - go.opentelemetry.io/otel/sdk/metric v1.30.0 // indirect - go.opentelemetry.io/otel/trace v1.30.0 // indirect + go.opentelemetry.io/otel v1.32.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.32.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.32.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.32.0 // indirect + go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.32.0 // indirect + go.opentelemetry.io/otel/exporters/zipkin v1.32.0 // indirect + go.opentelemetry.io/otel/metric v1.32.0 // indirect + go.opentelemetry.io/otel/sdk v1.32.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.32.0 // indirect + go.opentelemetry.io/otel/trace v1.32.0 // indirect go.opentelemetry.io/proto/otlp v1.3.1 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect diff --git a/go.sum b/go.sum index 85cea23eb8..caa834d618 100644 --- a/go.sum +++ b/go.sum @@ -357,8 +357,8 @@ github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdb github.com/bsm/gomega v1.26.0/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= -github.com/bufbuild/httplb v0.3.0 h1:sCMPD+89ydD3atcVareDsiv/kUT+pLHolENMoCGZJV8= -github.com/bufbuild/httplb v0.3.0/go.mod h1:qDNs7dSFxIhKi/DA/rCCPVzbQfHs1JVxPMl9EvrbL4Q= +github.com/bufbuild/httplb v0.3.1 h1:eY3bDouZyqcyEdUL4/NibxoVh7mXCMKVne1859TDxwQ= +github.com/bufbuild/httplb v0.3.1/go.mod h1:oMeYRvMM4jbtYhwIwWwKnrnWxe2eeXrp2sqEqaLkuJs= github.com/buger/goterm v1.0.4 h1:Z9YvGmOih81P0FbVtEYTFF6YsSgxSUKEhf/f9bTMXbY= github.com/buger/goterm v1.0.4/go.mod h1:HiFWV3xnkolgrBV3mY8m0X0Pumt4zg4QhbdOzQtB8tE= github.com/bytedance/sonic v1.11.6/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4= @@ -398,8 +398,8 @@ github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMe github.com/colinmarc/hdfs/v2 v2.1.1/go.mod h1:M3x+k8UKKmxtFu++uAZ0OtDU8jR3jnaZIAc6yK4Ue0c= github.com/compose-spec/compose-go/v2 v2.1.3 h1:bD67uqLuL/XgkAK6ir3xZvNLFPxPScEi1KW7R5esrLE= github.com/compose-spec/compose-go/v2 v2.1.3/go.mod h1:lFN0DrMxIncJGYAXTfWuajfwj5haBJqrBkarHcnjJKc= -github.com/confluentinc/confluent-kafka-go/v2 v2.6.1 h1:XFkytnGvk/ZcY2qU0ql4E4h+ftBaGqkLO7tlZ4kRbr4= -github.com/confluentinc/confluent-kafka-go/v2 v2.6.1/go.mod h1:hScqtFIGUI1wqHIgM3mjoqEou4VweGGGX7dMpcUKves= +github.com/confluentinc/confluent-kafka-go/v2 v2.8.0 h1:0HlcSNWg4LpLA9nIjzUMIqWHI+w0S68UN7alXAc3TeA= +github.com/confluentinc/confluent-kafka-go/v2 v2.8.0/go.mod h1:hScqtFIGUI1wqHIgM3mjoqEou4VweGGGX7dMpcUKves= github.com/containerd/console v1.0.4 h1:F2g4+oChYvBTsASRTz8NP6iIAi97J3TtSAsLbIFn4ro= github.com/containerd/console v1.0.4/go.mod h1:YynlIjWYF8myEu6sdkwKIvGQq+cOckRm6So2avqoYAk= github.com/containerd/containerd v1.6.32 h1:zc3RN2fLrPZPH4mZziyaRVIqCN6zoFpeN0SOVDEgCTA= @@ -531,10 +531,11 @@ github.com/fsnotify/fsevents v0.2.0/go.mod h1:B3eEk39i4hz8y1zaWS/wPrAP4O6wkIl7HQ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= -github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= -github.com/fsouza/fake-gcs-server v1.50.0 h1:MRW1OuLyHnFKKLGNILiH+x6CMKqn/R05auJ5ET5PCyk= -github.com/fsouza/fake-gcs-server v1.50.0/go.mod h1:itn0kDInXYbYXZ+2dLch83bR8lpp7YQ5czkZnH6IRH8= +github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M= +github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/fsouza/fake-gcs-server v1.50.2 h1:ulrS1pavCOCbMZfN5ZPgBRMFWclON9xDsuLBniXtQoE= +github.com/fsouza/fake-gcs-server v1.50.2/go.mod h1:VU6Zgei4647KuT4XER8WHv5Hcj2NIySndyG8gfvwckA= github.com/fvbommel/sortorder v1.0.2 h1:mV4o8B2hKboCdkJm+a7uX/SIpZob4JzUpc5GGnM45eo= github.com/fvbommel/sortorder v1.0.2/go.mod h1:uk88iVf1ovNn1iLfgUVU2F9o5eO30ui720w+kxuqRs0= github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= @@ -611,8 +612,8 @@ github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22 github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= -github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= -github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/goccy/go-json v0.10.4 h1:JSwxQzIqKfmFX1swYPpUThQZp/Ka4wzJdK0LWVytLPM= +github.com/goccy/go-json v0.10.4/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/goccy/go-reflect v1.2.0 h1:O0T8rZCuNmGXewnATuKYnkL0xm6o8UNOJZd/gOkb9ms= github.com/goccy/go-reflect v1.2.0/go.mod h1:n0oYZn8VcV2CkWTxi8B9QjkCoq6GTtCEdfmR66YhFtE= github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 h1:ZpnhV/YsD2/4cESfV5+Hoeu/iUR3ruzNvZ+yQfO03a0= @@ -935,8 +936,8 @@ github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa02 github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.1.0/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= -github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM= -github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/klauspost/cpuid/v2 v2.2.9 h1:66ze0taIn2H33fBvCkXuv9BmCwDfafmiIVpKV9kKGuY= +github.com/klauspost/cpuid/v2 v2.2.9/go.mod h1:rqkxqrZ1EhYM9G+hXH7YdowN5R5RGN6NK4QwQ3WMXF8= github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -1009,8 +1010,8 @@ github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8Ie github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= github.com/minio/minio-go/v7 v7.0.34/go.mod h1:nCrRzjoSUQh8hgKKtu3Y708OLvRLtuASMg2/nvmbarw= -github.com/minio/minio-go/v7 v7.0.82 h1:tWfICLhmp2aFPXL8Tli0XDTHj2VB/fNf0PC1f/i1gRo= -github.com/minio/minio-go/v7 v7.0.82/go.mod h1:84gmIilaX4zcvAWWzJ5Z1WI5axN+hAbM5w25xf8xvC0= +github.com/minio/minio-go/v7 v7.0.83 h1:W4Kokksvlz3OKf3OqIlzDNKd4MERlC2oN8YptwJ0+GA= +github.com/minio/minio-go/v7 v7.0.83/go.mod h1:57YXpvc5l3rjPdhqNrDsvVlY0qPI6UTk1bflAe+9doY= github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= @@ -1116,8 +1117,8 @@ github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/sftp v1.13.6 h1:JFZT4XbOU7l77xGSpOdW+pwIMqP044IyjXX6FGyEKFo= -github.com/pkg/sftp v1.13.6/go.mod h1:tz1ryNURKu77RL+GuCzmoJYxQczL3wLNNpPWagdg4Qk= +github.com/pkg/sftp v1.13.7 h1:uv+I3nNJvlKZIQGSr8JVQLNHFU9YhhNpvC14Y6KgmSM= +github.com/pkg/sftp v1.13.7/go.mod h1:KMKI0t3T6hfA+lTR/ssZdunHo+uwq7ghoN09/FSu3DY= github.com/pkg/xattr v0.4.10 h1:Qe0mtiNFHQZ296vRgUjRCoPHPqH7VdTOrZx3g0T+pGA= github.com/pkg/xattr v0.4.10/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU= github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= @@ -1129,13 +1130,13 @@ github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g= github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U= -github.com/prometheus/client_golang v1.20.4 h1:Tgh3Yr67PaOv/uTqloMsCEdeuFTatm5zIq5+qNN23vI= -github.com/prometheus/client_golang v1.20.4/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= +github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= +github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= -github.com/prometheus/common v0.59.1 h1:LXb1quJHWm1P6wq/U824uxYi4Sg0oGvNeUm1z5dJoX0= -github.com/prometheus/common v0.59.1/go.mod h1:GpWM7dewqmVYcd7SmRaiWVe9SSqjf0UrwnYnpEZNuT0= +github.com/prometheus/common v0.60.1 h1:FUas6GcOw66yB/73KC+BOZoFJmbo/1pojoILArPAaSc= +github.com/prometheus/common v0.60.1/go.mod h1:h0LYf1R1deLSKtD4Vdg8gy4RuOvENW2J/h19V5NADQw= github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/r3labs/sse v0.0.0-20210224172625-26fe804710bc h1:zAsgcP8MhzAbhMnB1QQ2O7ZhWYVGYSR2iVcjzQuPV+o= @@ -1150,8 +1151,8 @@ github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUc github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= -github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= -github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= github.com/rs/cors v1.11.1 h1:eU3gRzXLRK57F5rKMGMZURNdIG4EoAmX8k94r9wXWHA= github.com/rs/cors v1.11.1/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= @@ -1173,8 +1174,8 @@ github.com/rudderlabs/goqu/v10 v10.3.1 h1:rnfX+b4EwBWQ2UQfIGeEW299JBBkK5biEbnf7K github.com/rudderlabs/goqu/v10 v10.3.1/go.mod h1:LH2vI5gGHBxEQuESqFyk5ZA2anGINc8o25hbidDWOYw= github.com/rudderlabs/parquet-go v0.0.2 h1:ZXRdZdimB0PdJtmxeSSxfI0fDQ3kZjwzBxRi6Ut1J8k= github.com/rudderlabs/parquet-go v0.0.2/go.mod h1:g6guum7o8uhj/uNhunnt7bw5Vabu/goI5i21/3fnxWQ= -github.com/rudderlabs/rudder-go-kit v0.45.0 h1:y8ModVsl2rAdqnqv/di82cddkaEBT1qZvpNNh0ZLfYQ= -github.com/rudderlabs/rudder-go-kit v0.45.0/go.mod h1:NrHCi0KSzHSMFXQu0t2kgJcE4ClAKklVXfb2glADvQ4= +github.com/rudderlabs/rudder-go-kit v0.45.1 h1:dC2MGDkaehsTfNtUI1juHpTGaDa5iE3YFCwdniRbFyg= +github.com/rudderlabs/rudder-go-kit v0.45.1/go.mod h1:6+7vmN8DV2RXfT9UqbV/BeGqvuPsBzgKitBdvaY6zIc= github.com/rudderlabs/rudder-observability-kit v0.0.3 h1:vZtuZRkGX+6rjaeKtxxFE2YYP6QlmAcVcgecTOjvz+Q= github.com/rudderlabs/rudder-observability-kit v0.0.3/go.mod h1:6UjAh3H6rkE0fFLh7z8ZGQEQbKtUkRfhWOf/OUhfqW8= github.com/rudderlabs/rudder-schemas v0.5.4 h1:QzI6vIC38W0jGJu6E0vdwh4IAtSKx8ziqff+JL0xmIE= @@ -1383,34 +1384,34 @@ go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0. go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 h1:TT4fX+nBOA/+LUkobKGW1ydGcn+G3vRw9+g5HwCphpk= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0/go.mod h1:L7UH0GbB0p47T4Rri3uHjbpCFYrVrwc1I25QhNPiGK8= go.opentelemetry.io/otel v0.14.0/go.mod h1:vH5xEuwy7Rts0GNtsCW3HYQoZDY+OmBJ6t1bFGGlxgw= -go.opentelemetry.io/otel v1.30.0 h1:F2t8sK4qf1fAmY9ua4ohFS/K+FUuOPemHUIXHtktrts= -go.opentelemetry.io/otel v1.30.0/go.mod h1:tFw4Br9b7fOS+uEao81PJjVMjW/5fvNCbpsDIXqP0pc= +go.opentelemetry.io/otel v1.32.0 h1:WnBN+Xjcteh0zdk01SVqV55d/m62NJLJdIyb4y/WO5U= +go.opentelemetry.io/otel v1.32.0/go.mod h1:00DCVSB0RQcnzlwyTfqtxSm+DRr9hpYrHjNGiBHVQIg= go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.42.0 h1:ZtfnDL+tUrs1F0Pzfwbg2d59Gru9NCH3bgSHBM6LDwU= go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.42.0/go.mod h1:hG4Fj/y8TR/tlEDREo8tWstl9fO9gcFkn4xrx0Io8xU= -go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.30.0 h1:WypxHH02KX2poqqbaadmkMYalGyy/vil4HE4PM4nRJc= -go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.30.0/go.mod h1:U79SV99vtvGSEBeeHnpgGJfTsnsdkWLpPN/CcHAzBSI= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.32.0 h1:j7ZSD+5yn+lo3sGV69nW04rRR0jhYnBwjuX3r0HvnK0= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.32.0/go.mod h1:WXbYJTUaZXAbYd8lbgGuvih0yuCfOFC5RJoYnoLcGz8= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.42.0 h1:wNMDy/LVGLj2h3p6zg4d0gypKfWKSWI14E1C4smOgl8= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.42.0/go.mod h1:YfbDdXAAkemWJK3H/DshvlrxqFB2rtW4rY6ky/3x/H0= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.30.0 h1:lsInsfvhVIfOI6qHVyysXMNDnjO9Npvl7tlDPJFBVd4= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.30.0/go.mod h1:KQsVNh4OjgjTG0G6EiNi1jVpnaeeKsKMRwbLN+f1+8M= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.30.0 h1:m0yTiGDLUvVYaTFbAvCkVYIYcvwKt3G7OLoN77NUs/8= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.30.0/go.mod h1:wBQbT4UekBfegL2nx0Xk1vBcnzyBPsIVm9hRG4fYcr4= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.32.0 h1:IJFEoHiytixx8cMiVAO+GmHR6Frwu+u5Ur8njpFO6Ac= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.32.0/go.mod h1:3rHrKNtLIoS0oZwkY2vxi+oJcwFRWdtUyRII+so45p8= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.32.0 h1:9kV11HXBHZAvuPUZxmMWrH8hZn/6UnHX4K0mu36vNsU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.32.0/go.mod h1:JyA0FHXe22E1NeNiHmVp7kFHglnexDQ7uRWDiiJ1hKQ= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.21.0 h1:digkEZCJWobwBqMwC0cwCq8/wkkRy/OowZg5OArWZrM= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.21.0/go.mod h1:/OpE/y70qVkndM0TrxT4KBoN3RsFZP0QaofcfYrj76I= go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.29.0 h1:WDdP9acbMYjbKIyJUhTvtzj601sVJOqgWdUxSdR/Ysc= go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.29.0/go.mod h1:BLbf7zbNIONBLPwvFnwNHGj4zge8uTCM/UPIVW1Mq2I= -go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.30.0 h1:kn1BudCgwtE7PxLqcZkErpD8GKqLZ6BSzeW9QihQJeM= -go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.30.0/go.mod h1:ljkUDtAMdleoi9tIG1R6dJUpVwDcYjw3J2Q6Q/SuiC0= -go.opentelemetry.io/otel/exporters/zipkin v1.30.0 h1:1uYaSfxiCLdJATlGEtYjQe4jZYfqCjVwxeSTMXe8VF4= -go.opentelemetry.io/otel/exporters/zipkin v1.30.0/go.mod h1:r/4BhMc3kiKxD61wGh9J3NVQ3/cZ45F2NHkQgVnql48= -go.opentelemetry.io/otel/metric v1.30.0 h1:4xNulvn9gjzo4hjg+wzIKG7iNFEaBMX00Qd4QIZs7+w= -go.opentelemetry.io/otel/metric v1.30.0/go.mod h1:aXTfST94tswhWEb+5QjlSqG+cZlmyXy/u8jFpor3WqQ= -go.opentelemetry.io/otel/sdk v1.30.0 h1:cHdik6irO49R5IysVhdn8oaiR9m8XluDaJAs4DfOrYE= -go.opentelemetry.io/otel/sdk v1.30.0/go.mod h1:p14X4Ok8S+sygzblytT1nqG98QG2KYKv++HE0LY/mhg= -go.opentelemetry.io/otel/sdk/metric v1.30.0 h1:QJLT8Pe11jyHBHfSAgYH7kEmT24eX792jZO1bo4BXkM= -go.opentelemetry.io/otel/sdk/metric v1.30.0/go.mod h1:waS6P3YqFNzeP01kuo/MBBYqaoBJl7efRQHOaydhy1Y= -go.opentelemetry.io/otel/trace v1.30.0 h1:7UBkkYzeg3C7kQX8VAidWh2biiQbtAKjyIML8dQ9wmc= -go.opentelemetry.io/otel/trace v1.30.0/go.mod h1:5EyKqTzzmyqB9bwtCCq6pDLktPK6fmGf/Dph+8VI02o= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.32.0 h1:cC2yDI3IQd0Udsux7Qmq8ToKAx1XCilTQECZ0KDZyTw= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.32.0/go.mod h1:2PD5Ex6z8CFzDbTdOlwyNIUywRr1DN0ospafJM1wJ+s= +go.opentelemetry.io/otel/exporters/zipkin v1.32.0 h1:6O8HgLHPXtXE9QEKEWkBImL9mEKCGEl+m+OncVO53go= +go.opentelemetry.io/otel/exporters/zipkin v1.32.0/go.mod h1:+MFvorlowjy0iWnsKaNxC1kzczSxe71mw85h4p8yEvg= +go.opentelemetry.io/otel/metric v1.32.0 h1:xV2umtmNcThh2/a/aCP+h64Xx5wsj8qqnkYZktzNa0M= +go.opentelemetry.io/otel/metric v1.32.0/go.mod h1:jH7CIbbK6SH2V2wE16W05BHCtIDzauciCRLoc/SyMv8= +go.opentelemetry.io/otel/sdk v1.32.0 h1:RNxepc9vK59A8XsgZQouW8ue8Gkb4jpWtJm9ge5lEG4= +go.opentelemetry.io/otel/sdk v1.32.0/go.mod h1:LqgegDBjKMmb2GC6/PrTnteJG39I8/vJCAP9LlJXEjU= +go.opentelemetry.io/otel/sdk/metric v1.32.0 h1:rZvFnvmvawYb0alrYkjraqJq0Z4ZUJAiyYCU9snn1CU= +go.opentelemetry.io/otel/sdk/metric v1.32.0/go.mod h1:PWeZlq0zt9YkYAp3gjKZ0eicRYvOh1Gd+X99x6GHpCQ= +go.opentelemetry.io/otel/trace v1.32.0 h1:WIC9mYrXf8TmY/EXuULKc8hR17vE+Hjv2cssQDe03fM= +go.opentelemetry.io/otel/trace v1.32.0/go.mod h1:+i4rkvCraA+tG6AzwloGaCtkx53Fa+L+V8e9a7YvhT8= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0= go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8= @@ -1924,8 +1925,8 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= -google.golang.org/protobuf v1.36.1 h1:yBPeRvTftaleIgM3PZ/WBIZ7XM/eEYAaEyCwvyjq/gk= -google.golang.org/protobuf v1.36.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +google.golang.org/protobuf v1.36.2 h1:R8FeyR1/eLmkutZOM5CWghmo5itiG9z0ktFlTVLuTmU= +google.golang.org/protobuf v1.36.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/alexcesaro/statsd.v2 v2.0.0 h1:FXkZSCZIH17vLCO5sO2UucTHsH9pc+17F6pl3JVCwMc= gopkg.in/alexcesaro/statsd.v2 v2.0.0/go.mod h1:i0ubccKGzBVNBpdGV5MocxyA/XlLUJzA7SLonnE4drU= gopkg.in/cenkalti/backoff.v1 v1.1.0 h1:Arh75ttbsvlpVA7WtVpH4u9h6Zl46xuptxqLxPiSo4Y= From 4be15cb1bcb59daf91638f1e2a4f8471822c705d Mon Sep 17 00:00:00 2001 From: shekhar-rudder <85345786+shekhar-rudder@users.noreply.github.com> Date: Wed, 8 Jan 2025 16:47:29 +0530 Subject: [PATCH 10/11] feat: implement backoff for snowpipe streaming authorization errors (#5399) --- .../snowpipestreaming/channel.go | 27 ++-- .../snowpipestreaming/snowpipestreaming.go | 95 +++++++++++++- .../snowpipestreaming_test.go | 122 ++++++++++++++++++ .../testdata/successful_user_records.txt | 2 + .../snowpipestreaming/types.go | 14 ++ 5 files changed, 242 insertions(+), 18 deletions(-) create mode 100644 router/batchrouter/asyncdestinationmanager/snowpipestreaming/testdata/successful_user_records.txt diff --git a/router/batchrouter/asyncdestinationmanager/snowpipestreaming/channel.go b/router/batchrouter/asyncdestinationmanager/snowpipestreaming/channel.go index 5e01150e44..9ed16ca00b 100644 --- a/router/batchrouter/asyncdestinationmanager/snowpipestreaming/channel.go +++ b/router/batchrouter/asyncdestinationmanager/snowpipestreaming/channel.go @@ -2,6 +2,7 @@ package snowpipestreaming import ( "context" + "errors" "fmt" "github.com/rudderlabs/rudder-go-kit/logger" @@ -12,6 +13,11 @@ import ( whutils "github.com/rudderlabs/rudder-server/warehouse/utils" ) +var ( + errAuthz = errors.New("snowpipe authorization error") + errBackoff = errors.New("snowpipe backoff error") +) + // initializeChannelWithSchema creates a new channel for the given table if it doesn't exist. // If the channel already exists, it checks for new columns and adds them to the table. // It returns the channel response after creating or recreating the channel. @@ -66,7 +72,7 @@ func (m *Manager) addColumns(ctx context.Context, namespace, tableName string, c snowflakeManager.Cleanup(ctx) }() if err = snowflakeManager.AddColumns(ctx, tableName, columns); err != nil { - return fmt.Errorf("adding column: %w", err) + return fmt.Errorf("adding column: %w, %w", errAuthz, err) } return nil } @@ -157,10 +163,10 @@ func (m *Manager) handleSchemaError( snowflakeManager.Cleanup(ctx) }() if err := snowflakeManager.CreateSchema(ctx); err != nil { - return nil, fmt.Errorf("creating schema: %w", err) + return nil, fmt.Errorf("creating schema: %w, %w", errAuthz, err) } if err := snowflakeManager.CreateTable(ctx, channelReq.TableConfig.Table, eventSchema); err != nil { - return nil, fmt.Errorf("creating table: %w", err) + return nil, fmt.Errorf("creating table: %w, %w", errAuthz, err) } return m.api.CreateChannel(ctx, channelReq) } @@ -185,7 +191,7 @@ func (m *Manager) handleTableError( snowflakeManager.Cleanup(ctx) }() if err := snowflakeManager.CreateTable(ctx, channelReq.TableConfig.Table, eventSchema); err != nil { - return nil, fmt.Errorf("creating table: %w", err) + return nil, fmt.Errorf("creating table: %w, %w", errAuthz, err) } return m.api.CreateChannel(ctx, channelReq) } @@ -225,6 +231,9 @@ func (m *Manager) deleteChannel(ctx context.Context, tableName, channelID string } func (m *Manager) createSnowflakeManager(ctx context.Context, namespace string) (manager.Manager, error) { + if m.isInBackoff() { + return nil, fmt.Errorf("skipping snowflake manager creation due to backoff: %w", errBackoff) + } modelWarehouse := whutils.ModelWarehouse{ WorkspaceID: m.destination.WorkspaceID, Destination: *m.destination, @@ -234,13 +243,5 @@ func (m *Manager) createSnowflakeManager(ctx context.Context, namespace string) } modelWarehouse.Destination.Config["useKeyPairAuth"] = true // Since we are currently only supporting key pair auth - sf, err := manager.New(whutils.SnowpipeStreaming, m.appConfig, m.logger, m.statsFactory) - if err != nil { - return nil, fmt.Errorf("creating snowflake manager: %w", err) - } - err = sf.Setup(ctx, modelWarehouse, whutils.NewNoOpUploader()) - if err != nil { - return nil, fmt.Errorf("setting up snowflake manager: %w", err) - } - return sf, nil + return m.managerCreator(ctx, modelWarehouse, m.appConfig, m.logger, m.statsFactory) } diff --git a/router/batchrouter/asyncdestinationmanager/snowpipestreaming/snowpipestreaming.go b/router/batchrouter/asyncdestinationmanager/snowpipestreaming/snowpipestreaming.go index fef146dfdb..4393395127 100644 --- a/router/batchrouter/asyncdestinationmanager/snowpipestreaming/snowpipestreaming.go +++ b/router/batchrouter/asyncdestinationmanager/snowpipestreaming/snowpipestreaming.go @@ -4,6 +4,7 @@ import ( "bufio" "context" stdjson "encoding/json" + "errors" "fmt" "net/http" "os" @@ -12,6 +13,7 @@ import ( "sync" "time" + "github.com/cenkalti/backoff/v4" "github.com/hashicorp/go-retryablehttp" jsoniter "github.com/json-iterator/go" "github.com/samber/lo" @@ -31,6 +33,7 @@ import ( "github.com/rudderlabs/rudder-server/router/batchrouter/asyncdestinationmanager/snowpipestreaming/internal/model" "github.com/rudderlabs/rudder-server/utils/misc" "github.com/rudderlabs/rudder-server/utils/timeutil" + "github.com/rudderlabs/rudder-server/warehouse/integrations/manager" whutils "github.com/rudderlabs/rudder-server/warehouse/utils" ) @@ -38,13 +41,13 @@ var json = jsoniter.ConfigCompatibleWithStandardLibrary func New( conf *config.Config, - logger logger.Logger, + log logger.Logger, statsFactory stats.Stats, destination *backendconfig.DestinationT, ) *Manager { m := &Manager{ appConfig: conf, - logger: logger.Child("snowpipestreaming").Withn( + logger: log.Child("snowpipestreaming").Withn( obskit.WorkspaceID(destination.WorkspaceID), obskit.DestinationID(destination.ID), obskit.DestinationType(destination.DestinationDefinition.Name), @@ -67,6 +70,9 @@ func New( m.config.client.retryMax = conf.GetInt("SnowpipeStreaming.Client.retryMax", 5) m.config.instanceID = conf.GetString("INSTANCE_ID", "1") m.config.maxBufferCapacity = conf.GetReloadableInt64Var(512*bytesize.KB, bytesize.B, "SnowpipeStreaming.maxBufferCapacity") + m.config.backoff.initialInterval = conf.GetReloadableDurationVar(1, time.Second, "SnowpipeStreaming.backoffInitialIntervalInSeconds") + m.config.backoff.multiplier = conf.GetReloadableFloat64Var(2.0, "SnowpipeStreaming.backoffMultiplier") + m.config.backoff.maxInterval = conf.GetReloadableDurationVar(1, time.Hour, "SnowpipeStreaming.backoffMaxIntervalInHours") tags := stats.Tags{ "module": "batch_router", @@ -100,6 +106,17 @@ func New( snowpipeapi.New(m.appConfig, m.statsFactory, m.config.client.url, m.requestDoer), destination, ) + m.managerCreator = func(ctx context.Context, modelWarehouse whutils.ModelWarehouse, conf *config.Config, logger logger.Logger, statsFactory stats.Stats) (manager.Manager, error) { + sf, err := manager.New(whutils.SnowpipeStreaming, conf, logger, statsFactory) + if err != nil { + return nil, fmt.Errorf("creating snowflake manager: %w", err) + } + err = sf.Setup(ctx, modelWarehouse, whutils.NewNoOpUploader()) + if err != nil { + return nil, fmt.Errorf("setting up snowflake manager: %w", err) + } + return sf, nil + } return m } @@ -121,6 +138,10 @@ func (m *Manager) retryableClient() *retryablehttp.Client { return client } +func (m *Manager) Now() time.Time { + return m.now() +} + func (m *Manager) Transform(job *jobsdb.JobT) (string, error) { return common.GetMarshalledData(string(job.EventPayload), job.JobID) } @@ -152,7 +173,15 @@ func (m *Manager) Upload(asyncDest *common.AsyncDestinationStruct) common.AsyncU discardsChannel, err := m.initializeChannelWithSchema(ctx, asyncDest.Destination.ID, &destConf, discardsTable(), discardsSchema()) if err != nil { - return m.abortJobs(asyncDest, fmt.Errorf("failed to prepare discards channel: %w", err).Error()) + switch { + case errors.Is(err, errAuthz): + m.setBackOff() + return m.failedJobs(asyncDest, err.Error()) + case errors.Is(err, errBackoff): + return m.failedJobs(asyncDest, err.Error()) + default: + return m.abortJobs(asyncDest, fmt.Errorf("failed to prepare discards channel: %w", err).Error()) + } } m.logger.Infon("Prepared discards channel") @@ -184,9 +213,21 @@ func (m *Manager) Upload(asyncDest *common.AsyncDestinationStruct) common.AsyncU importInfos []*importInfo discardImportInfo *importInfo ) + shouldResetBackoff := true // backoff should be reset if authz error is not encountered for any of the tables + isBackoffSet := false // should not be set again if already set for _, info := range uploadInfos { imInfo, discardImInfo, err := m.sendEventsToSnowpipe(ctx, asyncDest.Destination.ID, &destConf, info) if err != nil { + switch { + case errors.Is(err, errAuthz): + shouldResetBackoff = false + if !isBackoffSet { + isBackoffSet = true + m.setBackOff() + } + case errors.Is(err, errBackoff): + shouldResetBackoff = false + } m.logger.Warnn("Failed to send events to Snowpipe", logger.NewStringField("table", info.tableName), obskit.Error(err), @@ -206,6 +247,9 @@ func (m *Manager) Upload(asyncDest *common.AsyncDestinationStruct) common.AsyncU discardImportInfo.Offset = discardImInfo.Offset } } + if shouldResetBackoff { + m.resetBackoff() + } if discardImportInfo != nil { importInfos = append(importInfos, discardImportInfo) } @@ -245,7 +289,7 @@ func (m *Manager) eventsFromFile(fileName string, eventsCount int) ([]*event, er events := make([]*event, 0, eventsCount) - formattedTS := m.now().Format(misc.RFC3339Milli) + formattedTS := m.Now().Format(misc.RFC3339Milli) scanner := bufio.NewScanner(file) scanner.Buffer(nil, int(m.config.maxBufferCapacity.Load())) @@ -289,7 +333,7 @@ func (m *Manager) sendEventsToSnowpipe( } log.Infon("Prepared channel", logger.NewStringField("channelID", channelResponse.ChannelID)) - formattedTS := m.now().Format(misc.RFC3339Milli) + formattedTS := m.Now().Format(misc.RFC3339Milli) var discardInfos []discardInfo for _, tableEvent := range info.events { discardInfos = append(discardInfos, getDiscardedRecordsFromEvent(tableEvent, channelResponse.SnowpipeSchema, info.tableName, formattedTS)...) @@ -362,6 +406,16 @@ func (m *Manager) abortJobs(asyncDest *common.AsyncDestinationStruct, abortReaso } } +func (m *Manager) failedJobs(asyncDest *common.AsyncDestinationStruct, failedReason string) common.AsyncUploadOutput { + m.stats.jobs.failed.Count(len(asyncDest.ImportingJobIDs)) + return common.AsyncUploadOutput{ + FailedJobIDs: asyncDest.ImportingJobIDs, + FailedCount: len(asyncDest.ImportingJobIDs), + FailedReason: failedReason, + DestinationID: asyncDest.Destination.ID, + } +} + // Poll checks the status of multiple imports using the import ID from pollInput. // For the once which have reached the terminal state (success or failure), it caches the import infos in polledImportInfoMap. Later if Poll is called again, it does not need to do the status check again. // Once all the imports have reached the terminal state, if any imports have failed, it deletes the channels for those imports. @@ -549,3 +603,34 @@ func (m *Manager) GetUploadStats(input common.GetUploadStatsInput) common.GetUpl }, } } + +func (m *Manager) isInBackoff() bool { + if m.backoff.next.IsZero() { + return false + } + return m.Now().Before(m.backoff.next) +} + +func (m *Manager) resetBackoff() { + m.backoff.next = time.Time{} + m.backoff.attempts = 0 +} + +func (m *Manager) setBackOff() { + b := backoff.NewExponentialBackOff( + backoff.WithInitialInterval(m.config.backoff.initialInterval.Load()), + backoff.WithMultiplier(m.config.backoff.multiplier.Load()), + backoff.WithClockProvider(m), + backoff.WithRandomizationFactor(0), + backoff.WithMaxElapsedTime(0), + backoff.WithMaxInterval(m.config.backoff.maxInterval.Load()), + ) + b.Reset() + m.backoff.attempts++ + + var d time.Duration + for index := int64(0); index < int64(m.backoff.attempts); index++ { + d = b.NextBackOff() + } + m.backoff.next = m.Now().Add(d) +} diff --git a/router/batchrouter/asyncdestinationmanager/snowpipestreaming/snowpipestreaming_test.go b/router/batchrouter/asyncdestinationmanager/snowpipestreaming/snowpipestreaming_test.go index 912a546cb1..3826cb62c7 100644 --- a/router/batchrouter/asyncdestinationmanager/snowpipestreaming/snowpipestreaming_test.go +++ b/router/batchrouter/asyncdestinationmanager/snowpipestreaming/snowpipestreaming_test.go @@ -2,8 +2,10 @@ package snowpipestreaming import ( "context" + "fmt" "net/http" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -16,7 +18,11 @@ import ( backendconfig "github.com/rudderlabs/rudder-server/backend-config" "github.com/rudderlabs/rudder-server/jobsdb" "github.com/rudderlabs/rudder-server/router/batchrouter/asyncdestinationmanager/common" + internalapi "github.com/rudderlabs/rudder-server/router/batchrouter/asyncdestinationmanager/snowpipestreaming/internal/api" "github.com/rudderlabs/rudder-server/router/batchrouter/asyncdestinationmanager/snowpipestreaming/internal/model" + "github.com/rudderlabs/rudder-server/utils/timeutil" + "github.com/rudderlabs/rudder-server/warehouse/integrations/manager" + "github.com/rudderlabs/rudder-server/warehouse/integrations/snowflake" whutils "github.com/rudderlabs/rudder-server/warehouse/utils" ) @@ -43,6 +49,25 @@ func (m *mockAPI) GetStatus(_ context.Context, channelID string) (*model.StatusR return m.getStatusOutputMap[channelID]() } +type mockManager struct { + manager.Manager + createSchemaErr error +} + +func newMockManager(m manager.Manager) *mockManager { + return &mockManager{ + Manager: m, + } +} + +func (m *mockManager) CreateSchema(context.Context) error { + return m.createSchemaErr +} + +func (m *mockManager) CreateTable(context.Context, string, whutils.ModelTableSchema) error { + return nil +} + var ( usersChannelResponse = &model.ChannelResponse{ ChannelID: "test-users-channel", @@ -77,6 +102,7 @@ func TestSnowpipeStreaming(t *testing.T) { DestinationDefinition: backendconfig.DestinationDefinitionT{ Name: "SNOWPIPE_STREAMING", }, + Config: make(map[string]interface{}), } t.Run("Upload with invalid file path", func(t *testing.T) { @@ -100,6 +126,7 @@ func TestSnowpipeStreaming(t *testing.T) { "status": "aborted", }).LastValue()) }) + t.Run("Upload with invalid record in file", func(t *testing.T) { statsStore, err := memstats.New() require.NoError(t, err) @@ -310,6 +337,101 @@ func TestSnowpipeStreaming(t *testing.T) { "status": "failed", }).LastValue()) }) + + t.Run("Upload with unauthorized schema error should add backoff", func(t *testing.T) { + statsStore, err := memstats.New() + require.NoError(t, err) + + sm := New(config.New(), logger.NOP, statsStore, destination) + sm.channelCache.Store("RUDDER_DISCARDS", rudderDiscardsChannelResponse) + sm.api = &mockAPI{ + createChannelOutputMap: map[string]func() (*model.ChannelResponse, error){ + "USERS": func() (*model.ChannelResponse, error) { + return &model.ChannelResponse{Code: internalapi.ErrSchemaDoesNotExistOrNotAuthorized}, nil + }, + }, + } + managerCreatorCallCount := 0 + sm.managerCreator = func(_ context.Context, _ whutils.ModelWarehouse, _ *config.Config, _ logger.Logger, _ stats.Stats) (manager.Manager, error) { + sf := snowflake.New(config.New(), logger.NOP, stats.NOP) + managerCreatorCallCount++ + mm := newMockManager(sf) + mm.createSchemaErr = fmt.Errorf("failed to create schema") + return mm, nil + } + sm.config.backoff.initialInterval = config.SingleValueLoader(time.Second * 10) + asyncDestStruct := &common.AsyncDestinationStruct{ + Destination: destination, + FileName: "testdata/successful_user_records.txt", + } + require.False(t, sm.isInBackoff()) + output1 := sm.Upload(asyncDestStruct) + require.Equal(t, 2, output1.FailedCount) + require.Equal(t, 0, output1.AbortCount) + require.Equal(t, 1, managerCreatorCallCount) + require.True(t, sm.isInBackoff()) + + sm.Upload(asyncDestStruct) + // client is not created again due to backoff error + require.Equal(t, 1, managerCreatorCallCount) + require.True(t, sm.isInBackoff()) + + sm.now = func() time.Time { + return timeutil.Now().Add(time.Second * 5) + } + require.True(t, sm.isInBackoff()) + sm.now = func() time.Time { + return timeutil.Now().Add(time.Second * 20) + } + require.False(t, sm.isInBackoff()) + sm.Upload(asyncDestStruct) + // client created again since backoff duration has been exceeded + require.Equal(t, 2, managerCreatorCallCount) + require.True(t, sm.isInBackoff()) + + sm.managerCreator = func(_ context.Context, _ whutils.ModelWarehouse, _ *config.Config, _ logger.Logger, _ stats.Stats) (manager.Manager, error) { + sf := snowflake.New(config.New(), logger.NOP, stats.NOP) + managerCreatorCallCount++ + return newMockManager(sf), nil + } + sm.now = func() time.Time { + return timeutil.Now().Add(time.Second * 50) + } + sm.Upload(asyncDestStruct) + require.Equal(t, 3, managerCreatorCallCount) + require.False(t, sm.isInBackoff()) + }) + + t.Run("Upload with discards table authorization error should mark the job as failed", func(t *testing.T) { + statsStore, err := memstats.New() + require.NoError(t, err) + + sm := New(config.New(), logger.NOP, statsStore, destination) + sm.api = &mockAPI{ + createChannelOutputMap: map[string]func() (*model.ChannelResponse, error){ + "RUDDER_DISCARDS": func() (*model.ChannelResponse, error) { + return &model.ChannelResponse{Code: internalapi.ErrSchemaDoesNotExistOrNotAuthorized}, nil + }, + }, + } + sm.managerCreator = func(_ context.Context, _ whutils.ModelWarehouse, _ *config.Config, _ logger.Logger, _ stats.Stats) (manager.Manager, error) { + sf := snowflake.New(config.New(), logger.NOP, stats.NOP) + mm := newMockManager(sf) + mm.createSchemaErr = fmt.Errorf("failed to create schema") + return mm, nil + } + output := sm.Upload(&common.AsyncDestinationStruct{ + ImportingJobIDs: []int64{1}, + Destination: destination, + FileName: "testdata/successful_user_records.txt", + }) + require.Equal(t, 1, output.FailedCount) + require.Equal(t, 0, output.AbortCount) + require.NotEmpty(t, output.FailedReason) + require.Empty(t, output.AbortReason) + require.Equal(t, true, sm.isInBackoff()) + }) + t.Run("Upload insert error for all events", func(t *testing.T) { statsStore, err := memstats.New() require.NoError(t, err) diff --git a/router/batchrouter/asyncdestinationmanager/snowpipestreaming/testdata/successful_user_records.txt b/router/batchrouter/asyncdestinationmanager/snowpipestreaming/testdata/successful_user_records.txt new file mode 100644 index 0000000000..007ae8bafe --- /dev/null +++ b/router/batchrouter/asyncdestinationmanager/snowpipestreaming/testdata/successful_user_records.txt @@ -0,0 +1,2 @@ +{"message":{"metadata":{"table":"USERS","columns":{"ID":"int","NAME":"string","AGE":"int","RECEIVED_AT":"datetime"}},"data":{"ID":1,"NAME":"Alice","AGE":30,"RECEIVED_AT":"2023-05-12T04:36:50.199Z"}},"metadata":{"job_id":1001}} +{"message":{"metadata":{"table":"USERS","columns":{"ID":"int","NAME":"string","AGE":"int","RECEIVED_AT":"datetime"}},"data":{"ID":1,"NAME":"Alice","AGE":30,"RECEIVED_AT":"2023-05-12T04:36:50.199Z"}},"metadata":{"job_id":1003}} diff --git a/router/batchrouter/asyncdestinationmanager/snowpipestreaming/types.go b/router/batchrouter/asyncdestinationmanager/snowpipestreaming/types.go index 3376f6104a..f42832fc10 100644 --- a/router/batchrouter/asyncdestinationmanager/snowpipestreaming/types.go +++ b/router/batchrouter/asyncdestinationmanager/snowpipestreaming/types.go @@ -15,6 +15,7 @@ import ( "github.com/rudderlabs/rudder-server/router/batchrouter/asyncdestinationmanager/snowpipestreaming/internal/model" backendconfig "github.com/rudderlabs/rudder-server/backend-config" + "github.com/rudderlabs/rudder-server/warehouse/integrations/manager" whutils "github.com/rudderlabs/rudder-server/warehouse/utils" ) @@ -25,6 +26,7 @@ type ( statsFactory stats.Stats destination *backendconfig.DestinationT requestDoer requestDoer + managerCreator func(ctx context.Context, modelWarehouse whutils.ModelWarehouse, conf *config.Config, logger logger.Logger, statsFactory stats.Stats) (manager.Manager, error) now func() time.Time api api channelCache sync.Map @@ -44,6 +46,18 @@ type ( } instanceID string maxBufferCapacity config.ValueLoader[int64] + backoff struct { + multiplier config.ValueLoader[float64] + initialInterval config.ValueLoader[time.Duration] + maxInterval config.ValueLoader[time.Duration] + } + } + backoff struct { + // If an attempt was made to create a resource but it failed likely due to permission issues, + // then the next attempt to create a SF connection will be made after "next". + // This approach prevents repeatedly activating the warehouse even though the permission issue remains unresolved. + attempts int + next time.Time } stats struct { From ddf04ffb010a62bd001abe654fc8fcdb4715cbdd Mon Sep 17 00:00:00 2001 From: devops-github-rudderstack <88187154+devops-github-rudderstack@users.noreply.github.com> Date: Thu, 9 Jan 2025 10:37:08 +0530 Subject: [PATCH 11/11] chore: sync release v1.40.1 to main branch (#5419) --- CHANGELOG.md | 7 + warehouse/internal/model/warehouse.go | 4 + warehouse/router/identities.go | 3 +- warehouse/router/router.go | 45 +- warehouse/router/router_test.go | 3 +- warehouse/router/scheduling_test.go | 11 +- .../router/testdata/sql/seed_tracker_test.sql | 23 - warehouse/router/tracker.go | 254 ++++----- warehouse/router/tracker_test.go | 519 +++++++++++------- 9 files changed, 500 insertions(+), 369 deletions(-) delete mode 100644 warehouse/router/testdata/sql/seed_tracker_test.sql diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ff73fe014..c8a6b1da93 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## [1.40.1](https://github.com/rudderlabs/rudder-server/compare/v1.40.0...v1.40.1) (2025-01-08) + + +### Bug Fixes + +* warehouse router tracker ([#5407](https://github.com/rudderlabs/rudder-server/issues/5407)) ([8e314b6](https://github.com/rudderlabs/rudder-server/commit/8e314b6160c37283fbbe889f1dbad86f994943b1)) + ## [1.40.0](https://github.com/rudderlabs/rudder-server/compare/v1.39.0...v1.40.0) (2025-01-06) diff --git a/warehouse/internal/model/warehouse.go b/warehouse/internal/model/warehouse.go index 3cfe020c0e..10e7c07377 100644 --- a/warehouse/internal/model/warehouse.go +++ b/warehouse/internal/model/warehouse.go @@ -62,3 +62,7 @@ func (w *Warehouse) GetPreferAppendSetting() bool { } return value } + +func (w *Warehouse) IsEnabled() bool { + return w.Source.Enabled && w.Destination.Enabled +} diff --git a/warehouse/router/identities.go b/warehouse/router/identities.go index 51edc96eee..aa6862c6fc 100644 --- a/warehouse/router/identities.go +++ b/warehouse/router/identities.go @@ -18,7 +18,6 @@ import ( "github.com/rudderlabs/rudder-server/rruntime" "github.com/rudderlabs/rudder-server/utils/misc" - "github.com/rudderlabs/rudder-server/utils/timeutil" warehouseutils "github.com/rudderlabs/rudder-server/warehouse/utils" ) @@ -309,7 +308,7 @@ func (r *Router) initPrePopulateDestIdentitiesUpload(warehouse model.Warehouse) RETURNING id `, warehouseutils.WarehouseUploadsTable) - now := timeutil.Now() + now := r.now() row := r.db.QueryRow( sqlStatement, warehouse.Source.ID, diff --git a/warehouse/router/router.go b/warehouse/router/router.go index b2e66d249d..15b54a067d 100644 --- a/warehouse/router/router.go +++ b/warehouse/router/router.go @@ -105,16 +105,17 @@ type Router struct { stagingFilesBatchSize config.ValueLoader[int] warehouseSyncFreqIgnore config.ValueLoader[bool] cronTrackerRetries config.ValueLoader[int64] + uploadBufferTimeInMin config.ValueLoader[time.Duration] } stats struct { - processingPendingJobsStat stats.Measurement - processingAvailableWorkersStat stats.Measurement - processingPickupLagStat stats.Measurement - processingPickupWaitTimeStat stats.Measurement - - schedulerWarehouseLengthStat stats.Measurement - schedulerTotalSchedulingTimeStat stats.Measurement + processingPendingJobsStat stats.Gauge + processingAvailableWorkersStat stats.Gauge + processingPickupLagStat stats.Timer + processingPickupWaitTimeStat stats.Timer + schedulerWarehouseLengthStat stats.Gauge + schedulerTotalSchedulingTimeStat stats.Timer + cronTrackerExecTimestamp stats.Gauge } } @@ -150,7 +151,7 @@ func New( r.tenantManager = tenantManager r.bcManager = bcManager r.destType = destType - r.now = time.Now + r.now = timeutil.Now r.triggerStore = triggerStore r.createJobMarkerMap = make(map[string]time.Time) r.createUploadAlways = createUploadAlways @@ -200,7 +201,7 @@ func (r *Router) Start(ctx context.Context) error { return nil })) g.Go(crash.NotifyWarehouse(func() error { - return r.CronTracker(gCtx) + return r.cronTracker(gCtx) })) return g.Wait() } @@ -616,7 +617,7 @@ func (r *Router) handlePriorityForWaitingUploads(ctx context.Context, warehouse func (r *Router) uploadStartAfterTime() time.Time { if r.config.enableJitterForSyncs.Load() { - return timeutil.Now().Add(time.Duration(rand.Intn(15)) * time.Second) + return r.now().Add(time.Duration(rand.Intn(15)) * time.Second) } return r.now() } @@ -711,14 +712,32 @@ func (r *Router) loadReloadableConfig(whName string) { r.config.enableJitterForSyncs = r.conf.GetReloadableBoolVar(false, "Warehouse.enableJitterForSyncs") r.config.warehouseSyncFreqIgnore = r.conf.GetReloadableBoolVar(false, "Warehouse.warehouseSyncFreqIgnore") r.config.cronTrackerRetries = r.conf.GetReloadableInt64Var(5, 1, "Warehouse.cronTrackerRetries") + r.config.uploadBufferTimeInMin = r.conf.GetReloadableDurationVar(180, time.Minute, "Warehouse.uploadBufferTimeInMin") } func (r *Router) loadStats() { - tags := stats.Tags{"destType": r.destType} + tags := stats.Tags{"module": moduleName, "destType": r.destType} r.stats.processingPendingJobsStat = r.statsFactory.NewTaggedStat("wh_processing_pending_jobs", stats.GaugeType, tags) r.stats.processingAvailableWorkersStat = r.statsFactory.NewTaggedStat("wh_processing_available_workers", stats.GaugeType, tags) r.stats.processingPickupLagStat = r.statsFactory.NewTaggedStat("wh_processing_pickup_lag", stats.TimerType, tags) r.stats.processingPickupWaitTimeStat = r.statsFactory.NewTaggedStat("wh_processing_pickup_wait_time", stats.TimerType, tags) - r.stats.schedulerWarehouseLengthStat = r.statsFactory.NewTaggedStat("wh_scheduler.warehouse_length", stats.GaugeType, tags) - r.stats.schedulerTotalSchedulingTimeStat = r.statsFactory.NewTaggedStat("wh_scheduler.total_scheduling_time", stats.TimerType, tags) + r.stats.schedulerWarehouseLengthStat = r.statsFactory.NewTaggedStat("wh_scheduler_warehouse_length", stats.GaugeType, tags) + r.stats.schedulerTotalSchedulingTimeStat = r.statsFactory.NewTaggedStat("wh_scheduler_total_scheduling_time", stats.TimerType, tags) + r.stats.cronTrackerExecTimestamp = r.statsFactory.NewTaggedStat("warehouse_cron_tracker_timestamp_seconds", stats.GaugeType, tags) +} + +func (r *Router) copyWarehouses() []model.Warehouse { + r.configSubscriberLock.RLock() + defer r.configSubscriberLock.RUnlock() + + warehouses := make([]model.Warehouse, len(r.warehouses)) + copy(warehouses, r.warehouses) + return warehouses +} + +func (r *Router) getNowSQL() string { + if r.nowSQL != "" { + return r.nowSQL + } + return "NOW()" } diff --git a/warehouse/router/router_test.go b/warehouse/router/router_test.go index 9feb3dfd34..c0759305d8 100644 --- a/warehouse/router/router_test.go +++ b/warehouse/router/router_test.go @@ -32,6 +32,7 @@ import ( "github.com/rudderlabs/rudder-server/services/notifier" migrator "github.com/rudderlabs/rudder-server/services/sql-migrator" "github.com/rudderlabs/rudder-server/utils/pubsub" + "github.com/rudderlabs/rudder-server/utils/timeutil" "github.com/rudderlabs/rudder-server/warehouse/bcm" "github.com/rudderlabs/rudder-server/warehouse/encoding" sqlmiddleware "github.com/rudderlabs/rudder-server/warehouse/integrations/middleware/sqlquerywrapper" @@ -143,7 +144,7 @@ func TestRouter(t *testing.T) { db := sqlmiddleware.New(pgResource.DB) - now := time.Now() + now := timeutil.Now() repoUpload := repo.NewUploads(db, repo.WithNow(func() time.Time { return now diff --git a/warehouse/router/scheduling_test.go b/warehouse/router/scheduling_test.go index ec1146c023..1c8d2b1fc6 100644 --- a/warehouse/router/scheduling_test.go +++ b/warehouse/router/scheduling_test.go @@ -18,6 +18,7 @@ import ( backendConfig "github.com/rudderlabs/rudder-server/backend-config" migrator "github.com/rudderlabs/rudder-server/services/sql-migrator" + "github.com/rudderlabs/rudder-server/utils/timeutil" sqlmiddleware "github.com/rudderlabs/rudder-server/warehouse/integrations/middleware/sqlquerywrapper" "github.com/rudderlabs/rudder-server/warehouse/internal/model" "github.com/rudderlabs/rudder-server/warehouse/internal/repo" @@ -235,7 +236,7 @@ func TestRouter_CanCreateUpload(t *testing.T) { Identifier: "test_identifier_upload_frequency_exceeded", } - now := time.Now() + now := timeutil.Now() r := Router{} r.conf = config.New() @@ -260,7 +261,7 @@ func TestRouter_CanCreateUpload(t *testing.T) { Identifier: "test_identifier_upload_frequency_exceeded", } - now := time.Now() + now := timeutil.Now() r := Router{} r.conf = config.New() @@ -316,7 +317,7 @@ func TestRouter_CanCreateUpload(t *testing.T) { }, } - now := time.Now() + now := timeutil.Now() r := Router{} r.conf = config.New() @@ -344,7 +345,7 @@ func TestRouter_CanCreateUpload(t *testing.T) { }, } - now := time.Now() + now := timeutil.Now() r := Router{} r.conf = config.New() @@ -453,7 +454,7 @@ func TestRouter_CanCreateUpload(t *testing.T) { return tc.now } - r.updateCreateJobMarker(w, time.Now()) + r.updateCreateJobMarker(w, now) err := r.canCreateUpload(context.Background(), w) if tc.wantErr != nil { diff --git a/warehouse/router/testdata/sql/seed_tracker_test.sql b/warehouse/router/testdata/sql/seed_tracker_test.sql deleted file mode 100644 index 5e41a5ed1a..0000000000 --- a/warehouse/router/testdata/sql/seed_tracker_test.sql +++ /dev/null @@ -1,23 +0,0 @@ -BEGIN; -INSERT INTO wh_staging_files (id, location, schema, source_id, destination_id, status, total_events, first_event_at, - last_event_at, created_at, updated_at, metadata) -VALUES (1, 'a.json.gz', '{}', 'test-sourceID', 'test-destinationID', 'succeeded', 1, NOW(), NOW(), - '2022-12-06 15:23:37.100685', NOW(), '{}'), - (2, 'a.json.gz', '{}', 'test-sourceID', 'test-destinationID', 'succeeded', 1, NOW(), NOW(), - '2022-12-06 15:24:37.100685', NOW(), '{}'), - (3, 'a.json.gz', '{}', 'test-sourceID', 'test-destinationID', 'succeeded', 1, NOW(), NOW(), - '2022-12-06 15:25:37.100685', NOW(), '{}'), - (4, 'a.json.gz', '{}', 'test-sourceID', 'test-destinationID', 'succeeded', 1, NOW(), NOW(), - '2022-12-06 15:26:37.100685', NOW(), '{}'), - (5, 'a.json.gz', '{}', 'test-sourceID', 'test-destinationID', 'succeeded', 1, NOW(), NOW(), - '2022-12-06 15:27:37.100685', NOW(), '{}'), - (6, 'a.json.gz', '{}', 'test-sourceID', 'test-destinationID-1', 'succeeded', 1, NOW(), NOW(), - '2022-12-06 15:27:37.100685', NOW(), '{}'); -INSERT INTO wh_uploads(id, source_id, namespace, destination_id, destination_type, start_staging_file_id, - end_staging_file_id, start_load_file_id, end_load_file_id, status, schema, error, first_event_at, - last_event_at, last_exec_at, timings, created_at, updated_at, metadata, - in_progress, workspace_id) -VALUES (1, 'test-sourceID', 'test-namespace', 'test-destinationID', 'POSTGRES', 0, 0, 0, 0, 'exported_data', '{}', - '{}', NULL, NULL, NULL, NULL, '2022-12-06 15:30:00', '2022-12-06 15:45:00', '{}', TRUE, - 'test-workspaceID'); -COMMIT; diff --git a/warehouse/router/tracker.go b/warehouse/router/tracker.go index 9b00dd06fc..2902785a88 100644 --- a/warehouse/router/tracker.go +++ b/warehouse/router/tracker.go @@ -10,196 +10,170 @@ import ( "github.com/cenkalti/backoff/v4" - "github.com/rudderlabs/rudder-go-kit/config" "github.com/rudderlabs/rudder-go-kit/stats" - obskit "github.com/rudderlabs/rudder-observability-kit/go/labels" "github.com/rudderlabs/rudder-server/utils/misc" - "github.com/rudderlabs/rudder-server/utils/timeutil" "github.com/rudderlabs/rudder-server/warehouse/internal/model" - warehouseutils "github.com/rudderlabs/rudder-server/warehouse/utils" + whutils "github.com/rudderlabs/rudder-server/warehouse/utils" ) -// CronTracker Track the status of the staging file whether it has reached the terminal state or not for every warehouse +// cronTracker Track the status of the staging file whether it has reached the terminal state or not for every warehouse // we pick the staging file which is oldest within the range NOW() - 2 * syncFrequency and NOW() - 3 * syncFrequency -func (r *Router) CronTracker(ctx context.Context) error { - cronTrackerExecTimestamp := r.statsFactory.NewTaggedStat("warehouse_cron_tracker_timestamp_seconds", stats.GaugeType, stats.Tags{ - "module": moduleName, - "destType": r.destType, - }) +// and checks if the corresponding upload has reached the terminal state or not. +// If the upload has not reached the terminal state, then we send a gauge metric with value 1 else 0 +func (r *Router) cronTracker(ctx context.Context) error { for { - - execTime := time.Now() - cronTrackerExecTimestamp.Gauge(execTime.Unix()) - - r.configSubscriberLock.RLock() - warehouses := append([]model.Warehouse{}, r.warehouses...) - r.configSubscriberLock.RUnlock() - - for _, warehouse := range warehouses { - b := backoff.WithContext(backoff.WithMaxRetries(backoff.NewExponentialBackOff(), uint64(r.config.cronTrackerRetries.Load())), ctx) - err := backoff.Retry(func() error { - return r.Track(ctx, &warehouse, r.conf) - }, b) - if err != nil { - r.logger.Errorn( - "cron tracker failed for", - obskit.SourceID(warehouse.Source.ID), - obskit.DestinationID(warehouse.Destination.ID), - obskit.Error(err), - ) - break + execTime := r.now() + r.stats.cronTrackerExecTimestamp.Gauge(execTime.Unix()) + + for _, warehouse := range r.copyWarehouses() { + if err := r.retryTrackSync(ctx, &warehouse); err != nil { + if ctx.Err() != nil { + return nil //nolint:nilerr + } + return fmt.Errorf("cron tracker: %w", err) } } nextExecTime := execTime.Add(r.config.uploadStatusTrackFrequency) select { case <-ctx.Done(): - r.logger.Infon("context is cancelled, stopped running tracking") + r.logger.Infon("Context cancelled. Exiting cron tracker") return nil case <-time.After(time.Until(nextExecTime)): } } } -// Track tracks the status of the warehouse uploads for the corresponding cases: -// 1. Staging files is not picked. -// 2. Upload job is struck -func (r *Router) Track( - ctx context.Context, - warehouse *model.Warehouse, - config *config.Config, -) error { - var ( - createdAt sql.NullTime - exists bool - syncFrequency = "1440" - now = timeutil.Now - nowSQL = "NOW()" - failedStatusRegex = "%_failed" - timeWindow = config.GetDuration("Warehouse.uploadBufferTimeInMin", 180, time.Minute) - source = warehouse.Source - destination = warehouse.Destination - ) - - if r.nowSQL != "" { - nowSQL = r.nowSQL - } - if r.now != nil { - now = r.now +func (r *Router) retryTrackSync(ctx context.Context, warehouse *model.Warehouse) error { + o := func() error { + return r.trackSync(ctx, warehouse) } + b := backoff.WithContext( + backoff.WithMaxRetries( + backoff.NewExponentialBackOff(), + uint64(r.config.cronTrackerRetries.Load()), + ), + ctx, + ) + return backoff.Retry(o, b) +} - trackUploadMissingStat := r.statsFactory.NewTaggedStat("warehouse_track_upload_missing", stats.GaugeType, stats.Tags{ - "workspaceId": warehouse.WorkspaceID, - "module": moduleName, - "destType": r.destType, - "sourceId": source.ID, - "destinationId": destination.ID, - "warehouseID": misc.GetTagName( - destination.ID, - source.Name, - destination.Name, - misc.TailTruncateStr(source.ID, 6)), - }) - trackUploadMissingStat.Gauge(0) - - if !source.Enabled || !destination.Enabled { +func (r *Router) trackSync(ctx context.Context, warehouse *model.Warehouse) error { + if !warehouse.IsEnabled() || r.isWithinExcludeWindow(warehouse) { return nil } - excludeWindow := warehouse.GetMapDestinationConfig(model.ExcludeWindowSetting) - excludeWindowStartTime, excludeWindowEndTime := excludeWindowStartEndTimes(excludeWindow) - if checkCurrentTimeExistsInExcludeWindow(now(), excludeWindowStartTime, excludeWindowEndTime) { + createdAt, err := r.getOldestStagingFile(ctx, warehouse) + if err != nil { + return err + } + if createdAt.IsZero() { return nil } - if sf := warehouse.GetStringDestinationConfig(r.conf, model.SyncFrequencySetting); sf != "" { - syncFrequency = sf - } - if value, err := strconv.Atoi(syncFrequency); err == nil { - timeWindow += time.Duration(value) * time.Minute + exists, err := r.checkUploadStatus(ctx, warehouse, createdAt) + if err != nil { + return err } + r.recordUploadMissingMetric(warehouse, exists) + return nil +} + +func (r *Router) isWithinExcludeWindow(warehouse *model.Warehouse) bool { + excludeWindow := warehouse.GetMapDestinationConfig(model.ExcludeWindowSetting) + startTime, endTime := excludeWindowStartEndTimes(excludeWindow) + return checkCurrentTimeExistsInExcludeWindow(r.now(), startTime, endTime) +} + +func (r *Router) getOldestStagingFile(ctx context.Context, warehouse *model.Warehouse) (time.Time, error) { + nowSQL := r.getNowSQL() + timeWindow := r.calculateTimeWindow(warehouse) + query := fmt.Sprintf(` - SELECT - created_at - FROM - %[1]s - WHERE - source_id = $1 AND - destination_id = $2 AND - created_at > %[2]s - $3 * INTERVAL '1 MIN' AND - created_at < %[2]s - $4 * INTERVAL '1 MIN' - ORDER BY - id DESC - LIMIT - 1; - `, - warehouseutils.WarehouseStagingFilesTable, + SELECT created_at + FROM `+whutils.WarehouseStagingFilesTable+` + WHERE source_id = $1 + AND destination_id = $2 + AND created_at > %[1]s - $3 * INTERVAL '1 MIN' + AND created_at < %[1]s - $4 * INTERVAL '1 MIN' + ORDER BY id DESC + LIMIT 1;`, nowSQL, ) - queryArgs := []interface{}{ - source.ID, - destination.ID, + queryArgs := []any{ + warehouse.Source.ID, + warehouse.Destination.ID, 2 * timeWindow / time.Minute, timeWindow / time.Minute, } + var createdAt sql.NullTime err := r.db.QueryRowContext(ctx, query, queryArgs...).Scan(&createdAt) - if errors.Is(err, sql.ErrNoRows) { - return nil + if err != nil && errors.Is(err, sql.ErrNoRows) { + return time.Time{}, nil } if err != nil { - return fmt.Errorf("fetching last upload time for source: %s and destination: %s: %w", source.ID, destination.ID, err) + return time.Time{}, fmt.Errorf("fetching oldest staging file for source %s and destination %s: %w", + warehouse.Source.ID, warehouse.Destination.ID, err) } - if !createdAt.Valid { - return fmt.Errorf("invalid last upload time for source: %s and destination: %s", source.ID, destination.ID) + return time.Time{}, fmt.Errorf("invalid created_at time for source %s and destination %s", + warehouse.Source.ID, warehouse.Destination.ID) } + return createdAt.Time, nil +} - query = ` - SELECT - EXISTS ( - SELECT - 1 - FROM - ` + warehouseutils.WarehouseUploadsTable + ` - WHERE - source_id = $1 AND - destination_id = $2 AND - ( - status = $3 - OR status = $4 - OR status LIKE $5 - ) AND - updated_at > $6 - ); - ` - queryArgs = []interface{}{ - source.ID, - destination.ID, +func (r *Router) calculateTimeWindow(warehouse *model.Warehouse) time.Duration { + timeWindow := r.config.uploadBufferTimeInMin.Load() + syncFrequency := warehouse.GetStringDestinationConfig(r.conf, model.SyncFrequencySetting) + if syncFrequency != "" { + if value, err := strconv.Atoi(syncFrequency); err == nil { + timeWindow += time.Duration(value) * time.Minute + } + } + return timeWindow +} + +func (r *Router) checkUploadStatus(ctx context.Context, warehouse *model.Warehouse, createdAt time.Time) (bool, error) { + query := ` + SELECT EXISTS ( + SELECT 1 + FROM ` + whutils.WarehouseUploadsTable + ` + WHERE source_id = $1 AND destination_id = $2 AND + (status = $3 OR status = $4 OR status LIKE $5) AND + updated_at > $6 + );` + queryArgs := []any{ + warehouse.Source.ID, + warehouse.Destination.ID, model.ExportedData, model.Aborted, - failedStatusRegex, - createdAt.Time.Format(misc.RFC3339Milli), + "%_failed", + createdAt.Format(misc.RFC3339Milli), } - err = r.db.QueryRowContext(ctx, query, queryArgs...).Scan(&exists) + var exists bool + err := r.db.QueryRowContext(ctx, query, queryArgs...).Scan(&exists) if err != nil && !errors.Is(err, sql.ErrNoRows) { - return fmt.Errorf("fetching last upload status for source: %s and destination: %s: %w", source.ID, destination.ID, err) + return false, fmt.Errorf("checking upload status for source %s and destination %s: %w", + warehouse.Source.ID, warehouse.Destination.ID, err) } + return exists, nil +} - if !exists { - r.logger.Warnn("pending staging files not picked", - obskit.SourceID(source.ID), - obskit.SourceType(source.SourceDefinition.Name), - obskit.DestinationID(destination.ID), - obskit.DestinationType(destination.DestinationDefinition.Name), - obskit.WorkspaceID(warehouse.WorkspaceID), - ) - - trackUploadMissingStat.Gauge(1) +func (r *Router) recordUploadMissingMetric(warehouse *model.Warehouse, exists bool) { + metric := r.statsFactory.NewTaggedStat("warehouse_track_upload_missing", stats.GaugeType, stats.Tags{ + "module": moduleName, + "workspaceId": warehouse.WorkspaceID, + "destType": r.destType, + "sourceId": warehouse.Source.ID, + "destinationId": warehouse.Destination.ID, + }) + if exists { + metric.Gauge(0) + } else { + metric.Gauge(1) } - - return nil } diff --git a/warehouse/router/tracker_test.go b/warehouse/router/tracker_test.go index 5de2942076..ea7b68e38a 100644 --- a/warehouse/router/tracker_test.go +++ b/warehouse/router/tracker_test.go @@ -2,230 +2,379 @@ package router import ( "context" - "errors" - "os" + "fmt" "testing" "time" "github.com/ory/dockertest/v3" "github.com/stretchr/testify/require" - "go.uber.org/mock/gomock" "github.com/rudderlabs/rudder-go-kit/config" "github.com/rudderlabs/rudder-go-kit/logger" - "github.com/rudderlabs/rudder-go-kit/logger/mock_logger" "github.com/rudderlabs/rudder-go-kit/stats" "github.com/rudderlabs/rudder-go-kit/stats/memstats" "github.com/rudderlabs/rudder-go-kit/testhelper/docker/resource/postgres" backendconfig "github.com/rudderlabs/rudder-server/backend-config" migrator "github.com/rudderlabs/rudder-server/services/sql-migrator" - "github.com/rudderlabs/rudder-server/utils/misc" - "github.com/rudderlabs/rudder-server/warehouse/integrations/middleware/sqlquerywrapper" + "github.com/rudderlabs/rudder-server/utils/timeutil" + sqlmiddleware "github.com/rudderlabs/rudder-server/warehouse/integrations/middleware/sqlquerywrapper" "github.com/rudderlabs/rudder-server/warehouse/internal/model" - warehouseutils "github.com/rudderlabs/rudder-server/warehouse/utils" + "github.com/rudderlabs/rudder-server/warehouse/internal/repo" + whutils "github.com/rudderlabs/rudder-server/warehouse/utils" ) -func TestRouter_Track(t *testing.T) { - var ( - workspaceID = "test-workspaceID" - sourceID = "test-sourceID" - sourceName = "test-sourceName" - destID = "test-destinationID" - destName = "test-destinationName" - destType = warehouseutils.POSTGRES - ) - - testcases := []struct { - name string - destID string - destDisabled bool - wantErr error - missing bool - NowSQL string - exclusionWindow map[string]any - uploadBufferTime string - }{ - { - name: "unknown destination", - destID: "unknown-destination", - }, - { - name: "disabled destination", - destID: destID, - destDisabled: true, - }, - { - name: "successful upload exists", - destID: destID, - missing: false, - }, - { - name: "successful upload exists with upload buffer time", - destID: destID, - missing: false, - uploadBufferTime: "0m", - }, - { - name: "exclusion window", - destID: destID, - missing: false, - exclusionWindow: map[string]any{ - "excludeWindowStartTime": "05:09", - "excludeWindowEndTime": "09:07", +func TestRouter_CronTrack(t *testing.T) { + t.Run("source / destination disabled", func(t *testing.T) { + ctx := context.Background() + + statsStore, err := memstats.New() + require.NoError(t, err) + + warehouse := model.Warehouse{ + WorkspaceID: "test-workspaceID", + Source: backendconfig.SourceT{ + ID: "test-sourceID", + Name: "test-sourceName", + Enabled: false, }, - }, - { - name: "no successful upload exists", - destID: "test-destinationID-1", - missing: true, - }, - { - name: "throw error while fetching last upload time", - destID: destID, - missing: false, - NowSQL: "ABC", - wantErr: errors.New("fetching last upload time for source: test-sourceID and destination: test-destinationID: pq: column \"abc\" does not exist"), - }, - } - - for _, tc := range testcases { - tc := tc - - t.Run(tc.name, func(t *testing.T) { - t.Parallel() - - pool, err := dockertest.NewPool("") - require.NoError(t, err) - - pgResource, err := postgres.Setup(pool, t) - require.NoError(t, err) - - t.Log("db:", pgResource.DBDsn) - - err = (&migrator.Migrator{ - Handle: pgResource.DB, - MigrationsTable: "wh_schema_migrations", - }).Migrate("warehouse") - require.NoError(t, err) - - sqlStatement, err := os.ReadFile("testdata/sql/seed_tracker_test.sql") - require.NoError(t, err) - - _, err = pgResource.DB.Exec(string(sqlStatement)) - require.NoError(t, err) - - statsStore, err := memstats.New() - require.NoError(t, err) - - ctx := context.Background() - nowSQL := "'2022-12-06 15:40:00'::timestamp" - - now, err := time.Parse(misc.RFC3339Milli, "2022-12-06T06:19:00.169Z") - require.NoError(t, err) - - conf := config.New() - if tc.uploadBufferTime != "" { - conf.Set("Warehouse.uploadBufferTimeInMin", tc.uploadBufferTime) - } else { - conf.Set("Warehouse.uploadBufferTimeInMin", 0) - } - - warehouse := model.Warehouse{ - WorkspaceID: workspaceID, - Source: backendconfig.SourceT{ - ID: sourceID, - Name: sourceName, - Enabled: true, + Destination: backendconfig.DestinationT{ + ID: "test-destinationID", + Name: "test-destinationName", + Enabled: false, + Config: map[string]any{ + "syncFrequency": "30", }, - Destination: backendconfig.DestinationT{ - ID: tc.destID, - Name: destName, - Enabled: !tc.destDisabled, - Config: map[string]any{ - "syncFrequency": "10", - "excludeWindow": tc.exclusionWindow, + }, + } + + r := Router{ + conf: config.New(), + destType: whutils.POSTGRES, + now: timeutil.Now, + statsFactory: statsStore, + logger: logger.NOP, + } + + require.NoError(t, r.trackSync(ctx, &warehouse)) + require.Nil(t, statsStore.Get("warehouse_track_upload_missing", stats.Tags{ + "module": moduleName, + "workspaceId": warehouse.WorkspaceID, + "destType": r.destType, + "sourceId": warehouse.Source.ID, + "destinationId": warehouse.Destination.ID, + })) + }) + t.Run("exclusion window", func(t *testing.T) { + ctx := context.Background() + + statsStore, err := memstats.New() + require.NoError(t, err) + + warehouse := model.Warehouse{ + WorkspaceID: "test-workspaceID", + Source: backendconfig.SourceT{ + ID: "test-sourceID", + Name: "test-sourceName", + Enabled: true, + }, + Destination: backendconfig.DestinationT{ + ID: "test-destinationID", + Name: "test-destinationName", + Enabled: true, + Config: map[string]any{ + "syncFrequency": "30", + "excludeWindow": map[string]any{ + "excludeWindowStartTime": "05:09", + "excludeWindowEndTime": "09:07", }, }, - } + }, + } + + r := Router{ + conf: config.New(), + destType: whutils.POSTGRES, + now: func() time.Time { + return time.Date(2023, 1, 1, 6, 19, 0, 0, time.UTC) + }, + statsFactory: statsStore, + logger: logger.NOP, + } + + require.NoError(t, r.trackSync(ctx, &warehouse)) + require.Nil(t, statsStore.Get("warehouse_track_upload_missing", stats.Tags{ + "module": moduleName, + "workspaceId": warehouse.WorkspaceID, + "destType": r.destType, + "sourceId": warehouse.Source.ID, + "destinationId": warehouse.Destination.ID, + })) + }) + t.Run("no staging files present", func(t *testing.T) { + db, ctx := setupDB(t), context.Background() - if tc.NowSQL != "" { - nowSQL = tc.NowSQL - } + statsStore, err := memstats.New() + require.NoError(t, err) - handle := Router{ - conf: config.New(), - destType: destType, - now: func() time.Time { - return now + warehouse := model.Warehouse{ + WorkspaceID: "test-workspaceID", + Source: backendconfig.SourceT{ + ID: "test-sourceID", + Name: "test-sourceName", + Enabled: true, + }, + Destination: backendconfig.DestinationT{ + ID: "test-destinationID", + Name: "test-destinationName", + Enabled: true, + Config: map[string]any{ + "syncFrequency": "30", }, - nowSQL: nowSQL, - statsFactory: statsStore, - db: sqlquerywrapper.New(pgResource.DB), - logger: logger.NOP, - } - - err = handle.Track(ctx, &warehouse, conf) - if tc.wantErr != nil { - require.EqualError(t, err, tc.wantErr.Error()) - return - } - require.NoError(t, err) - - m := statsStore.Get("warehouse_track_upload_missing", stats.Tags{ - "module": moduleName, - "workspaceId": warehouse.WorkspaceID, - "destType": handle.destType, - "sourceId": warehouse.Source.ID, - "destinationId": warehouse.Destination.ID, - "warehouseID": misc.GetTagName( - warehouse.Destination.ID, - warehouse.Source.Name, - warehouse.Destination.Name, - misc.TailTruncateStr(warehouse.Source.ID, 6)), + }, + } + + r := Router{ + conf: config.New(), + destType: whutils.POSTGRES, + now: timeutil.Now, + statsFactory: statsStore, + db: db, + logger: logger.NOP, + } + r.config.uploadBufferTimeInMin = config.SingleValueLoader(30 * time.Minute) + + require.NoError(t, r.trackSync(ctx, &warehouse)) + require.Nil(t, statsStore.Get("warehouse_track_upload_missing", stats.Tags{ + "module": moduleName, + "workspaceId": warehouse.WorkspaceID, + "destType": r.destType, + "sourceId": warehouse.Source.ID, + "destinationId": warehouse.Destination.ID, + })) + }) + t.Run("staging files without missing uploads", func(t *testing.T) { + testCases := []struct { + name string + status string + }{ + {name: "ExportedData", status: model.ExportedData}, + {name: "Aborted", status: model.Aborted}, + {name: "Failed", status: model.ExportingDataFailed}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + db, ctx := setupDB(t), context.Background() + + now := time.Date(2023, 1, 1, 6, 19, 0, 0, time.UTC) + nowSQL := fmt.Sprintf("'%s'::timestamp", now.Format(time.DateTime)) + + repoStaging := repo.NewStagingFiles(db, repo.WithNow(func() time.Time { + return now.Add(-time.Hour*1 - time.Minute*30) + })) + repoUpload := repo.NewUploads(db, repo.WithNow(func() time.Time { + return now.Add(-time.Hour * 1) + })) + + stagingID, err := repoStaging.Insert(ctx, &model.StagingFileWithSchema{ + StagingFile: model.StagingFile{ + WorkspaceID: "test-workspaceID", + SourceID: "test-sourceID", + DestinationID: "test-destinationID", + }, + }) + require.NoError(t, err) + + _, err = repoUpload.CreateWithStagingFiles(ctx, model.Upload{ + WorkspaceID: "test-workspaceID", + Namespace: "namespace", + SourceID: "test-sourceID", + DestinationID: "test-destinationID", + DestinationType: whutils.POSTGRES, + Status: tc.status, + }, []*model.StagingFile{{ + ID: stagingID, + WorkspaceID: "test-workspaceID", + SourceID: "test-sourceID", + DestinationID: "test-destinationID", + }}) + require.NoError(t, err) + + statsStore, err := memstats.New() + require.NoError(t, err) + + warehouse := model.Warehouse{ + WorkspaceID: "test-workspaceID", + Source: backendconfig.SourceT{ + ID: "test-sourceID", + Name: "test-sourceName", + Enabled: true, + }, + Destination: backendconfig.DestinationT{ + ID: "test-destinationID", + Name: whutils.POSTGRES, + Enabled: true, + Config: map[string]any{ + "syncFrequency": "30", + }, + }, + } + + r := Router{ + conf: config.New(), + destType: whutils.POSTGRES, + now: func() time.Time { + return now + }, + nowSQL: nowSQL, + statsFactory: statsStore, + db: db, + logger: logger.NOP, + } + r.config.uploadBufferTimeInMin = config.SingleValueLoader(30 * time.Minute) + + require.NoError(t, r.trackSync(ctx, &warehouse)) + + uploadMissingStat := statsStore.Get("warehouse_track_upload_missing", stats.Tags{ + "module": moduleName, + "workspaceId": warehouse.WorkspaceID, + "destType": r.destType, + "sourceId": warehouse.Source.ID, + "destinationId": warehouse.Destination.ID, + }) + require.NotNil(t, uploadMissingStat) + require.EqualValues(t, 0, uploadMissingStat.LastValue()) }) + } + }) + t.Run("staging files with missing uploads", func(t *testing.T) { + db, ctx := setupDB(t), context.Background() - if tc.missing { - require.EqualValues(t, m.LastValue(), 1) - } else { - require.EqualValues(t, m.LastValue(), 0) - } + now := time.Date(2023, 1, 1, 6, 19, 0, 0, time.UTC) + nowSQL := fmt.Sprintf("'%s'::timestamp", now.Format(time.DateTime)) + + repoStaging := repo.NewStagingFiles(db, repo.WithNow(func() time.Time { + return now.Add(-time.Hour*1 - time.Minute*30) + })) + + _, err := repoStaging.Insert(ctx, &model.StagingFileWithSchema{ + StagingFile: model.StagingFile{ + WorkspaceID: "test-workspaceID", + SourceID: "test-sourceID", + DestinationID: "test-destinationID", + }, }) - } -} + require.NoError(t, err) -func TestRouter_CronTracker(t *testing.T) { + statsStore, err := memstats.New() + require.NoError(t, err) + + warehouse := model.Warehouse{ + WorkspaceID: "test-workspaceID", + Source: backendconfig.SourceT{ + ID: "test-sourceID", + Name: "test-sourceName", + Enabled: true, + }, + Destination: backendconfig.DestinationT{ + ID: "test-destinationID", + Name: whutils.POSTGRES, + Enabled: true, + Config: map[string]any{ + "syncFrequency": "30", + }, + }, + } + + r := Router{ + conf: config.New(), + destType: whutils.POSTGRES, + now: func() time.Time { + return now + }, + nowSQL: nowSQL, + statsFactory: statsStore, + db: db, + logger: logger.NOP, + } + r.config.uploadBufferTimeInMin = config.SingleValueLoader(30 * time.Minute) + + require.NoError(t, r.trackSync(ctx, &warehouse)) + + uploadMissingStat := statsStore.Get("warehouse_track_upload_missing", stats.Tags{ + "module": moduleName, + "workspaceId": warehouse.WorkspaceID, + "destType": r.destType, + "sourceId": warehouse.Source.ID, + "destinationId": warehouse.Destination.ID, + }) + require.NotNil(t, uploadMissingStat) + require.EqualValues(t, 1, uploadMissingStat.LastValue()) + }) t.Run("context cancelled", func(t *testing.T) { - t.Parallel() + db, ctx := setupDB(t), context.Background() - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(ctx) cancel() - mockCtrl := gomock.NewController(t) - mockLogger := mock_logger.NewMockLogger(mockCtrl) - statsStore, err := memstats.New() require.NoError(t, err) + warehouse := model.Warehouse{ + WorkspaceID: "test-workspaceID", + Source: backendconfig.SourceT{ + ID: "test-sourceID", + Name: "test-sourceName", + Enabled: true, + }, + Destination: backendconfig.DestinationT{ + ID: "test-destinationID", + Name: "test-destinationName", + Enabled: true, + Config: map[string]any{ + "syncFrequency": "30", + }, + }, + } + r := Router{ - logger: mockLogger, + conf: config.New(), + destType: whutils.POSTGRES, + now: time.Now, statsFactory: statsStore, - destType: warehouseutils.POSTGRES, + db: db, + logger: logger.NOP, + warehouses: []model.Warehouse{warehouse}, } + r.config.uploadBufferTimeInMin = config.SingleValueLoader(30 * time.Minute) + r.config.cronTrackerRetries = config.SingleValueLoader(int64(5)) + r.stats.cronTrackerExecTimestamp = statsStore.NewTaggedStat("warehouse_cron_tracker_timestamp_seconds", stats.GaugeType, stats.Tags{"module": moduleName, "destType": r.destType}) + + require.NoError(t, r.cronTracker(ctx)) + require.Nil(t, statsStore.Get("warehouse_track_upload_missing", stats.Tags{ + "module": moduleName, + "workspaceId": warehouse.WorkspaceID, + "destType": r.destType, + "sourceId": warehouse.Source.ID, + "destinationId": warehouse.Destination.ID, + })) + }) +} - mockLogger.EXPECT().Infon("context is cancelled, stopped running tracking").Times(1) +func setupDB(t testing.TB) *sqlmiddleware.DB { + t.Helper() - executionTime := time.Now().Unix() - err = r.CronTracker(ctx) - require.NoError(t, err) + pool, err := dockertest.NewPool("") + require.NoError(t, err) - m := statsStore.GetByName("warehouse_cron_tracker_timestamp_seconds") - require.Equal(t, len(m), 1) - require.Equal(t, m[0].Name, "warehouse_cron_tracker_timestamp_seconds") - require.Equal(t, m[0].Tags, stats.Tags{ - "module": moduleName, - "destType": warehouseutils.POSTGRES, - }) - require.GreaterOrEqual(t, m[0].Value, float64(executionTime)) - }) + pgResource, err := postgres.Setup(pool, t) + require.NoError(t, err) + + require.NoError(t, (&migrator.Migrator{ + Handle: pgResource.DB, + MigrationsTable: "wh_schema_migrations", + }).Migrate("warehouse")) + + return sqlmiddleware.New(pgResource.DB) }