Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore: data plane changes for credentials in transformation #4715

Merged
merged 15 commits into from
Jun 13, 2024
Merged

chore: data plane changes for credentials in transformation #4715

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
8a398fc
chore: data plane changes for credentials in transformation
kanishkkatara May 28, 2024
56a33a0
chore: passing id in credentials
kanishkkatara May 30, 2024
1e8b2fc
chore: changes in tranformer tests
kanishkkatara Jun 3, 2024
b8d453c
fix: changes in processor
kanishkkatara Jun 4, 2024
64e9ed7
Merge branch 'master' into feature/dat-1204-data-plane-flow
kanishkkatara Jun 4, 2024
1ab68d9
chore: removed unused method
kanishkkatara Jun 4, 2024
417c0f6
chore: added test for processor
kanishkkatara Jun 7, 2024
0ad71b6
fix: changes in tests
kanishkkatara Jun 11, 2024
ed15100
chore: transformer processor contract test (#4787)
achettyiitr Jun 11, 2024
70b5e0e
Revert "fix: changes in tests"
kanishkkatara Jun 12, 2024
5be93fc
Revert "chore: added test for processor"
kanishkkatara Jun 12, 2024
6a78a12
fix: reverted changes in config_builder
kanishkkatara Jun 12, 2024
83a448f
chore: added TODO comment
kanishkkatara Jun 12, 2024
03ac269
Merge branch 'master' into feature/dat-1204-data-plane-flow
kanishkkatara Jun 12, 2024
9069873
Merge branch 'master' into feature/dat-1204-data-plane-flow
achettyiitr Jun 13, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions backend-config/backend-config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,7 @@ func filterProcessorEnabledDestinations(config ConfigT) ConfigT {
var modifiedConfig ConfigT
modifiedConfig.Libraries = config.Libraries
modifiedConfig.Sources = make([]SourceT, 0)
modifiedConfig.Credentials = config.Credentials
for _, source := range config.Sources {
var destinations []DestinationT
for _, destination := range source.Destinations { // TODO skipcq: CRT-P0006
Expand Down
7 changes: 7 additions & 0 deletions backend-config/types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,12 @@ type SourceT struct {
}
}

type Credential struct {
Key string `json:"key"`
Value string `json:"value"`
IsSecret bool `json:"isSecret"`
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

do we need IsSecret at server end? I suppose workspace config gives the raw Value rt?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, we receive IsSecret from the workspace config, and we need to send it to the transformer

}

func (s *SourceT) IsReplaySource() bool {
return s.OriginalID != ""
}
Expand All @@ -87,6 +93,7 @@ type ConfigT struct {
ConnectionFlags ConnectionFlags `json:"flags"`
Settings Settings `json:"settings"`
UpdatedAt time.Time `json:"updatedAt"`
Credentials map[string]Credential `json:"credentials"`
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The addition of the Credentials field in ConfigT is appropriate for managing multiple credentials. However, there's an issue with an undefined EventReplayConfig type used elsewhere in the file.

+ import "path/to/event_replay_config" // Ensure this path is correct

Committable suggestion was skipped due to low confidence.

}

func (c *ConfigT) SourcesMap() map[string]*SourceT {
Expand Down
26 changes: 26 additions & 0 deletions processor/processor.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -144,6 +144,7 @@ type Handle struct {
eventSchemaV2Enabled bool
archivalEnabled config.ValueLoader[bool]
eventAuditEnabled map[string]bool
credentialsMap map[string][]transformer.Credential
kanishkkatara marked this conversation as resolved.
Show resolved Hide resolved
}

drainConfig struct {
Expand Down Expand Up @@ -797,6 +798,7 @@ func (proc *Handle) backendConfigSubscriber(ctx context.Context) {
sourceIdDestinationMap = make(map[string][]backendconfig.DestinationT)
sourceIdSourceMap = map[string]backendconfig.SourceT{}
eventAuditEnabled = make(map[string]bool)
credentialsMap = make(map[string][]transformer.Credential)
)
for workspaceID, wConfig := range config {
for i := range wConfig.Sources {
Expand All @@ -819,6 +821,14 @@ func (proc *Handle) backendConfigSubscriber(ctx context.Context) {
}
workspaceLibrariesMap[workspaceID] = wConfig.Libraries
eventAuditEnabled[workspaceID] = wConfig.Settings.EventAuditEnabled
credentialsMap[workspaceID] = lo.MapToSlice(wConfig.Credentials, func(key string, value backendconfig.Credential) transformer.Credential {
return transformer.Credential{
ID: key,
Key: value.Key,
Value: value.Value,
IsSecret: value.IsSecret,
}
})
}
proc.config.configSubscriberLock.Lock()
proc.config.oneTrustConsentCategoriesMap = oneTrustConsentCategoriesMap
Expand All @@ -828,6 +838,7 @@ func (proc *Handle) backendConfigSubscriber(ctx context.Context) {
proc.config.sourceIdDestinationMap = sourceIdDestinationMap
proc.config.sourceIdSourceMap = sourceIdSourceMap
proc.config.eventAuditEnabled = eventAuditEnabled
proc.config.credentialsMap = credentialsMap
proc.config.configSubscriberLock.Unlock()
if !initDone {
initDone = true
Expand All @@ -836,6 +847,19 @@ func (proc *Handle) backendConfigSubscriber(ctx context.Context) {
}
}

func ConvertMapToList(credentialsMap map[string]backendconfig.Credential) []transformer.Credential {
achettyiitr marked this conversation as resolved.
Show resolved Hide resolved
var credentialsList []transformer.Credential
for id, credential := range credentialsMap {
credentialsList = append(credentialsList, transformer.Credential{
ID: id,
Key: credential.Key,
Value: credential.Value,
IsSecret: credential.IsSecret,
})
}
return credentialsList
}

func (proc *Handle) getWorkspaceLibraries(workspaceID string) backendconfig.LibrariesT {
proc.config.configSubscriberLock.RLock()
defer proc.config.configSubscriberLock.RUnlock()
Expand Down Expand Up @@ -1098,6 +1122,7 @@ func (proc *Handle) getTransformerEvents(
Message: userTransformedEvent.Output,
Metadata: *eventMetadata,
Destination: *destination,
Credentials: proc.config.credentialsMap[commonMetaData.WorkspaceID],
}
eventsToTransform = append(eventsToTransform, updatedEvent)
}
Expand Down Expand Up @@ -1949,6 +1974,7 @@ func (proc *Handle) processJobsForDest(partition string, subJobs subJob) *transf
shallowEventCopy.Metadata.TransformationID = destination.Transformations[0].ID
shallowEventCopy.Metadata.TransformationVersionID = destination.Transformations[0].VersionID
}
shallowEventCopy.Credentials = proc.config.credentialsMap[destination.WorkspaceID]
filterConfig(&shallowEventCopy)
metadata := shallowEventCopy.Metadata
srcAndDestKey := getKeyFromSourceAndDest(metadata.SourceID, metadata.DestinationID)
Expand Down
8 changes: 8 additions & 0 deletions processor/transformer/transformer.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,14 @@ type TransformerEvent struct {
Metadata Metadata `json:"metadata"`
Destination backendconfig.DestinationT `json:"destination"`
Libraries []backendconfig.LibraryT `json:"libraries"`
Credentials []Credential `json:"credentials"`
}

type Credential struct {
ID string `json:"id"`
Key string `json:"key"`
Value string `json:"value"`
IsSecret bool `json:"isSecret"`
}

func isJobTerminated(status int) bool {
Expand Down
56 changes: 56 additions & 0 deletions processor/transformer/transformer_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -203,6 +203,14 @@ func TestTransformer(t *testing.T) {
"src-key-1": msgID,
"forceStatusCode": statusCode,
},
Credentials: []Credential{
{
ID: "test-credential",
Key: "test-key",
Value: "test-value",
IsSecret: false,
},
},
}

tResp := TransformerResponse{
Expand Down Expand Up @@ -238,6 +246,14 @@ func TestTransformer(t *testing.T) {
Message: map[string]interface{}{
"src-key-1": msgID,
},
Credentials: []Credential{
{
ID: "test-credential",
Key: "test-key",
Value: "test-value",
IsSecret: false,
},
},
})

testCases := []struct {
Expand Down Expand Up @@ -347,6 +363,14 @@ func TestTransformer(t *testing.T) {
Message: map[string]interface{}{
"src-key-1": msgID,
},
Credentials: []Credential{
{
ID: "test-credential",
Key: "test-key",
Value: "test-value",
IsSecret: false,
},
},
})

elt := &endlessLoopTransformer{
Expand Down Expand Up @@ -404,6 +428,14 @@ func TestTransformer(t *testing.T) {
},
},
},
Credentials: []Credential{
{
ID: "test-credential",
Key: "test-key",
Value: "test-value",
IsSecret: false,
},
},
})

testCases := []struct {
Expand Down Expand Up @@ -529,6 +561,14 @@ func TestTransformer(t *testing.T) {
},
},
},
Credentials: []Credential{
{
ID: "test-credential",
Key: "test-key",
Value: "test-value",
IsSecret: false,
},
},
})

testCases := []struct {
Expand Down Expand Up @@ -637,6 +677,14 @@ func TestTransformer(t *testing.T) {
},
},
},
Credentials: []Credential{
{
ID: "test-credential",
Key: "test-key",
Value: "test-value",
IsSecret: false,
},
},
})

t.Run("Destination transformations", func(t *testing.T) {
Expand Down Expand Up @@ -712,6 +760,14 @@ func TestTransformer(t *testing.T) {
},
},
},
Credentials: []Credential{
{
ID: "test-credential",
Key: "test-key",
Value: "test-value",
IsSecret: false,
},
},
})

rsp := tr.Transform(context.TODO(), events, 10)
Expand Down
12 changes: 12 additions & 0 deletions schema-forwarder/internal/testdata/configdata.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -189,4 +189,16 @@ var SampleBackendConfig = backendconfig.ConfigT{
},
EventAuditEnabled: false,
},
Credentials: map[string]backendconfig.Credential{
"cred1": {
Key: "key1",
Value: "value1",
IsSecret: false,
},
"cred2": {
Key: "key2",
Value: "value2",
IsSecret: true,
},
},
}
Loading