From 5577d6f6eb4e4dc5017d804272296e23f10ab2d9 Mon Sep 17 00:00:00 2001 From: meghaniankov Date: Tue, 30 Jan 2024 16:29:40 +0000 Subject: [PATCH 1/8] add zipkin tracing provider config --- pkg/envoy/boilerplate.go | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/pkg/envoy/boilerplate.go b/pkg/envoy/boilerplate.go index 50041ce..d677d1d 100644 --- a/pkg/envoy/boilerplate.go +++ b/pkg/envoy/boilerplate.go @@ -11,6 +11,7 @@ import ( endpoint "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" listener "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" route "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + tracing "github.com/envoyproxy/go-control-plane/envoy/config/trace/v3" eal "github.com/envoyproxy/go-control-plane/envoy/extensions/access_loggers/file/v3" gal "github.com/envoyproxy/go-control-plane/envoy/extensions/access_loggers/grpc/v3" eauthz "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/ext_authz/v3" @@ -205,6 +206,16 @@ func makeFileAccessLog(cfg AccessLogger) *eal.FileAccessLog { return accessLogConfig } +func makeZipkinTracingProvider() *tracing.ZipkinConfig { + zipkinTracingProviderConfig := &tracing.ZipkinConfig{ + CollectorCluster: "zipkin", + CollectorEndpoint: "/api/v2/spans", + CollectorEndpointVersion: tracing.ZipkinConfig_HTTP_JSON, + } + + return zipkinTracingProviderConfig +} + func (c *KubernetesConfigurator) makeConnectionManager(virtualHosts []*route.VirtualHost) (*hcm.HttpConnectionManager, error) { // Access Logs accessLogConfig := makeFileAccessLog(c.accessLogger) @@ -261,6 +272,8 @@ func (c *KubernetesConfigurator) makeConnectionManager(virtualHosts []*route.Vir return &hcm.HttpConnectionManager{}, err } + zipkinTracingProvider, err := anypb.New(makeZipkinTracingProvider()) + return &hcm.HttpConnectionManager{ CodecType: hcm.HttpConnectionManager_AUTO, StatPrefix: "ingress_http", @@ -276,7 +289,12 @@ func (c *KubernetesConfigurator) makeConnectionManager(virtualHosts []*route.Vir VirtualHosts: virtualHosts, }, }, - Tracing: &hcm.HttpConnectionManager_Tracing{}, + Tracing: &hcm.HttpConnectionManager_Tracing{ + Provider: &tracing.Tracing_Http{ + Name: "config.trace.v3.Tracing.Http", + ConfigType: &tracing.Tracing_Http_TypedConfig{TypedConfig: zipkinTracingProvider}, + }, + }, AccessLog: accessLoggers, UseRemoteAddress: &wrapperspb.BoolValue{Value: c.useRemoteAddress}, }, nil From 78811f69012eca99e3ce5ac8712c854befd005e5 Mon Sep 17 00:00:00 2001 From: meghaniankov Date: Tue, 30 Jan 2024 16:34:16 +0000 Subject: [PATCH 2/8] temp build image on any branch --- .github/workflows/push.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/push.yaml b/.github/workflows/push.yaml index e9bac1d..b67249d 100644 --- a/.github/workflows/push.yaml +++ b/.github/workflows/push.yaml @@ -26,7 +26,6 @@ jobs: name: bin path: bin/ docker-build-push: - if: github.ref_name == 'master' || startsWith(github.ref, 'refs/tags/v') needs: build runs-on: ubuntu-latest steps: From 6d5c95c68d70c38b41f9b1062fdd4436d552f2b8 Mon Sep 17 00:00:00 2001 From: meghaniankov Date: Wed, 31 Jan 2024 12:14:34 +0000 Subject: [PATCH 3/8] make tracing provider config optional --- cmd/root.go | 2 ++ pkg/envoy/boilerplate.go | 27 ++++++++++++++++++--------- pkg/envoy/configurator.go | 5 +++-- 3 files changed, 23 insertions(+), 11 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index 1743115..120b437 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -91,6 +91,7 @@ func init() { rootCmd.PersistentFlags().Int32("max-ejection-percentage", -1, "maximal percentage of hosts ejected via outlier detection. Set to >=0 to activate outlier detection in envoy.") rootCmd.PersistentFlags().Int64("host-selection-retry-attempts", -1, "Number of host selection retry attempts. Set to value >=0 to enable") rootCmd.PersistentFlags().String("retry-on", "5xx", "default comma-separated list of retry policies") + rootCmd.PersistentFlags().String("tracing-provider", "", "HTTP Connection Manager tracing provider block to include") rootCmd.PersistentFlags().Duration("upstream-healthcheck-interval", 10*time.Second, "duration of the upstream health check interval") rootCmd.PersistentFlags().Duration("upstream-healthcheck-timeout", 5*time.Second, "timeout of the upstream healthchecks") rootCmd.PersistentFlags().Uint32("upstream-healthcheck-healthy", 3, "number of successful healthchecks before the backend is considered healthy") @@ -123,6 +124,7 @@ func init() { viper.BindPFlag("maxEjectionPercentage", rootCmd.PersistentFlags().Lookup("max-ejection-percentage")) viper.BindPFlag("hostSelectionRetryAttempts", rootCmd.PersistentFlags().Lookup("host-selection-retry-attempts")) viper.BindPFlag("retryOn", rootCmd.PersistentFlags().Lookup("retry-on")) + viper.BindPFlag("tracingProvider", rootCmd.PersistentFlags().Lookup("tracing-provider")) viper.BindPFlag("upstreamHealthCheck.interval", rootCmd.PersistentFlags().Lookup("upstream-healthcheck-interval")) viper.BindPFlag("upstreamHealthCheck.timeout", rootCmd.PersistentFlags().Lookup("upstream-healthcheck-timeout")) viper.BindPFlag("upstreamHealthCheck.healthyThreshold", rootCmd.PersistentFlags().Lookup("upstream-healthcheck-healthy")) diff --git a/pkg/envoy/boilerplate.go b/pkg/envoy/boilerplate.go index d677d1d..ca1f735 100644 --- a/pkg/envoy/boilerplate.go +++ b/pkg/envoy/boilerplate.go @@ -216,7 +216,7 @@ func makeZipkinTracingProvider() *tracing.ZipkinConfig { return zipkinTracingProviderConfig } -func (c *KubernetesConfigurator) makeConnectionManager(virtualHosts []*route.VirtualHost) (*hcm.HttpConnectionManager, error) { +func (c *KubernetesConfigurator) makeConnectionManager(virtualHosts []*route.VirtualHost, tracingProvider string) (*hcm.HttpConnectionManager, error) { // Access Logs accessLogConfig := makeFileAccessLog(c.accessLogger) anyAccessLogConfig, err := anypb.New(accessLogConfig) @@ -272,7 +272,21 @@ func (c *KubernetesConfigurator) makeConnectionManager(virtualHosts []*route.Vir return &hcm.HttpConnectionManager{}, err } - zipkinTracingProvider, err := anypb.New(makeZipkinTracingProvider()) + tracingProviderConfig := &hcm.HttpConnectionManager_Tracing{} + + if tracingProvider == "zipkin" { + zipkinTracingProvider, err := anypb.New(makeZipkinTracingProvider()) + if err != nil { + log.Fatal(err) + } + + tracingProviderConfig = &hcm.HttpConnectionManager_Tracing{ + Provider: &tracing.Tracing_Http{ + Name: "config.trace.v3.Tracing.Http", + ConfigType: &tracing.Tracing_Http_TypedConfig{TypedConfig: zipkinTracingProvider}, + }, + } + } return &hcm.HttpConnectionManager{ CodecType: hcm.HttpConnectionManager_AUTO, @@ -289,19 +303,14 @@ func (c *KubernetesConfigurator) makeConnectionManager(virtualHosts []*route.Vir VirtualHosts: virtualHosts, }, }, - Tracing: &hcm.HttpConnectionManager_Tracing{ - Provider: &tracing.Tracing_Http{ - Name: "config.trace.v3.Tracing.Http", - ConfigType: &tracing.Tracing_Http_TypedConfig{TypedConfig: zipkinTracingProvider}, - }, - }, + Tracing: tracingProviderConfig, AccessLog: accessLoggers, UseRemoteAddress: &wrapperspb.BoolValue{Value: c.useRemoteAddress}, }, nil } func (c *KubernetesConfigurator) makeFilterChain(certificate Certificate, virtualHosts []*route.VirtualHost) (listener.FilterChain, error) { - httpConnectionManager, err := c.makeConnectionManager(virtualHosts) + httpConnectionManager, err := c.makeConnectionManager(virtualHosts, c.tracingProvider) if err != nil { return listener.FilterChain{}, fmt.Errorf("failed to get httpConnectionManager: %s", err) } diff --git a/pkg/envoy/configurator.go b/pkg/envoy/configurator.go index fc13a9a..ebeae64 100644 --- a/pkg/envoy/configurator.go +++ b/pkg/envoy/configurator.go @@ -69,6 +69,7 @@ type KubernetesConfigurator struct { httpGrpcLogger HttpGrpcLogger accessLogger AccessLogger defaultRetryOn string + tracingProvider string previousConfig *envoyConfiguration listenerVersion string @@ -85,7 +86,7 @@ func NewKubernetesConfigurator(nodeID string, certificates []Certificate, ca str return c } -//Generate creates a new snapshot +// Generate creates a new snapshot func (c *KubernetesConfigurator) Generate(ingresses []*k8s.Ingress, secrets []*v1.Secret) (cache.Snapshot, error) { c.Lock() defer c.Unlock() @@ -236,7 +237,7 @@ func (c *KubernetesConfigurator) generateHTTPFilterChain(config *envoyConfigurat virtualHosts = append(virtualHosts, vhost) } - httpConnectionManager, err := c.makeConnectionManager(virtualHosts) + httpConnectionManager, err := c.makeConnectionManager(virtualHosts, c.tracingProvider) if err != nil { return nil, err } From 5a75ea89d847c9f7026be2d8d470483624d38b29 Mon Sep 17 00:00:00 2001 From: meghaniankov Date: Wed, 31 Jan 2024 15:32:23 +0000 Subject: [PATCH 4/8] fix tracing-provider config --- cmd/root.go | 1 + pkg/envoy/boilerplate.go | 6 +++--- pkg/envoy/configurator.go | 2 +- pkg/envoy/options.go | 7 +++++++ 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index 120b437..441e5fb 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -243,6 +243,7 @@ func main(*cobra.Command, []string) error { envoy.WithSyncSecrets(c.SyncSecrets), envoy.WithDefaultRetryOn(viper.GetString("retryOn")), envoy.WithAccessLog(c.AccessLogger), + envoy.WithTracingProvider(viper.GetString("tracingProvider")), ) snapshotter := envoy.NewSnapshotter(envoyCache, configurator, aggregator) diff --git a/pkg/envoy/boilerplate.go b/pkg/envoy/boilerplate.go index ca1f735..aec1a58 100644 --- a/pkg/envoy/boilerplate.go +++ b/pkg/envoy/boilerplate.go @@ -216,7 +216,7 @@ func makeZipkinTracingProvider() *tracing.ZipkinConfig { return zipkinTracingProviderConfig } -func (c *KubernetesConfigurator) makeConnectionManager(virtualHosts []*route.VirtualHost, tracingProvider string) (*hcm.HttpConnectionManager, error) { +func (c *KubernetesConfigurator) makeConnectionManager(virtualHosts []*route.VirtualHost) (*hcm.HttpConnectionManager, error) { // Access Logs accessLogConfig := makeFileAccessLog(c.accessLogger) anyAccessLogConfig, err := anypb.New(accessLogConfig) @@ -274,7 +274,7 @@ func (c *KubernetesConfigurator) makeConnectionManager(virtualHosts []*route.Vir tracingProviderConfig := &hcm.HttpConnectionManager_Tracing{} - if tracingProvider == "zipkin" { + if c.tracingProvider == "zipkin" { zipkinTracingProvider, err := anypb.New(makeZipkinTracingProvider()) if err != nil { log.Fatal(err) @@ -310,7 +310,7 @@ func (c *KubernetesConfigurator) makeConnectionManager(virtualHosts []*route.Vir } func (c *KubernetesConfigurator) makeFilterChain(certificate Certificate, virtualHosts []*route.VirtualHost) (listener.FilterChain, error) { - httpConnectionManager, err := c.makeConnectionManager(virtualHosts, c.tracingProvider) + httpConnectionManager, err := c.makeConnectionManager(virtualHosts) if err != nil { return listener.FilterChain{}, fmt.Errorf("failed to get httpConnectionManager: %s", err) } diff --git a/pkg/envoy/configurator.go b/pkg/envoy/configurator.go index ebeae64..fc3be3c 100644 --- a/pkg/envoy/configurator.go +++ b/pkg/envoy/configurator.go @@ -237,7 +237,7 @@ func (c *KubernetesConfigurator) generateHTTPFilterChain(config *envoyConfigurat virtualHosts = append(virtualHosts, vhost) } - httpConnectionManager, err := c.makeConnectionManager(virtualHosts, c.tracingProvider) + httpConnectionManager, err := c.makeConnectionManager(virtualHosts) if err != nil { return nil, err } diff --git a/pkg/envoy/options.go b/pkg/envoy/options.go index 4ccef75..7cc4e68 100644 --- a/pkg/envoy/options.go +++ b/pkg/envoy/options.go @@ -85,3 +85,10 @@ func WithAccessLog(accessLogger AccessLogger) option { c.accessLogger = accessLogger } } + +// WithTracingProvider configures the tracing provider for HTTP connection manager +func WithTracingProvider(tracingProvider string) option { + return func(c *KubernetesConfigurator) { + c.tracingProvider = tracingProvider + } +} From c83d6704445fef03ae8cd41f601bc367009981b9 Mon Sep 17 00:00:00 2001 From: meghaniankov Date: Wed, 31 Jan 2024 16:12:39 +0000 Subject: [PATCH 5/8] refactor --- pkg/envoy/boilerplate.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/envoy/boilerplate.go b/pkg/envoy/boilerplate.go index aec1a58..0e804e0 100644 --- a/pkg/envoy/boilerplate.go +++ b/pkg/envoy/boilerplate.go @@ -272,7 +272,7 @@ func (c *KubernetesConfigurator) makeConnectionManager(virtualHosts []*route.Vir return &hcm.HttpConnectionManager{}, err } - tracingProviderConfig := &hcm.HttpConnectionManager_Tracing{} + tracingConfig := &hcm.HttpConnectionManager_Tracing{} if c.tracingProvider == "zipkin" { zipkinTracingProvider, err := anypb.New(makeZipkinTracingProvider()) @@ -280,7 +280,7 @@ func (c *KubernetesConfigurator) makeConnectionManager(virtualHosts []*route.Vir log.Fatal(err) } - tracingProviderConfig = &hcm.HttpConnectionManager_Tracing{ + tracingConfig = &hcm.HttpConnectionManager_Tracing{ Provider: &tracing.Tracing_Http{ Name: "config.trace.v3.Tracing.Http", ConfigType: &tracing.Tracing_Http_TypedConfig{TypedConfig: zipkinTracingProvider}, @@ -303,7 +303,7 @@ func (c *KubernetesConfigurator) makeConnectionManager(virtualHosts []*route.Vir VirtualHosts: virtualHosts, }, }, - Tracing: tracingProviderConfig, + Tracing: tracingConfig, AccessLog: accessLoggers, UseRemoteAddress: &wrapperspb.BoolValue{Value: c.useRemoteAddress}, }, nil From addab84034ccdb1887c28f05894d7390a6735e28 Mon Sep 17 00:00:00 2001 From: meghaniankov Date: Thu, 1 Feb 2024 11:56:48 +0000 Subject: [PATCH 6/8] refactor --- pkg/envoy/boilerplate.go | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/pkg/envoy/boilerplate.go b/pkg/envoy/boilerplate.go index 0e804e0..d2579df 100644 --- a/pkg/envoy/boilerplate.go +++ b/pkg/envoy/boilerplate.go @@ -277,14 +277,12 @@ func (c *KubernetesConfigurator) makeConnectionManager(virtualHosts []*route.Vir if c.tracingProvider == "zipkin" { zipkinTracingProvider, err := anypb.New(makeZipkinTracingProvider()) if err != nil { - log.Fatal(err) + log.Fatalf("failed to set zipkin tracing provider config: %s", err) } - tracingConfig = &hcm.HttpConnectionManager_Tracing{ - Provider: &tracing.Tracing_Http{ - Name: "config.trace.v3.Tracing.Http", - ConfigType: &tracing.Tracing_Http_TypedConfig{TypedConfig: zipkinTracingProvider}, - }, + tracingConfig.Provider = &tracing.Tracing_Http{ + Name: "config.trace.v3.Tracing.Http", + ConfigType: &tracing.Tracing_Http_TypedConfig{TypedConfig: zipkinTracingProvider}, } } From 8774f42f2324b5d330c8fa79cb26f7c018b4665d Mon Sep 17 00:00:00 2001 From: meghaniankov Date: Thu, 1 Feb 2024 11:57:41 +0000 Subject: [PATCH 7/8] docker-build-push on master or tag only --- .github/workflows/push.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/push.yaml b/.github/workflows/push.yaml index b67249d..e9bac1d 100644 --- a/.github/workflows/push.yaml +++ b/.github/workflows/push.yaml @@ -26,6 +26,7 @@ jobs: name: bin path: bin/ docker-build-push: + if: github.ref_name == 'master' || startsWith(github.ref, 'refs/tags/v') needs: build runs-on: ubuntu-latest steps: From 4566964c11d3ef67bdb925c8cc47c63db6be9dd2 Mon Sep 17 00:00:00 2001 From: meghaniankov Date: Thu, 1 Feb 2024 12:14:23 +0000 Subject: [PATCH 8/8] add tracing-provider flag details on README --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 4c6a376..6d2d405 100644 --- a/README.md +++ b/README.md @@ -202,6 +202,7 @@ The Yggdrasil-specific metrics which are available from the API are: --max-ejection-percentage int32 maximal percentage of hosts ejected via outlier detection. Set to >=0 to activate outlier detection in envoy. (default -1) --node-name string envoy node name --retry-on string default comma-separated list of retry policies (default "5xx") +--tracing-provider name of HTTP Connection Manager tracing provider to include - currently only zipkin config is supported --upstream-healthcheck-healthy uint32 number of successful healthchecks before the backend is considered healthy (default 3) --upstream-healthcheck-interval duration duration of the upstream health check interval (default 10s) --upstream-healthcheck-timeout duration timeout of the upstream healthchecks (default 5s)