Merge master, fix conflicts

Signed-off-by: Laurent Marchaud <[email protected]>

.tool-versions

@@ -0,0 +1 @@
+golang 1.17.13

Makefile

@@ -5,14 +5,16 @@ BIN_DARWIN = $(BIN)-darwin-$(ARCH)
 
 SOURCES := $(shell find . -iname '*.go')
 
-.PHONY: test clean all
+.PHONY: test clean all build-linux
 
-all: build-darwin build-linux
+all: build-darwin $(BIN_LINUX)
 
 build-darwin: $(SOURCES)
 	GOARCH=$(ARCH) GOOS=darwin go build -o $(BIN_DARWIN)
 
-build-linux: $(SOURCES)
+build-linux: $(BIN_LINUX)
+
+$(BIN_LINUX): $(SOURCES)
 	GOARCH=$(ARCH) GOOS=linux CGO_ENABLED=0 go build -o $(BIN_LINUX)
 
 test: $(SOURCES)
@@ -22,7 +24,7 @@ bench: $(SOURCES)
 	go test -run=XX -bench=. $(shell go list ./... | grep -v /vendor)
 
 docker: Dockerfile $(BIN_LINUX)
-	docker image build -t quay.io/uswitch/yggdrasil:devel .
+	docker image build -t registry.airship.rvu.cloud/cloud/yggdrasil:devel .
 
 clean:
 	rm -rf bin/

README.md

@@ -1,7 +1,7 @@
 # Yggdrasil
 Yggdrasil is an Envoy control plane that configures listeners and clusters based off Kubernetes ingresses from multiple Kube Clusters. This allows you to have an envoy cluster acting as a mutli-cluster loadbalancer for Kubernetes. This was something we needed as we wanted our apps to be highly available in the event of a cluster outage but did not want the solution to live inside of Kubernetes itself.
 
-`Note:` Currently we support version 1.19.x of Envoy.</br>
+`Note:` Currently we support versions 1.20.x to 1.26.x of Envoy.</br>
 `Note:` Yggdrasil now uses [Go modules](https://github.com/golang/go/wiki/Modules) to handle dependencies.
 
 ## Usage
@@ -178,17 +178,18 @@ The Yggdrasil-specific metrics which are available from the API are:
 --ca string                                   trustedCA
 --cert string                                 certfile
 --config string                               config file
+--config-dump                                 Enable config dump endpoint at /configdump on the health-address HTTP server
 --debug                                       Log at debug level
 --envoy-listener-ipv4-address string          IPv4 address by the envoy proxy to accept incoming connections (default "0.0.0.0")
 --envoy-port uint32                           port by the envoy proxy to accept incoming connections (default 10000)
 --health-address string                       yggdrasil health API listen address (default "0.0.0.0:8081")
---help                                        help for yggdrasil
+-h, --help                                        help for yggdrasil
 --host-selection-retry-attempts int           Number of host selection retry attempts. Set to value >=0 to enable (default -1)
---retry-on                                    Default comma-separated list of retry policies (default 5xx)
 --http-ext-authz-allow-partial-message        When this field is true, Envoy will buffer the message until max_request_bytes is reached (default true)
 --http-ext-authz-cluster string               The name of the upstream gRPC cluster
 --http-ext-authz-failure-mode-allow           Changes filters behaviour on errors (default true)
 --http-ext-authz-max-request-bytes uint32     Sets the maximum size of a message body that the filter will hold in memory (default 8192)
+--http-ext-authz-pack-as-bytes                When this field is true, Envoy will send the body as raw bytes.
 --http-ext-authz-timeout duration             The timeout for the gRPC request. This is the timeout for a specific request. (default 200ms)
 --http-grpc-logger-cluster string             The name of the upstream gRPC cluster
 --http-grpc-logger-name string                Name of the access log
@@ -200,6 +201,7 @@ The Yggdrasil-specific metrics which are available from the API are:
 --kube-config stringArray                     Path to kube config
 --max-ejection-percentage int32               maximal percentage of hosts ejected via outlier detection. Set to >=0 to activate outlier detection in envoy. (default -1)
 --node-name string                            envoy node name
+--retry-on string                             default comma-separated list of retry policies (default "5xx")
 --upstream-healthcheck-healthy uint32         number of successful healthchecks before the backend is considered healthy (default 3)
 --upstream-healthcheck-interval duration      duration of the upstream health check interval (default 10s)
 --upstream-healthcheck-timeout duration       timeout of the upstream healthchecks (default 5s)

cmd/root.go

@@ -46,6 +46,7 @@ type config struct {
 	UseRemoteAddress           bool                      `json:"useRemoteAddress"`
 	HttpExtAuthz               envoy.HttpExtAuthz        `json:"httpExtAuthz"`
 	HttpGrpcLogger             envoy.HttpGrpcLogger      `json:"httpGrpcLogger"`
+	AccessLogger               envoy.AccessLogger        `json:"accessLogger"`
 }
 
 // Hasher returns node ID as an ID
@@ -63,7 +64,7 @@ var rootCmd = &cobra.Command{
 	RunE:  main,
 }
 
-//Execute runs the function
+// Execute runs the function
 func Execute() {
 	if err := rootCmd.Execute(); err != nil {
 		log.Fatal(err)
@@ -83,6 +84,7 @@ func init() {
 	rootCmd.PersistentFlags().StringSlice("ingress-classes", nil, "Ingress classes to watch")
 	rootCmd.PersistentFlags().StringArrayVar(&kubeConfig, "kube-config", nil, "Path to kube config")
 	rootCmd.PersistentFlags().Bool("debug", false, "Log at debug level")
+	rootCmd.PersistentFlags().Bool("config-dump", false, "Enable config dump endpoint at /configdump on the health-address HTTP server")
 	rootCmd.PersistentFlags().Uint32("upstream-port", 443, "port used to connect to the upstream ingresses")
 	rootCmd.PersistentFlags().String("envoy-listener-ipv4-address", "0.0.0.0", "IPv4 address by the envoy proxy to accept incoming connections")
 	rootCmd.PersistentFlags().Uint32("envoy-port", 10000, "port by the envoy proxy to accept incoming connections")
@@ -103,8 +105,11 @@ func init() {
 	rootCmd.PersistentFlags().Duration("http-ext-authz-timeout", 200*time.Millisecond, "The timeout for the gRPC request. This is the timeout for a specific request.")
 	rootCmd.PersistentFlags().Uint32("http-ext-authz-max-request-bytes", 8192, "Sets the maximum size of a message body that the filter will hold in memory")
 	rootCmd.PersistentFlags().Bool("http-ext-authz-allow-partial-message", true, "When this field is true, Envoy will buffer the message until max_request_bytes is reached")
+	rootCmd.PersistentFlags().Bool("http-ext-authz-pack-as-bytes", false, "When this field is true, Envoy will send the body as raw bytes.")
 	rootCmd.PersistentFlags().Bool("http-ext-authz-failure-mode-allow", true, "Changes filters behaviour on errors")
+
 	viper.BindPFlag("debug", rootCmd.PersistentFlags().Lookup("debug"))
+	viper.BindPFlag("configDump", rootCmd.PersistentFlags().Lookup("config-dump"))
 	viper.BindPFlag("address", rootCmd.PersistentFlags().Lookup("address"))
 	viper.BindPFlag("healthAddress", rootCmd.PersistentFlags().Lookup("health-address"))
 	viper.BindPFlag("nodeName", rootCmd.PersistentFlags().Lookup("node-name"))
@@ -132,6 +137,7 @@ func init() {
 	viper.BindPFlag("httpExtAuthz.timeout", rootCmd.PersistentFlags().Lookup("http-ext-authz-timeout"))
 	viper.BindPFlag("httpExtAuthz.maxRequestBytes", rootCmd.PersistentFlags().Lookup("http-ext-authz-max-request-bytes"))
 	viper.BindPFlag("httpExtAuthz.allowPartialMessage", rootCmd.PersistentFlags().Lookup("http-ext-authz-allow-partial-message"))
+	viper.BindPFlag("httpExtAuthz.packAsBytes", rootCmd.PersistentFlags().Lookup("http-ext-authz-pack-as-bytes"))
 	viper.BindPFlag("httpExtAuthz.FailureModeAllow", rootCmd.PersistentFlags().Lookup("http-ext-authz-failure-mode-allow"))
 }
 
@@ -234,14 +240,15 @@ func main(*cobra.Command, []string) error {
 		envoy.WithHttpGrpcLogger(c.HttpGrpcLogger),
 		envoy.WithSyncSecrets(c.SyncSecrets),
 		envoy.WithDefaultRetryOn(viper.GetString("retryOn")),
+		envoy.WithAccessLog(c.AccessLogger),
 	)
 	snapshotter := envoy.NewSnapshotter(envoyCache, configurator, aggregator)
 
 	go snapshotter.Run(aggregator)
 	go aggregator.Run()
 
 	envoyServer := server.NewServer(ctx, envoyCache, &callbacks{})
-	go runEnvoyServer(envoyServer, viper.GetString("address"), viper.GetString("healthAddress"), ctx.Done())
+	go runEnvoyServer(envoyServer, snapshotter, viper.GetBool("configDump"), viper.GetString("address"), viper.GetString("healthAddress"), ctx.Done())
 
 	<-stopCh
 	return nil

cmd/server.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"net"
 	"net/http"
@@ -16,6 +17,8 @@ import (
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	log "github.com/sirupsen/logrus"
 	"google.golang.org/grpc"
+
+	"github.com/uswitch/yggdrasil/pkg/envoy"
 )
 
 type callbacks struct {
@@ -39,7 +42,7 @@ func (c *callbacks) OnStreamClosed(int64) {}
 func (c *callbacks) OnStreamRequest(int64, *discovery.DiscoveryRequest) error {
 	return nil
 }
-func (c *callbacks) OnStreamResponse(int64, *discovery.DiscoveryRequest, *discovery.DiscoveryResponse) {
+func (c *callbacks) OnStreamResponse(context.Context, int64, *discovery.DiscoveryRequest, *discovery.DiscoveryResponse) {
 }
 func (c *callbacks) OnFetchRequest(context.Context, *discovery.DiscoveryRequest) error {
 	c.fetchReq++
@@ -49,7 +52,7 @@ func (c *callbacks) OnFetchResponse(*discovery.DiscoveryRequest, *discovery.Disc
 	c.fetchResp++
 }
 
-func runEnvoyServer(envoyServer server.Server, address string, healthAddress string, stopCh <-chan struct{}) {
+func runEnvoyServer(envoyServer server.Server, snapshotter *envoy.Snapshotter, enableConfigDump bool, address string, healthAddress string, stopCh <-chan struct{}) {
 
 	grpcServer := grpc.NewServer(
 		grpc.StreamInterceptor(grpc_prometheus.StreamServerInterceptor),
@@ -76,6 +79,9 @@ func runEnvoyServer(envoyServer server.Server, address string, healthAddress str
 
 	healthMux.Handle("/metrics", promhttp.Handler())
 	healthMux.HandleFunc("/healthz", health)
+	if enableConfigDump {
+		healthMux.HandleFunc("/configdump", handleConfigDump(snapshotter))
+	}
 
 	go func() {
 		if err = grpcServer.Serve(lis); err != nil {
@@ -97,3 +103,34 @@ func runEnvoyServer(envoyServer server.Server, address string, healthAddress str
 func health(w http.ResponseWriter, r *http.Request) {
 	w.WriteHeader(200)
 }
+
+type ConfigDumpError struct {
+	Error   error
+	Message string
+}
+
+func handleConfigDump(snapshotter *envoy.Snapshotter) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		if r.Method != http.MethodGet {
+			w.WriteHeader(http.StatusMethodNotAllowed)
+			return
+		}
+
+		snapshot, err := snapshotter.ConfigDump()
+		if err != nil {
+			respErr := ConfigDumpError{
+				Error:   err,
+				Message: "Unable to get current snapshot from snapshotter, see error for details.",
+			}
+
+			w.Header().Add("Content-Type", "application/json")
+			w.WriteHeader(http.StatusInternalServerError)
+			json.NewEncoder(w).Encode(respErr)
+			return
+		}
+
+		w.Header().Add("Content-Type", "application/json")
+		w.WriteHeader(http.StatusOK)
+		json.NewEncoder(w).Encode(snapshot)
+	}
+}

docs/ACCESSLOG.md

@@ -0,0 +1,37 @@
+# Access Log
+
+The Access log format is configurable via the Yggdrasil config file only. It is defined as a json object as follows:
+
+```json
+{
+    "accessLogger": {
+        "format": {
+            "bytes_received":            "%BYTES_RECEIVED%",
+            "bytes_sent":                "%BYTES_SENT%",
+            "downstream_local_address":  "%DOWNSTREAM_LOCAL_ADDRESS%",
+            "downstream_remote_address": "%DOWNSTREAM_REMOTE_ADDRESS%",
+            "duration":                  "%DURATION%",
+            "forwarded_for":             "%REQ(X-FORWARDED-FOR)%",
+            "protocol":                  "%PROTOCOL%",
+            "request_id":                "%REQ(X-REQUEST-ID)%",
+            "request_method":            "%REQ(:METHOD)%",
+            "request_path":              "%REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%",
+            "response_code":             "%RESPONSE_CODE%",
+            "response_flags":            "%RESPONSE_FLAGS%",
+            "start_time":                "%START_TIME(%s.%3f)%",
+            "upstream_cluster":          "%UPSTREAM_CLUSTER%",
+            "upstream_host":             "%UPSTREAM_HOST%",
+            "upstream_local_address":    "%UPSTREAM_LOCAL_ADDRESS%",
+            "upstream_service_time":     "%RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)%",
+            "user_agent":                "%REQ(USER-AGENT)%"
+        }
+    }
+}
+
+```
+
+The config above would be the same as the default access logger config shipped with Yggdasil. Thus if no format is provided this will be the format used.
+
+[See Envoy docs for more on access log formats](https://www.envoyproxy.io/docs/envoy/latest/configuration/observability/access_log/usage#config-access-log-default-format)
+
+The access log is written to `/var/log/envoy/access.log` which is not currently configurable.

docs/GETTINGSTARTED.md

@@ -138,7 +138,7 @@ By default, Yggdrasil will use an upstream ingress port of 443 (HTTPS), as we ar
 With the Yggdrasil container running, we can now configure an envoy node. Pull an envoy v1.10 docker image with the following command:
 
 ```console
-$ docker pull envoyproxy/envoy:v1.19-latest
+$ docker pull envoyproxy/envoy:v1.26-latest
 ```
 
 Next, we will need to setup a minimal config file to create the admin listener for envoy, as well as pointing to our dynamic configuration provider - Yggdrasil:
@@ -190,7 +190,7 @@ Where `<yggdrasil-container-ip-address>` is the IP address of the Yggdrasil dock
 Run the envoy docker container with the following command, making sure to mount the minimal config file that you've created:
 
 ```console
-$ docker run -e ENVOY_UID=0 -w /var/log/envoy/ -v /path/to/envoy.yaml:/etc/envoy/envoy.yaml -p 10000:10000 -d envoyproxy/envoy:v1.19-latest --service-node envoy-node --service-cluster envoy-node --config-path /etc/envoy/envoy.yaml
+$ docker run -e ENVOY_UID=0 -w /var/log/envoy/ -v /path/to/envoy.yaml:/etc/envoy/envoy.yaml -p 10000:10000 -d envoyproxy/envoy:v1.26-latest --service-node envoy-node --service-cluster envoy-node --config-path /etc/envoy/envoy.yaml
 ```
 
 The working directory for the container is set to `/var/log/envoy/` in order to create it at runtime, as Yggdrasil will configure envoy to write access logs to this directory.

go.mod

@@ -3,15 +3,15 @@ module github.com/uswitch/yggdrasil
 go 1.17
 
 require (
-	github.com/envoyproxy/go-control-plane v0.9.9
+	github.com/envoyproxy/go-control-plane v0.10.3
 	github.com/golang/protobuf v1.5.2
 	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0
-	github.com/prometheus/client_golang v0.9.0
-	github.com/sirupsen/logrus v1.1.1
+	github.com/prometheus/client_golang v1.11.1
+	github.com/sirupsen/logrus v1.6.0
 	github.com/spf13/cobra v0.0.3
 	github.com/spf13/viper v1.2.1
-	google.golang.org/grpc v1.36.1
-	google.golang.org/protobuf v1.27.1
+	google.golang.org/grpc v1.45.0
+	google.golang.org/protobuf v1.28.0
 	k8s.io/api v0.24.2
 	k8s.io/apimachinery v0.24.2
 	k8s.io/client-go v0.24.2
@@ -20,27 +20,28 @@ require (
 require (
 	github.com/PuerkitoBio/purell v1.1.1 // indirect
 	github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
-	github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/census-instrumentation/opencensus-proto v0.3.0 // indirect
-	github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158 // indirect
+	github.com/cespare/xxhash/v2 v2.1.1 // indirect
+	github.com/cncf/xds/go v0.0.0-20220314180256-7f1daf1720fc // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/emicklei/go-restful/v3 v3.8.0 // indirect
-	github.com/envoyproxy/protoc-gen-validate v0.5.1-0.20210316024357-9db8e779b461 // indirect
+	github.com/envoyproxy/protoc-gen-validate v0.6.7 // indirect
 	github.com/fsnotify/fsnotify v1.4.9 // indirect
 	github.com/go-logr/logr v1.2.0 // indirect
 	github.com/go-openapi/jsonpointer v0.19.5 // indirect
 	github.com/go-openapi/jsonreference v0.19.5 // indirect
 	github.com/go-openapi/swag v0.19.14 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/google/gnostic v0.5.7-v3refs // indirect
-	github.com/google/go-cmp v0.5.5 // indirect
+	github.com/google/go-cmp v0.5.7 // indirect
 	github.com/google/gofuzz v1.1.0 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/imdario/mergo v0.3.5 // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/konsorten/go-windows-terminal-sequences v1.0.1 // indirect
+	github.com/konsorten/go-windows-terminal-sequences v1.0.3 // indirect
 	github.com/magiconair/properties v1.8.0 // indirect
 	github.com/mailru/easyjson v0.7.6 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
@@ -50,21 +51,20 @@ require (
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/pelletier/go-toml v1.2.0 // indirect
 	github.com/prometheus/client_model v0.2.1-0.20200623203004-60555c9708c7 // indirect
-	github.com/prometheus/common v0.0.0-20170427095455-13ba4ddd0caa // indirect
-	github.com/prometheus/procfs v0.0.0-20170519190837-65c1f6f8f0fc // indirect
-	github.com/spf13/afero v1.3.4 // indirect
+	github.com/prometheus/common v0.26.0 // indirect
+	github.com/prometheus/procfs v0.6.0 // indirect
+	github.com/spf13/afero v1.6.0 // indirect
 	github.com/spf13/cast v1.3.0 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	golang.org/x/crypto v0.0.0-20220214200702-86341886e292 // indirect
-	golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect
+	golang.org/x/net v0.7.0 // indirect
 	golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect
-	golang.org/x/sys v0.0.0-20220209214540-3681064d5158 // indirect
-	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
-	golang.org/x/text v0.3.7 // indirect
+	golang.org/x/sys v0.5.0 // indirect
+	golang.org/x/term v0.5.0 // indirect
+	golang.org/x/text v0.7.0 // indirect
 	golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1 // indirect
+	google.golang.org/genproto v0.0.0-20220329172620-7be39ac1afc7 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect