From bfab99c47b6d28db001f67a500a2ce9d42ecdebf Mon Sep 17 00:00:00 2001 From: Nell Jerram Date: Fri, 11 Oct 2024 17:11:53 +0100 Subject: [PATCH] Add Gateway API and Envoy Gateway install to Calico Enterprise --- .gitignore | 2 + Makefile | 26 +++- .../installation/core_controller.go | 20 +++ .../installation/core_controller_test.go | 6 + pkg/envoygateway/resources.go | 129 ++++++++++++++++++ pkg/render/passthru.go | 4 +- 6 files changed, 183 insertions(+), 4 deletions(-) create mode 100644 pkg/envoygateway/resources.go diff --git a/.gitignore b/.gitignore index 6f127c452c..ad1de93ec9 100644 --- a/.gitignore +++ b/.gitignore @@ -97,3 +97,5 @@ ut/ # Tar files created as part of running FVs *.tar + +/pkg/envoygateway/resources.yaml diff --git a/Makefile b/Makefile index 56df8eab9f..422957fdb5 100644 --- a/Makefile +++ b/Makefile @@ -220,8 +220,28 @@ else GIT_VERSION?=$(shell git describe --tags --dirty --always --abbrev=12) endif +ENVOY_GATEWAY_HELM_CHART ?= oci://docker.io/envoyproxy/gateway-helm +ENVOY_GATEWAY_VERSION ?= v1.1.2 +ENVOY_GATEWAY_PREFIX ?= calico-gateway +ENVOY_GATEWAY_NAMESPACE ?= calico-gateway-system + +pkg/envoygateway/resources.yaml: hack/bin/helm-$(BUILDARCH) + hack/bin/helm-$(BUILDARCH) template $(ENVOY_GATEWAY_PREFIX) $(ENVOY_GATEWAY_HELM_CHART) \ + --version $(ENVOY_GATEWAY_VERSION) \ + -n $(ENVOY_GATEWAY_NAMESPACE) \ + --create-namespace \ + --include-crds \ + > $@ + +hack/bin/helm-$(BUILDARCH): + mkdir -p hack/bin + curl -sSf -L --retry 5 -o hack/bin/helm3.tar.gz https://get.helm.sh/helm-v3.11.3-linux-$(BUILDARCH).tar.gz + tar -zxvf hack/bin/helm3.tar.gz -C hack/bin linux-$(BUILDARCH)/helm + mv hack/bin/linux-$(BUILDARCH)/helm hack/bin/helm-$(BUILDARCH) + rmdir hack/bin/linux-$(BUILDARCH) + build: $(BINDIR)/operator-$(ARCH) -$(BINDIR)/operator-$(ARCH): $(SRC_FILES) +$(BINDIR)/operator-$(ARCH): $(SRC_FILES) pkg/envoygateway/resources.yaml mkdir -p $(BINDIR) $(CONTAINERIZED) -e CGO_ENABLED=$(CGO_ENABLED) -e GOEXPERIMENT=$(GOEXPERIMENT) $(CALICO_BUILD) \ sh -c '$(GIT_CONFIG_SSH) \ @@ -284,14 +304,14 @@ GINKGO_ARGS?= -v -trace -r GINKGO_FOCUS?=.* .PHONY: ut -ut: +ut: pkg/envoygateway/resources.yaml -mkdir -p .go-pkg-cache report $(CONTAINERIZED) $(CALICO_BUILD) sh -c '$(GIT_CONFIG_SSH) \ ginkgo -focus="$(GINKGO_FOCUS)" $(GINKGO_ARGS) "$(UT_DIR)"' ## Run the functional tests fv: cluster-create load-container-images run-fvs cluster-destroy -run-fvs: +run-fvs: pkg/envoygateway/resources.yaml -mkdir -p .go-pkg-cache report $(CONTAINERIZED) $(CALICO_BUILD) sh -c '$(GIT_CONFIG_SSH) \ ginkgo -focus="$(GINKGO_FOCUS)" $(GINKGO_ARGS) "$(FV_DIR)"' diff --git a/pkg/controller/installation/core_controller.go b/pkg/controller/installation/core_controller.go index 57a342cc85..6f1a2a40ba 100644 --- a/pkg/controller/installation/core_controller.go +++ b/pkg/controller/installation/core_controller.go @@ -27,6 +27,7 @@ import ( "strings" "github.com/elastic/cloud-on-k8s/v2/pkg/utils/stringsutil" + "github.com/tigera/operator/pkg/envoygateway" relasticsearch "github.com/tigera/operator/pkg/render/common/elasticsearch" "github.com/go-logr/logr" @@ -1400,6 +1401,13 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile ) } + installEnvoyGateway := (instance.Spec.Variant == v1.TigeraSecureEnterprise) + if installEnvoyGateway { + if err = r.installEnvoyGateway(ctx, reqLogger); err != nil { + return reconcile.Result{}, err + } + } + imageSet, err := imageset.GetImageSet(ctx, r.client, instance.Spec.Variant) if err != nil { r.status.SetDegraded(operator.ResourceReadError, "Error getting ImageSet", err, reqLogger) @@ -1890,6 +1898,18 @@ func (r *ReconcileInstallation) updateCRDs(ctx context.Context, variant operator return nil } +func (r *ReconcileInstallation) installEnvoyGateway(ctx context.Context, log logr.Logger) error { + egComponent := render.NewPassthrough(envoygateway.GetResources(log)...) + // Specify nil for the CR so no ownership is put on the gateway resources. We do this so + // removing the Installation CR will not remove them. + handler := r.newComponentHandler(log, r.client, r.scheme, nil) + if err := handler.CreateOrUpdateOrDelete(ctx, egComponent, nil); err != nil { + r.status.SetDegraded(operator.ResourceUpdateError, "Error creating / updating gateway resource", err, log) + return err + } + return nil +} + func getConfigMap(client client.Client, cmName string) (*corev1.ConfigMap, error) { cm := &corev1.ConfigMap{} cmNamespacedName := types.NamespacedName{ diff --git a/pkg/controller/installation/core_controller_test.go b/pkg/controller/installation/core_controller_test.go index e7b507e629..ac7cd69837 100644 --- a/pkg/controller/installation/core_controller_test.go +++ b/pkg/controller/installation/core_controller_test.go @@ -28,6 +28,7 @@ import ( "github.com/stretchr/testify/mock" appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" schedv1 "k8s.io/api/scheduling/v1" @@ -118,6 +119,7 @@ var _ = Describe("Testing core-controller installation", func() { scheme = runtime.NewScheme() Expect(apis.AddToScheme(scheme)).NotTo(HaveOccurred()) Expect(appsv1.SchemeBuilder.AddToScheme(scheme)).ShouldNot(HaveOccurred()) + Expect(batchv1.SchemeBuilder.AddToScheme(scheme)).ShouldNot(HaveOccurred()) Expect(rbacv1.SchemeBuilder.AddToScheme(scheme)).ShouldNot(HaveOccurred()) Expect(schedv1.SchemeBuilder.AddToScheme(scheme)).ShouldNot(HaveOccurred()) Expect(operator.SchemeBuilder.AddToScheme(scheme)).NotTo(HaveOccurred()) @@ -561,6 +563,7 @@ var _ = Describe("Testing core-controller installation", func() { scheme = runtime.NewScheme() Expect(apis.AddToScheme(scheme)).NotTo(HaveOccurred()) Expect(appsv1.SchemeBuilder.AddToScheme(scheme)).ShouldNot(HaveOccurred()) + Expect(batchv1.SchemeBuilder.AddToScheme(scheme)).ShouldNot(HaveOccurred()) Expect(rbacv1.SchemeBuilder.AddToScheme(scheme)).ShouldNot(HaveOccurred()) Expect(schedv1.SchemeBuilder.AddToScheme(scheme)).ShouldNot(HaveOccurred()) Expect(operator.SchemeBuilder.AddToScheme(scheme)).NotTo(HaveOccurred()) @@ -768,6 +771,7 @@ var _ = Describe("Testing core-controller installation", func() { scheme = runtime.NewScheme() Expect(apis.AddToScheme(scheme)).NotTo(HaveOccurred()) Expect(appsv1.SchemeBuilder.AddToScheme(scheme)).ShouldNot(HaveOccurred()) + Expect(batchv1.SchemeBuilder.AddToScheme(scheme)).ShouldNot(HaveOccurred()) Expect(rbacv1.SchemeBuilder.AddToScheme(scheme)).ShouldNot(HaveOccurred()) Expect(schedv1.SchemeBuilder.AddToScheme(scheme)).ShouldNot(HaveOccurred()) Expect(operator.SchemeBuilder.AddToScheme(scheme)).NotTo(HaveOccurred()) @@ -1567,6 +1571,7 @@ var _ = Describe("Testing core-controller installation", func() { scheme = runtime.NewScheme() Expect(apis.AddToScheme(scheme)).NotTo(HaveOccurred()) Expect(appsv1.SchemeBuilder.AddToScheme(scheme)).ShouldNot(HaveOccurred()) + Expect(batchv1.SchemeBuilder.AddToScheme(scheme)).ShouldNot(HaveOccurred()) Expect(rbacv1.SchemeBuilder.AddToScheme(scheme)).ShouldNot(HaveOccurred()) Expect(schedv1.SchemeBuilder.AddToScheme(scheme)).ShouldNot(HaveOccurred()) Expect(operator.SchemeBuilder.AddToScheme(scheme)).NotTo(HaveOccurred()) @@ -1704,6 +1709,7 @@ var _ = Describe("Testing core-controller installation", func() { scheme = runtime.NewScheme() Expect(apis.AddToScheme(scheme)).NotTo(HaveOccurred()) Expect(appsv1.SchemeBuilder.AddToScheme(scheme)).ShouldNot(HaveOccurred()) + Expect(batchv1.SchemeBuilder.AddToScheme(scheme)).ShouldNot(HaveOccurred()) Expect(rbacv1.SchemeBuilder.AddToScheme(scheme)).ShouldNot(HaveOccurred()) Expect(schedv1.SchemeBuilder.AddToScheme(scheme)).ShouldNot(HaveOccurred()) Expect(operator.SchemeBuilder.AddToScheme(scheme)).NotTo(HaveOccurred()) diff --git a/pkg/envoygateway/resources.go b/pkg/envoygateway/resources.go new file mode 100644 index 0000000000..1b5a595592 --- /dev/null +++ b/pkg/envoygateway/resources.go @@ -0,0 +1,129 @@ +// Copyright (c) 2024 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package envoygateway + +import ( + _ "embed" + "fmt" + "strings" + "sync" + + "github.com/go-logr/logr" + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" + v1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + apiextenv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/yaml" // gopkg.in/yaml.v2 didn't parse all the fields but this package did +) + +var ( + //go:embed resources.yaml + resources string + + yamlDelimiter = "\n---\n" + lock sync.Mutex + cachedObjects []client.Object +) + +type yamlKind struct { + APIVersion string `yaml:"apiVersion"` + Kind string `yaml:"kind"` +} + +func GetResources(log logr.Logger) []client.Object { + lock.Lock() + defer lock.Unlock() + + if len(cachedObjects) == 0 { + for _, yml := range strings.Split(resources, yamlDelimiter) { + var yamlKind yamlKind + if err := yaml.Unmarshal([]byte(yml), &yamlKind); err != nil { + panic(fmt.Sprintf("unable to unmarshal YAML: %v:\n%v\n", err, yml)) + } + kindStr := yamlKind.APIVersion + "/" + yamlKind.Kind + log.Info(kindStr) + switch kindStr { + case "apiextensions.k8s.io/v1/CustomResourceDefinition": + obj := &apiextenv1.CustomResourceDefinition{} + if err := yaml.Unmarshal([]byte(yml), obj); err != nil { + panic(fmt.Sprintf("unable to unmarshal %v: %v", kindStr, err)) + } + cachedObjects = append(cachedObjects, obj) + case "apps/v1/Deployment": + obj := &appsv1.Deployment{} + if err := yaml.Unmarshal([]byte(yml), obj); err != nil { + panic(fmt.Sprintf("unable to unmarshal %v: %v", kindStr, err)) + } + cachedObjects = append(cachedObjects, obj) + case "batch/v1/Job": + obj := &batchv1.Job{} + if err := yaml.Unmarshal([]byte(yml), obj); err != nil { + panic(fmt.Sprintf("unable to unmarshal %v: %v", kindStr, err)) + } + cachedObjects = append(cachedObjects, obj) + case "rbac.authorization.k8s.io/v1/ClusterRole": + obj := &rbacv1.ClusterRole{} + if err := yaml.Unmarshal([]byte(yml), obj); err != nil { + panic(fmt.Sprintf("unable to unmarshal %v: %v", kindStr, err)) + } + cachedObjects = append(cachedObjects, obj) + case "rbac.authorization.k8s.io/v1/ClusterRoleBinding": + obj := &rbacv1.ClusterRoleBinding{} + if err := yaml.Unmarshal([]byte(yml), obj); err != nil { + panic(fmt.Sprintf("unable to unmarshal %v: %v", kindStr, err)) + } + cachedObjects = append(cachedObjects, obj) + case "rbac.authorization.k8s.io/v1/Role": + obj := &rbacv1.Role{} + if err := yaml.Unmarshal([]byte(yml), obj); err != nil { + panic(fmt.Sprintf("unable to unmarshal %v: %v", kindStr, err)) + } + cachedObjects = append(cachedObjects, obj) + case "rbac.authorization.k8s.io/v1/RoleBinding": + obj := &rbacv1.RoleBinding{} + if err := yaml.Unmarshal([]byte(yml), obj); err != nil { + panic(fmt.Sprintf("unable to unmarshal %v: %v", kindStr, err)) + } + cachedObjects = append(cachedObjects, obj) + case "v1/ConfigMap": + obj := &v1.ConfigMap{} + if err := yaml.Unmarshal([]byte(yml), obj); err != nil { + panic(fmt.Sprintf("unable to unmarshal %v: %v", kindStr, err)) + } + cachedObjects = append(cachedObjects, obj) + case "v1/Service": + obj := &v1.Service{} + if err := yaml.Unmarshal([]byte(yml), obj); err != nil { + panic(fmt.Sprintf("unable to unmarshal %v: %v", kindStr, err)) + } + cachedObjects = append(cachedObjects, obj) + case "v1/ServiceAccount": + obj := &v1.ServiceAccount{} + if err := yaml.Unmarshal([]byte(yml), obj); err != nil { + panic(fmt.Sprintf("unable to unmarshal %v: %v", kindStr, err)) + } + cachedObjects = append(cachedObjects, obj) + case "/": + // No-op. + default: + panic(fmt.Sprintf("unhandled type %v", kindStr)) + } + } + } + + return cachedObjects +} diff --git a/pkg/render/passthru.go b/pkg/render/passthru.go index 830f064fe9..020c33b1b5 100644 --- a/pkg/render/passthru.go +++ b/pkg/render/passthru.go @@ -15,6 +15,8 @@ package render import ( + "reflect" + "github.com/go-logr/logr" operatorv1 "github.com/tigera/operator/api/v1" rmeta "github.com/tigera/operator/pkg/render/common/meta" @@ -59,7 +61,7 @@ func (p *passthroughComponent) Objects() (objsToCreate []client.Object, objsToDe if o == nil { continue } - p.log.V(1).Info("PassThrough processing object", "obj", o) + p.log.V(1).Info("PassThrough processing object", "type", reflect.TypeOf(o), "name", o.GetName(), "namespace", o.GetNamespace()) objs = append(objs, o) } if p.isDelete {