From e92ea860d27cb24faf038980337f7fbbd03fca38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patryk=20Ma=C5=82ek?= Date: Thu, 16 Jan 2025 18:16:50 +0100 Subject: [PATCH] tests: add ValidatingAdmissionPolicy tests --- .../dataplane_validation_policies.yaml | 44 +-- ...ataplane_validatingadmissionpolicy_test.go | 261 ++++++++++++++++++ test/helpers/kustomize/apply.go | 153 ++++++++++ 3 files changed, 440 insertions(+), 18 deletions(-) create mode 100644 test/crdsvalidation/dataplane_validatingadmissionpolicy_test.go create mode 100644 test/helpers/kustomize/apply.go diff --git a/config/default/validation_policies/dataplane_validation_policies.yaml b/config/default/validation_policies/dataplane_validation_policies.yaml index 61a78603e..a5d0a69bb 100644 --- a/config/default/validation_policies/dataplane_validation_policies.yaml +++ b/config/default/validation_policies/dataplane_validation_policies.yaml @@ -16,8 +16,12 @@ spec: resources: - "dataplanes" variables: + - name: network + expression: object.spec.network + - name: services + expression: variables.network.services - name: ingressPorts - expression: object.spec.network.services.ingress.ports + expression: variables.services.ingress.ports - name: podTemplateSpec expression: object.spec.deployment.podTemplateSpec - name: proxyContainer @@ -33,33 +37,37 @@ spec: variables.proxyContainer.env.filter(e, e.name == "KONG_PROXY_LISTEN") - name: envPortMaps expression: | - variables.envFilteredPortMaps.size() > 0 ? variables.envFilteredPortMaps[0] : null + variables.envFilteredPortMaps.size() > 0 ? variables.envFilteredPortMaps[0].value : null - name: envProxyListen expression: | - variables.envFilteredProxyListen.size() > 0 ? variables.envFilteredProxyListen[0] : null + variables.envFilteredProxyListen.size() > 0 ? variables.envFilteredProxyListen[0].value : null # Using string functions from: https://pkg.go.dev/github.com/google/cel-go/ext validations: - messageExpression: "'Each port from spec.network.services.ingress.ports has to have an accompanying port in KONG_PORT_MAPS env'" expression: | - variables.ingressPorts == null || - variables.envPortMaps == null || - variables.ingressPorts.all(p, variables.envPortMaps.value. - split(","). - exists(pm, - pm.split(":")[1].trim() == string(p.targetPort) - ) - ) + !has(object.spec.network) || + !has(object.spec.network.services) || + variables.ingressPorts == null || + variables.envPortMaps == null || + variables.ingressPorts.all(p, variables.envPortMaps. + split(","). + exists(pm, + pm.split(":")[1].trim() == string(p.targetPort) + ) + ) reason: Invalid - messageExpression: "'Each port from spec.network.services.ingress.ports has to have an accompanying port in KONG_PROXY_LISTEN env'" expression: | - variables.ingressPorts == null || - variables.envProxyListen == null || - variables.ingressPorts.all(p, variables.envProxyListen.value. - split(","). - exists(pm, - pm.trim().split(" ")[0].split(":")[1].trim() == string(p.targetPort) - ) + !has(object.spec.network) || + !has(object.spec.network.services) || + variables.ingressPorts == null || + variables.envProxyListen == null || + variables.ingressPorts.all(p, variables.envProxyListen. + split(","). + exists(pm, + pm.trim().split(" ")[0].split(":")[1].trim() == string(p.targetPort) ) + ) reason: Invalid --- apiVersion: admissionregistration.k8s.io/v1 diff --git a/test/crdsvalidation/dataplane_validatingadmissionpolicy_test.go b/test/crdsvalidation/dataplane_validatingadmissionpolicy_test.go new file mode 100644 index 000000000..35422c62c --- /dev/null +++ b/test/crdsvalidation/dataplane_validatingadmissionpolicy_test.go @@ -0,0 +1,261 @@ +package crdsvalidation + +import ( + "context" + "testing" + + "github.com/samber/lo" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + + operatorv1beta1 "github.com/kong/gateway-operator/api/v1beta1" + "github.com/kong/gateway-operator/modules/manager/scheme" + "github.com/kong/gateway-operator/test/envtest" + "github.com/kong/gateway-operator/test/helpers/kustomize" + + kcfgcrdsvalidation "github.com/kong/kubernetes-configuration/test/crdsvalidation" +) + +const ( + // KustomizePathValidationPolicies is the path to the Kustomize directory containing the validation policies. + KustomizePathValidationPolicies = "config/default/validation_policies/" +) + +func TestDataPlaneValidatingAdmissionPolicy(t *testing.T) { + t.Parallel() + ctx := context.Background() + scheme := scheme.Get() + cfg, ns := envtest.Setup(t, ctx, scheme) + kustomize.Apply(ctx, t, cfg, KustomizePathValidationPolicies) + + t.Run("ports", func(t *testing.T) { + kcfgcrdsvalidation.TestCasesGroup[*operatorv1beta1.DataPlane]{ + { + Name: "not providing spec fails", + TestObject: &operatorv1beta1.DataPlane{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "dp-", + Namespace: ns.Name, + }, + Spec: operatorv1beta1.DataPlaneSpec{}, + }, + ExpectedErrorMessage: lo.ToPtr("DataPlane requires an image to be set on proxy container"), + }, + { + Name: "providing correct ingress service ports and KONG_PORT_MAPS env succeeds", + TestObject: &operatorv1beta1.DataPlane{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "dp-", + Namespace: ns.Name, + }, + Spec: operatorv1beta1.DataPlaneSpec{ + DataPlaneOptions: operatorv1beta1.DataPlaneOptions{ + Deployment: operatorv1beta1.DataPlaneDeploymentOptions{ + DeploymentOptions: operatorv1beta1.DeploymentOptions{ + PodTemplateSpec: &corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "proxy", + Image: "kong:3.9", + Env: []corev1.EnvVar{ + { + Name: "KONG_PROXY_LISTEN", + Value: "0.0.0.0:8000 reuseport backlog=16384, 0.0.0.0:8443 http2 ssl reuseport backlog=16384", + }, + { + Name: "KONG_PORT_MAPS", + Value: "80:8000,443:8443", + }, + }, + }, + }, + }, + }, + }, + }, + Network: operatorv1beta1.DataPlaneNetworkOptions{ + Services: &operatorv1beta1.DataPlaneServices{ + Ingress: &operatorv1beta1.DataPlaneServiceOptions{ + Ports: []operatorv1beta1.DataPlaneServicePort{ + { + Name: "http", + Port: 80, + TargetPort: intstr.FromInt(8000), + }, + { + Name: "http", + Port: 443, + TargetPort: intstr.FromInt(8443), + }, + }, + }, + }, + }, + }, + }, + }, + }, + { + Name: "providing incorrect ingress service ports and KONG_PORT_MAPS env fails", + TestObject: &operatorv1beta1.DataPlane{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "dp-", + Namespace: ns.Name, + }, + Spec: operatorv1beta1.DataPlaneSpec{ + DataPlaneOptions: operatorv1beta1.DataPlaneOptions{ + Deployment: operatorv1beta1.DataPlaneDeploymentOptions{ + DeploymentOptions: operatorv1beta1.DeploymentOptions{ + PodTemplateSpec: &corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "proxy", + Image: "kong:3.9", + Env: []corev1.EnvVar{ + { + Name: "KONG_PROXY_LISTEN", + Value: "0.0.0.0:8000 reuseport backlog=16384, 0.0.0.0:8443 http2 ssl reuseport backlog=16384", + }, + { + Name: "KONG_PORT_MAPS", + Value: "80:8000,443:8443", + }, + }, + }, + }, + }, + }, + }, + }, + Network: operatorv1beta1.DataPlaneNetworkOptions{ + Services: &operatorv1beta1.DataPlaneServices{ + Ingress: &operatorv1beta1.DataPlaneServiceOptions{ + Ports: []operatorv1beta1.DataPlaneServicePort{ + { + Name: "http", + Port: 80, + // No matching port in KONG_PORT_MAPS + TargetPort: intstr.FromInt(8001), + }, + { + Name: "http", + Port: 443, + TargetPort: intstr.FromInt(8443), + }, + }, + }, + }, + }, + }, + }, + }, + ExpectedErrorMessage: lo.ToPtr("is forbidden: ValidatingAdmissionPolicy 'ports.dataplane.gateway-operator.konghq.com' with binding 'binding-ports.dataplane.gateway-operator.konghq.com' denied request: Each port from spec.network.services.ingress.ports has to have an accompanying port in KONG_PORT_MAPS env"), + }, + { + Name: "providing correct ingress service ports and KONG_PROXY_LISTEN env succeeds", + TestObject: &operatorv1beta1.DataPlane{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "dp-", + Namespace: ns.Name, + }, + Spec: operatorv1beta1.DataPlaneSpec{ + DataPlaneOptions: operatorv1beta1.DataPlaneOptions{ + Deployment: operatorv1beta1.DataPlaneDeploymentOptions{ + DeploymentOptions: operatorv1beta1.DeploymentOptions{ + PodTemplateSpec: &corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "proxy", + Image: "kong:3.9", + Env: []corev1.EnvVar{ + { + Name: "KONG_PROXY_LISTEN", + Value: "0.0.0.0:8000 reuseport backlog=16384, 0.0.0.0:8443 http2 ssl reuseport backlog=16384", + }, + { + Name: "KONG_PORT_MAPS", + Value: "80:8000,443:8443", + }, + }, + }, + }, + }, + }, + }, + }, + Network: operatorv1beta1.DataPlaneNetworkOptions{ + Services: &operatorv1beta1.DataPlaneServices{ + Ingress: &operatorv1beta1.DataPlaneServiceOptions{ + Ports: []operatorv1beta1.DataPlaneServicePort{ + { + Name: "http", + Port: 80, + TargetPort: intstr.FromInt(8000), + }, + }, + }, + }, + }, + }, + }, + }, + }, + { + Name: "providing incorrect ingress service ports and KONG_PROXY_LISTEN env fails", + TestObject: &operatorv1beta1.DataPlane{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "dp-", + Namespace: ns.Name, + }, + Spec: operatorv1beta1.DataPlaneSpec{ + DataPlaneOptions: operatorv1beta1.DataPlaneOptions{ + Deployment: operatorv1beta1.DataPlaneDeploymentOptions{ + DeploymentOptions: operatorv1beta1.DeploymentOptions{ + PodTemplateSpec: &corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "proxy", + Image: "kong:3.9", + Env: []corev1.EnvVar{ + { + Name: "KONG_PROXY_LISTEN", + Value: "0.0.0.0:8000 reuseport backlog=16384, 0.0.0.0:8443 http2 ssl reuseport backlog=16384", + }, + { + Name: "KONG_PORT_MAPS", + Value: "80:8000,443:8443", + }, + }, + }, + }, + }, + }, + }, + }, + Network: operatorv1beta1.DataPlaneNetworkOptions{ + Services: &operatorv1beta1.DataPlaneServices{ + Ingress: &operatorv1beta1.DataPlaneServiceOptions{ + Ports: []operatorv1beta1.DataPlaneServicePort{ + { + Name: "http", + Port: 80, + // No matching port in KONG_PROXY_LISTEN + TargetPort: intstr.FromInt(8001), + }, + }, + }, + }, + }, + }, + }, + }, + ExpectedErrorMessage: lo.ToPtr("is forbidden: ValidatingAdmissionPolicy 'ports.dataplane.gateway-operator.konghq.com' with binding 'binding-ports.dataplane.gateway-operator.konghq.com' denied request: Each port from spec.network.services.ingress.ports has to have an accompanying port in KONG_PORT_MAPS env"), + }, + }.RunWithConfig(t, cfg, scheme) + }) +} diff --git a/test/helpers/kustomize/apply.go b/test/helpers/kustomize/apply.go new file mode 100644 index 000000000..a694e4b0d --- /dev/null +++ b/test/helpers/kustomize/apply.go @@ -0,0 +1,153 @@ +package kustomize + +import ( + "bufio" + "bytes" + "context" + "errors" + "fmt" + "io" + "path" + "testing" + + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/serializer/yaml" + "k8s.io/apimachinery/pkg/types" + utilyaml "k8s.io/apimachinery/pkg/util/yaml" + "k8s.io/client-go/discovery" + memory "k8s.io/client-go/discovery/cached" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/rest" + "k8s.io/client-go/restmapper" + "sigs.k8s.io/kustomize/api/krusty" + "sigs.k8s.io/kustomize/kyaml/filesys" + + "github.com/kong/gateway-operator/pkg/utils/test" +) + +var decUnstructured = yaml.NewDecodingSerializer(unstructured.UnstructuredJSONScheme) + +// Apply applies a kustomization to the cluster using the given rest.Config. +func Apply(ctx context.Context, t *testing.T, cfg *rest.Config, dir string) { + t.Helper() + + k := krusty.MakeKustomizer(krusty.MakeDefaultOptions()) + fSys := filesys.MakeFsOnDisk() + resmap, err := k.Run(fSys, path.Join(test.ProjectRootPath(), dir)) + require.NoError(t, err) + + b, err := resmap.AsYaml() + require.NoError(t, err) + + res, err := apply(ctx, cfg, b) + require.NoError(t, err) + for _, r := range res { + t.Logf("Result: %s", r) + } +} + +func apply(ctx context.Context, restConfig *rest.Config, data []byte) (result []string, err error) { + chanMes, chanErr := readYaml(data) + for { + select { + case dataBytes, ok := <-chanMes: + { + if !ok { + return result, err + } + + // Get obj and dr + obj, dr, errClient := buildDynamicResourceClient(restConfig, dataBytes) + if errClient != nil { + err = errors.Join(errClient, err) + continue + } + + // Create or Update + _, errPatch := dr.Patch(ctx, obj.GetName(), types.ApplyPatchType, dataBytes, metav1.PatchOptions{ + FieldManager: "test", + }) + if errPatch != nil { + err = errors.Join(errPatch, err) + } else { + result = append(result, obj.GetName()+" applied.") + } + } + case errChan, ok := <-chanErr: + if !ok { + return result, err + } + if errChan == nil { + continue + } + err = errors.Join(errChan, err) + } + } +} + +func readYaml(data []byte) (<-chan []byte, <-chan error) { + var ( + chanErr = make(chan error) + chanBytes = make(chan []byte) + multidocReader = utilyaml.NewYAMLReader(bufio.NewReader(bytes.NewReader(data))) + ) + + go func() { + defer close(chanErr) + defer close(chanBytes) + + for { + buf, err := multidocReader.Read() + if err != nil { + if errors.Is(err, io.EOF) { + return + } + chanErr <- fmt.Errorf("failed to read yaml data : %w", err) + return + } + chanBytes <- buf + } + }() + return chanBytes, chanErr +} + +func buildDynamicResourceClient(restConfig *rest.Config, data []byte) (obj *unstructured.Unstructured, dr dynamic.ResourceInterface, err error) { + // Decode YAML manifest into unstructured.Unstructured + obj = &unstructured.Unstructured{} + _, gvk, err := decUnstructured.Decode(data, nil, obj) + if err != nil { + return obj, dr, fmt.Errorf("Decode yaml failed. : %w", err) + } + + dc, err := discovery.NewDiscoveryClientForConfig(restConfig) + if err != nil { + return nil, nil, fmt.Errorf("new dc failed : %w", err) + } + + mapper := restmapper.NewDeferredDiscoveryRESTMapper(memory.NewMemCacheClient(dc)) + + // Find GVR + mapping, err := mapper.RESTMapping(gvk.GroupKind(), gvk.Version) + if err != nil { + return obj, dr, fmt.Errorf("Mapping kind with version failed : %w", err) + } + + // Prepare dynamic client + dynamicClient, err := dynamic.NewForConfig(restConfig) + if err != nil { + return obj, dr, fmt.Errorf("Prepare dynamic client failed. : %w", err) + } + + // Obtain REST interface for the GVR + if mapping.Scope.Name() == meta.RESTScopeNameNamespace { + // namespaced resources should specify the namespace + dr = dynamicClient.Resource(mapping.Resource).Namespace(obj.GetNamespace()) + } else { + // for cluster-wide resources + dr = dynamicClient.Resource(mapping.Resource) + } + return obj, dr, nil +}