E2E Test Refactor POC

bin/start

@@ -55,6 +55,7 @@ while true ; do
     --oldest ) SUMMON_ENV=oldest ; shift ;;
     --current ) SUMMON_ENV=current ; shift ;;
     --next ) SUMMON_ENV=next ; shift ;;
+    --template=* ) TEMPLATE_OVERRIDE="${1#*=}" ; export TEMPLATE_OVERRIDE; shift ;;
     -h | --help ) print_help ; shift ;;
      * ) if [ -z "$1" ]; then break; else echo "$1 is not a valid option"; exit 1; fi;;
   esac
@@ -69,10 +70,17 @@ export RUN_IN_DOCKER
 export CONJUR_DEPLOYMENT
 export DEV
 export SUMMON_ENV
+export TEST
+export TEMPLATE_OVERRIDE
+
+echo $TEMPLATE_OVERRIDE
 
 # summon environment variable
 export CONJUR_MAJOR_VERSION=5
 
+# make sure we are in the project root
+cd $(git rev-parse --show-toplevel)
+
 source bootstrap.env
 
 pushd deploy

bootstrap.env

@@ -14,16 +14,16 @@ export APP_NAMESPACE_NAME=app-$UNIQUE_TEST_ID
 #######
 # Local DEV Env (uncomment all lines if using this configuration)
 #######
-# export PLATFORM=kubernetes
-# export TEST_PLATFORM=gke
-# export AUTHENTICATOR_ID=authn-dev-env
-# export APP_NAMESPACE_NAME=local-secrets-provider
-# export CONJUR_NAMESPACE_NAME=local-conjur
-# export STOP_RUNNING_ENV=true
-# export CONJUR_ACCOUNT=cucumber
-# export CONJUR_LOG_LEVEL=debug
-# export CONJUR_AUTHENTICATORS=authn-k8s/${AUTHENTICATOR_ID}
-# export DEV=true
-# export SECRETS_MODE=k8s # Supported: [k8s, k8s-rotation, p2f, p2f-rotation]
+export PLATFORM=kubernetes
+export TEST_PLATFORM=gke
+export AUTHENTICATOR_ID=authn-dev-env
+export APP_NAMESPACE_NAME=local-secrets-provider
+export CONJUR_NAMESPACE_NAME=local-conjur
+export STOP_RUNNING_ENV=true
+export CONJUR_ACCOUNT=cucumber
+export CONJUR_LOG_LEVEL=debug
+export CONJUR_AUTHENTICATORS=authn-k8s/${AUTHENTICATOR_ID}
+export DEV=true
+export SECRETS_MODE=k8s # Supported: [k8s, k8s-rotation, p2f, p2f-rotation]
 # Uncomment to deploy the Secrets Provider using HELM
 # export DEV_HELM=true

deploy/dev/config/k8s/secrets-provider-init-container.sh.yml

@@ -22,7 +22,7 @@ spec:
       serviceAccountName: ${APP_NAMESPACE_NAME}-sa
       containers:
       - image: debian
-        name: init-env-app
+        name: test-app
         command: ["sleep"]
         args: ["infinity"]
         env:

deploy/policy/load_policies.sh

@@ -33,7 +33,7 @@ done
 conjur variable set -i secrets/test_secret -v "some-secret"
 conjur variable set -i "secrets/var with spaces" -v "some-secret"
 conjur variable set -i "secrets/var+with+pluses" -v "some-secret"
-conjur variable set -i "secrets/umlaut" -v "some-secret"
+conjur variable set -i "secrets/umlaut" -v "ÄäÖöÜü"
 conjur variable set -i "secrets/encoded" -v "$(echo "secret-value" | tr -d '\n' | base64)" # == "c2VjcmV0LXZhbHVl"
 conjur variable set -i secrets/url -v "postgresql://test-app-backend.app-test.svc.cluster.local:5432"
 conjur variable set -i secrets/username -v "some-user"

deploy/utils.sh

@@ -590,6 +590,7 @@ wait_for_job() {
 
 deploy_env() {
   export SECRETS_MODE=${SECRETS_MODE:-"k8s"}
+  export TEMPLATE_OVERRIDE=${TEMPLATE_OVERRIDE:-""}
   local yaml_template_name="test-env"
 
   case $SECRETS_MODE in
@@ -627,6 +628,13 @@ deploy_env() {
       ;;
   esac
 
+  echo $TEMPLATE_OVERRIDE
+
+  if [ ! -z "$TEMPLATE_OVERRIDE" ]
+  then
+    yaml_template_name="$TEMPLATE_OVERRIDE"
+  fi
+
   generate_manifest_and_deploy $yaml_template_name
 }
 

e2e/consts.go

@@ -0,0 +1,39 @@
+package e2e
+
+const (
+	SecretsProviderImage = "secrets-provider-for-k8s"
+	TestImage            = "debian"
+	TestContainer        = "test-app"
+	CLIContainer         = "conjur-cli"
+
+	SecretsProviderPodLabels      = "app=test-env"
+	SecretsProviderDeploymentName = "test-env"
+	SecretsProviderNamespace      = "local-secrets-provider"
+
+	ConjurPodLabels          = "app=conjur-node"
+	ConjurServiceName        = "conjur-node"
+	ConjurDeploymentName     = "conjur-node"
+	ConjurNamespace          = "local-conjur"
+	ConjurServiceAccountName = "conjur-cluster"
+
+	ConjurOSSImage          = "cyberark/conjur"
+	ConjurOSSImageDest      = "cyberark/conjur:local-conjur"
+	ConjurOSSDeploymentName = "conjur-oss"
+	ConjurOSSPodLabels      = "app=conjur-oss"
+
+	PostgresImage     = "postgres:10"
+	PostgresImageDest = "postgres:local-conjur"
+
+	NginxImageDest = "nginx:local-conjur"
+
+	ConjurCLIPodLabels      = "app=conjur-cli"
+	ConjurCLIDeploymentName = "conjur-cli"
+
+	WaitForMaterializedViewRefreshSecond = 2
+
+	// Available templates:
+	K8sTemplate         = "secrets-provider-init-container"
+	K8sRotationTemplate = "secrets-provider-k8s-rotation"
+	P2fTemplate         = "secrets-provider-init-push-to-file"
+	P2fRotationTemplate = "secrets-provider-p2f-rotation"
+)

e2e/k8s_test.go

@@ -0,0 +1,91 @@
+package e2e
+
+import (
+	"bytes"
+	"context"
+	"encoding/base64"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"sigs.k8s.io/e2e-framework/pkg/envconf"
+	"sigs.k8s.io/e2e-framework/pkg/features"
+)
+
+func TestSecretsProvidedK8s(t *testing.T) {
+	f := features.New("K8s secrets provided").
+		Assess("test secret set in pod", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
+			var stdout, stderr bytes.Buffer
+			command := []string{"printenv", "|", "grep", "TEST_SECRET"}
+
+			RunCommandInSecretsProviderPod(cfg.Client(), command, &stdout, &stderr)
+
+			assert.Contains(t, stdout.String(), "supersecret")
+
+			return ctx
+		}).
+		Assess("umlaut secret set in pod", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
+			var stdout, stderr bytes.Buffer
+			command := []string{"printenv", "|", "grep", "VARIABLE_WITH_UMLAUT_SECRET"}
+
+			RunCommandInSecretsProviderPod(cfg.Client(), command, &stdout, &stderr)
+
+			assert.Contains(t, stdout.String(), "ÄäÖöÜü")
+
+			return ctx
+		})
+
+	testenv.Test(t, f.Feature())
+}
+
+func TestSecretsProvidedP2F(t *testing.T) {
+	f := features.New("P2F secrets provided").
+		Setup(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
+			ReloadWithTemplate(cfg.Client(), P2fTemplate)
+			return ctx
+		}).
+		Assess("secrets stored in file", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
+			var stdout, stderr bytes.Buffer
+			command := []string{"cat", "opt/secrets/conjur/group1.yaml"}
+
+			RunCommandInSecretsProviderPod(cfg.Client(), command, &stdout, &stderr)
+
+			yamlContent := "\"url\": \"postgresql://test-app-backend.app-test.svc.cluster.local:5432\"\n\"username\": \"some-user\"\n\"password\": \"7H1SiSmYp@5Sw0rd\"\n\"encoded\": \"secret-value\""
+
+			assert.Contains(t, stdout.String(), yamlContent)
+
+			return ctx
+		})
+
+	testenv.Test(t, f.Feature())
+}
+
+func TestSecretsProvidedP2FRotation(t *testing.T) {
+	f := features.New("P2F secrets provided and rotated").
+		Setup(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
+			ReloadWithTemplate(cfg.Client(), P2fRotationTemplate)
+			return ctx
+		}).
+		Assess("secrets rotated in file", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
+
+			var stdout, stderr bytes.Buffer
+			command := []string{"cat", "opt/secrets/conjur/group1.yaml"}
+
+			RunCommandInSecretsProviderPod(cfg.Client(), command, &stdout, &stderr)
+
+			yamlContent := "\"url\": \"postgresql://test-app-backend.app-test.svc.cluster.local:5432\"\n\"username\": \"some-user\"\n\"password\": \"7H1SiSmYp@5Sw0rd\"\n\"test\": \"supersecret\"\n\"encoded\": \"secret-value\""
+
+			assert.Contains(t, stdout.String(), yamlContent)
+
+			SetConjurSecret(cfg.Client(), "secrets/encoded", base64.StdEncoding.EncodeToString([]byte("rotated-secret-value")))
+			time.Sleep(15 * time.Second)
+			RunCommandInSecretsProviderPod(cfg.Client(), command, &stdout, &stderr)
+
+			yamlContent = "\"url\": \"postgresql://test-app-backend.app-test.svc.cluster.local:5432\"\n\"username\": \"some-user\"\n\"password\": \"7H1SiSmYp@5Sw0rd\"\n\"test\": \"supersecret\"\n\"encoded\": \"rotated-secret-value\""
+			assert.Contains(t, stdout.String(), yamlContent)
+
+			return ctx
+		})
+
+	testenv.Test(t, f.Feature())
+}

e2e/main_test.go

@@ -0,0 +1,96 @@
+package e2e
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/exec"
+	"testing"
+	"time"
+
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/e2e-framework/klient"
+	"sigs.k8s.io/e2e-framework/klient/conf"
+	"sigs.k8s.io/e2e-framework/klient/k8s"
+	"sigs.k8s.io/e2e-framework/klient/wait"
+	"sigs.k8s.io/e2e-framework/klient/wait/conditions"
+	"sigs.k8s.io/e2e-framework/pkg/env"
+	"sigs.k8s.io/e2e-framework/pkg/envconf"
+	"sigs.k8s.io/e2e-framework/pkg/envfuncs"
+)
+
+var (
+	testenv        env.Environment
+	KubeconfigFile string
+	k8sClient      klient.Client
+)
+
+func TestMain(m *testing.M) {
+	testenv = env.New()
+	path := conf.ResolveKubeConfigFile()
+	cfg := envconf.NewWithKubeConfig(path)
+	testenv = env.NewWithConfig(cfg)
+
+	testenv.Setup(
+		func(ctx context.Context, cfg *envconf.Config) (context.Context, error) {
+			fmt.Println("Setup")
+			k8sClient = cfg.Client()
+
+			cmd1 := exec.Command("../bin/build")
+			out, err := cmd1.CombinedOutput()
+			if err != nil {
+				fmt.Printf("Failed to execute command. %v, %s", err, out)
+			}
+
+			cmd2 := exec.Command("../bin/start", "--dev")
+			out, err = cmd2.CombinedOutput()
+			if err != nil {
+				fmt.Printf("Failed to execute command. %v, %s", err, out)
+			}
+
+			// Get the Secrets Provider Pod
+			var pods v1.PodList
+			err = k8sClient.Resources("local-secrets-provider").List(context.TODO(), &pods)
+			if err != nil {
+				fmt.Print(err)
+			}
+			fmt.Printf("found %d pods in namespace local-secrets-provider", len(pods.Items))
+			if len(pods.Items) == 0 {
+				fmt.Printf("no pods in namespace local-secrets-provider")
+			}
+
+			// Wait for the Secrets Provider Pod to be ready
+			pod := v1.Pod{ObjectMeta: metav1.ObjectMeta{Name: pods.Items[0].Name}}
+			err = wait.For(conditions.New(k8sClient.Resources(SecretsProviderNamespace)).PodReady(k8s.Object(&pod)), wait.WithTimeout(time.Minute*1))
+			if err != nil {
+				fmt.Print(err)
+			}
+
+			// Setup complete
+			fmt.Println("Setup done")
+
+			return ctx, nil
+		},
+	)
+	testenv.Finish(
+		envfuncs.DeleteNamespace("local-secrets-provider"),
+		envfuncs.DeleteNamespace("local-conjur"),
+	)
+	testenv.BeforeEachTest(
+		func(ctx context.Context, _ *envconf.Config, t *testing.T) (context.Context, error) {
+
+			return ctx, nil
+		},
+	)
+	testenv.AfterEachTest(
+		func(ctx context.Context, _ *envconf.Config, t *testing.T) (context.Context, error) {
+			envfuncs.DeleteNamespace("local-secrets-provider")
+
+			return ctx, nil
+		},
+	)
+
+	// launch package tests
+	os.Exit(testenv.Run(m))
+}