diff --git a/spec/5g/core_spec.cr b/spec/5g/core_spec.cr index 9a772355c..9e06127e6 100644 --- a/spec/5g/core_spec.cr +++ b/spec/5g/core_spec.cr @@ -14,45 +14,41 @@ describe "Core" do it "'smf_upf_heartbeat' should pass if the smf_upf core is resilient to network latency", tags: ["core"] do begin - ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample_open5gs/cnf-testsuite.yml") + ShellCmd.new_cnf_setup("cnf-config=sample-cnfs/sample_open5gs/cnf-testsuite.yml") result = ShellCmd.run_testsuite("smf_upf_heartbeat verbose") (/(PASSED).*(Chaos service degradation is less than 50%)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample_open5gs/cnf-testsuite.yml") - result[:status].success?.should be_true + ShellCmd.new_cnf_cleanup() end end it "'smf_upf_heartbeat' should fail if the smf_upf core is not resilient to network latency", tags: ["core"] do begin - ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample_open5gs/cnf-testsuite.yml") + ShellCmd.new_cnf_setup("cnf-config=sample-cnfs/sample_open5gs/cnf-testsuite.yml") result = ShellCmd.run_testsuite("smf_upf_heartbeat verbose baseline_count=300") (/(FAILED).*(Chaos service degradation is more than 50%)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample_open5gs/cnf-testsuite.yml") - result[:status].success?.should be_true + ShellCmd.new_cnf_cleanup() end end it "'suci_enabled' should pass if the 5G core has suci enabled", tags: ["5g"] do begin - ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample_open5gs/cnf-testsuite.yml") + ShellCmd.new_cnf_setup("cnf-config=sample-cnfs/sample_open5gs/cnf-testsuite.yml") result = ShellCmd.run_testsuite("suci_enabled verbose") (/(PASSED).*(Core uses SUCI 5g authentication)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample_open5gs/cnf-testsuite.yml") - result[:status].success?.should be_true + ShellCmd.new_cnf_cleanup() end end it "'suci_enabled' should fail if the 5G core does not have suci enabled", tags: ["5g"] do begin - ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample_open5gs_no_auth/cnf-testsuite.yml") + ShellCmd.new_cnf_setup("cnf-config=sample-cnfs/sample_open5gs_no_auth/cnf-testsuite.yml") result = ShellCmd.run_testsuite("suci_enabled verbose") (/(FAILED).*(Core does not use SUCI 5g authentication)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample_open5gs_no_auth/cnf-testsuite.yml") - result[:status].success?.should be_true + ShellCmd.new_cnf_cleanup() end end diff --git a/spec/5g/ran_spec.cr b/spec/5g/ran_spec.cr index 67b495cfa..c2b2899e0 100644 --- a/spec/5g/ran_spec.cr +++ b/spec/5g/ran_spec.cr @@ -38,14 +38,13 @@ describe "5g" do begin setup_success = setup_5g_network setup_success.should be_true - ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample-oran-ric/cnf-testsuite.yml") + ShellCmd.new_cnf_setup("cnf-config=sample-cnfs/sample-oran-ric/cnf-testsuite.yml") result = ShellCmd.run_testsuite("oran_e2_connection verbose") (/(PASSED).*(RAN connects to a RIC using the e2 standard interface)/ =~ result[:output]).should_not be_nil ensure result = Helm.delete("open5gs -n oran") result[:status].success?.should be_true - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample-oran-ric/cnf-testsuite.yml") - result[:status].success?.should be_true + ShellCmd.new_cnf_cleanup() end end @@ -53,14 +52,13 @@ describe "5g" do begin setup_success = setup_5g_network setup_success.should be_true - ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample-oran-noric/cnf-testsuite.yml") + ShellCmd.new_cnf_setup("cnf-config=sample-cnfs/sample-oran-noric/cnf-testsuite.yml") result = ShellCmd.run_testsuite("oran_e2_connection verbose") (/(FAILED).*(RAN does not connect to a RIC using the e2 standard interface)/ =~ result[:output]).should_not be_nil ensure result = Helm.delete("open5gs -n oran") result[:status].success?.should be_true - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample-oran-noric/cnf-testsuite.yml") - result[:status].success?.should be_true + ShellCmd.new_cnf_cleanup() end end diff --git a/spec/setup_spec.cr b/spec/setup_spec.cr index a2ffe959d..658a90779 100644 --- a/spec/setup_spec.cr +++ b/spec/setup_spec.cr @@ -21,22 +21,20 @@ describe "Setup" do it "'cnf_setup/cnf_cleanup' should install/cleanup with cnf-path arg as alias for cnf-config", tags: ["setup"] do begin - result = ShellCmd.cnf_setup("cnf-path=example-cnfs/coredns/cnf-testsuite.yml") + result = ShellCmd.new_cnf_setup("cnf-path=example-cnfs/coredns/cnf-testsuite.yml") (/Successfully setup coredns/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=example-cnfs/coredns/cnf-testsuite.yml") - result[:status].success?.should be_true + result = ShellCmd.new_cnf_cleanup() (/Successfully cleaned up/ =~ result[:output]).should_not be_nil end end it "'cnf_setup/cnf_cleanup' should install/cleanup a cnf with a cnf-testsuite.yml", tags: ["setup"] do begin - result = ShellCmd.cnf_setup("cnf-config=example-cnfs/coredns/cnf-testsuite.yml") + result = ShellCmd.new_cnf_setup("cnf-config=example-cnfs/coredns/cnf-testsuite.yml") (/Successfully setup coredns/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=example-cnfs/coredns/cnf-testsuite.yml") - result[:status].success?.should be_true + result = ShellCmd.new_cnf_cleanup() (/Successfully cleaned up/ =~ result[:output]).should_not be_nil end end @@ -44,50 +42,45 @@ describe "Setup" do it "'cnf_setup/cnf_cleanup' should work with cnf-testsuite.yml that has no directory associated with it", tags: ["setup"] do begin #TODO force cnfs/ to be deployment name and not the directory name - result = ShellCmd.cnf_setup("cnf-config=spec/fixtures/cnf-testsuite.yml verbose") + result = ShellCmd.new_cnf_setup("cnf-config=spec/fixtures/cnf-testsuite.yml verbose") (/Successfully setup coredns/ =~ result[:output]).should_not be_nil ensure - - result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=spec/fixtures/cnf-testsuite.yml verbose") - result[:status].success?.should be_true + result = ShellCmd.new_cnf_cleanup() (/Successfully cleaned up/ =~ result[:output]).should_not be_nil end end it "'cnf_setup/cnf_cleanup' should install/cleanup with helm_directory that descends multiple directories", tags: ["setup"] do begin - result = ShellCmd.cnf_setup("cnf-path=sample-cnfs/multi_helm_directories/cnf-testsuite.yml") + result = ShellCmd.new_cnf_setup("cnf-path=sample-cnfs/multi_helm_directories/cnf-testsuite.yml") (/Successfully setup coredns/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=sample-cnfs/multi_helm_directories/cnf-testsuite.yml") - result[:status].success?.should be_true + result = ShellCmd.new_cnf_cleanup() (/Successfully cleaned up/ =~ result[:output]).should_not be_nil end end it "'cnf_setup/cnf_cleanup' should properly install/uninstall old versions of cnf configs", tags: ["setup"] do begin - result = ShellCmd.cnf_setup("cnf-path=spec/fixtures/cnf-testsuite-v1-example.yml") + result = ShellCmd.new_cnf_setup("cnf-path=spec/fixtures/cnf-testsuite-v1-example.yml") result[:status].success?.should be_true (/Successfully setup coredns/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=spec/fixtures/cnf-testsuite-v1-example.yml") - result[:status].success?.should be_true + result = ShellCmd.new_cnf_cleanup() (/Successfully cleaned up/ =~ result[:output]).should_not be_nil end end it "'cnf_setup' should fail if another CNF is already installed", tags: ["setup"] do begin - result = ShellCmd.cnf_setup("cnf-path=sample-cnfs/sample_coredns/cnf-testsuite.yml") + result = ShellCmd.new_cnf_setup("cnf-path=sample-cnfs/sample_coredns/cnf-testsuite.yml") (/Successfully setup coredns/ =~ result[:output]).should_not be_nil - result = ShellCmd.cnf_setup("cnf-path=sample-cnfs/sample-minimal-cnf/cnf-testsuite.yml") + result = ShellCmd.new_cnf_setup("cnf-path=sample-cnfs/sample-minimal-cnf/cnf-testsuite.yml") (/A CNF is already set up. Setting up multiple CNFs is not allowed./ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=sample-cnfs/sample_coredns/cnf-testsuite.yml") - result[:status].success?.should be_true + result = ShellCmd.new_cnf_cleanup() (/Successfully cleaned up/ =~ result[:output]).should_not be_nil - result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=sample-cnfs/sample-minimal-cnf/cnf-testsuite.yml") + ShellCmd.new_cnf_cleanup(expect_failure: true) end end end diff --git a/spec/spec_helper.cr b/spec/spec_helper.cr index 7cc84b974..3aca87e6c 100644 --- a/spec/spec_helper.cr +++ b/spec/spec_helper.cr @@ -30,4 +30,24 @@ module ShellCmd end result end + + def self.new_cnf_setup(setup_params, cmd_prefix="", expect_failure=false) + result = run_testsuite("new_cnf_setup #{setup_params} timeout=300", cmd_prefix) + if !expect_failure + result[:status].success?.should be_true + else + result[:status].success?.should be_false + end + result + end + + def self.new_cnf_cleanup(expect_failure=false) + result = run_testsuite("new_cnf_cleanup") + if !expect_failure + result[:status].success?.should be_true + else + result[:status].success?.should be_false + end + result + end end diff --git a/spec/utils/cnf_manager_spec.cr b/spec/utils/cnf_manager_spec.cr index 29d0fa265..777322d40 100644 --- a/spec/utils/cnf_manager_spec.cr +++ b/spec/utils/cnf_manager_spec.cr @@ -25,15 +25,15 @@ describe "SampleUtils" do end it "'cnf_setup' should pass with a minimal cnf-testsuite.yml", tags: ["cnf-setup"] do - ShellCmd.cnf_setup("cnf-path=./sample-cnfs/sample-minimal-cnf/ skip_wait_for_install") + ShellCmd.new_cnf_setup("cnf-path=./sample-cnfs/sample-minimal-cnf/ skip_wait_for_install") ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=./sample-cnfs/sample-minimal-cnf/ force=true") + ShellCmd.new_cnf_cleanup() end it "'cnf_setup' should support cnf-config as an alias for cnf-path", tags: ["cnf-setup"] do - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample-minimal-cnf/ skip_wait_for_install") + ShellCmd.new_cnf_setup("cnf-config=./sample-cnfs/sample-minimal-cnf/ skip_wait_for_install") ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=./sample-cnfs/sample-minimal-cnf/ force=true") + ShellCmd.new_cnf_cleanup() end it "'points_yml' should parse and return the points yaml file", tags: ["points"] do @@ -348,13 +348,13 @@ describe "SampleUtils" do it "Helm_values should be used during the installation of a cnf", tags: ["cnf-config"] do begin # fails because doesn't have a service - ShellCmd.cnf_setup("cnf-path=./sample-cnfs/sample_coredns_values") + ShellCmd.new_cnf_setup("cnf-path=./sample-cnfs/sample_coredns_values") deployment_containers = KubectlClient::Get.resource_containers("deployment", "coredns-coredns", "cnf-default") image_tags = KubectlClient::Get.container_image_tags(deployment_containers) Log.info { "image_tags: #{image_tags}" } (/1.6.9/ =~ image_tags[0][:tag]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=./sample-cnfs/sample_coredns_values") + ShellCmd.new_cnf_cleanup() end end diff --git a/spec/utils/kubescape_spec.cr b/spec/utils/kubescape_spec.cr index 916751a92..dcafdd7f0 100644 --- a/spec/utils/kubescape_spec.cr +++ b/spec/utils/kubescape_spec.cr @@ -8,13 +8,13 @@ describe "K8sInstrumentation" do end it "'#scan and #test_by_test_name' should return the results of a kubescape scan", tags: ["kubescape"] do - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml") + ShellCmd.new_cnf_setup("cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml") Kubescape.scan results_json = Kubescape.parse test_json = Kubescape.test_by_test_name(results_json, "Network policies") (test_json).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.ymll") + ShellCmd.new_cnf_cleanup() end end diff --git a/spec/utils/utils_spec.cr b/spec/utils/utils_spec.cr index 0a8dacb47..a8a16d4ef 100644 --- a/spec/utils/utils_spec.cr +++ b/spec/utils/utils_spec.cr @@ -198,7 +198,7 @@ describe "Utils" do end it "'logger' or verbose output should be shown when verbose flag is set", tags: ["logger"] do - ShellCmd.cnf_setup("cnf-path=sample-cnfs/sample-coredns-cnf") + ShellCmd.new_cnf_setup("cnf-path=sample-cnfs/sample-coredns-cnf") result = ShellCmd.run_testsuite("helm_deploy verbose", cmd_prefix: "LOG_LEVEL=info") puts result[:output] result[:status].success?.should be_true diff --git a/spec/workload/compatibility_spec.cr b/spec/workload/compatibility_spec.cr index 563c43864..13a9d5e04 100644 --- a/spec/workload/compatibility_spec.cr +++ b/spec/workload/compatibility_spec.cr @@ -14,7 +14,7 @@ describe "Compatibility" do it "'cni_compatible' should pass if the cnf works with calico and flannel", tags: ["compatibility"] do begin - ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") + ShellCmd.new_cnf_setup("cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") retry_limit = 5 retries = 1 result = ShellCmd.run_testsuite("cni_compatible verbose") @@ -27,19 +27,18 @@ describe "Compatibility" do Log.info { "Status: #{result[:output]}" } (/(PASSED).*(CNF compatible with both Calico and Cilium)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") - result[:status].success?.should be_true + ShellCmd.new_cnf_cleanup() end end it "'increase_decrease_capacity' should pass ", tags: ["increase_decrease_capacity"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml verbose skip_wait_for_install") + ShellCmd.new_cnf_setup("cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml verbose skip_wait_for_install") result = ShellCmd.run_testsuite("increase_decrease_capacity verbose") result[:status].success?.should be_true (/(PASSED).*(Replicas increased to)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml") + ShellCmd.new_cnf_cleanup() end end end diff --git a/spec/workload/configuration_spec.cr b/spec/workload/configuration_spec.cr index 51b239105..588c4ce7b 100644 --- a/spec/workload/configuration_spec.cr +++ b/spec/workload/configuration_spec.cr @@ -19,73 +19,73 @@ describe CnfTestSuite do it "'liveness' should pass when livenessProbe is set", tags: ["liveness"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/k8s-multiple-deployments/cnf-testsuite.yml deploy_with_chart=false") + ShellCmd.new_cnf_setup("cnf-config=./sample-cnfs/k8s-multiple-deployments/cnf-testsuite.yml") result = ShellCmd.run_testsuite("liveness verbose", cmd_prefix:"LOG_LEVEL=debug") result[:status].success?.should be_true (/(PASSED).*(Helm liveness probe)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/k8s-multiple-deployments/cnf-testsuite.yml deploy_with_chart=false ") + ShellCmd.new_cnf_cleanup() end end it "'liveness' should fail when livenessProbe is not set", tags: ["liveness"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_coredns_bad_liveness/cnf-testsuite.yml verbose skip_wait_for_install") + ShellCmd.new_cnf_setup("cnf-config=./sample-cnfs/sample_coredns_bad_liveness/cnf-testsuite.yml verbose skip_wait_for_install") result = ShellCmd.run_testsuite("liveness verbose") result[:status].success?.should be_true (/(FAILED).*(No livenessProbe found)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("sample_coredns_bad_liveness_cleanup") + ShellCmd.new_cnf_cleanup() end end it "'readiness' should pass when readinessProbe is set", tags: ["readiness"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/k8s-multiple-deployments/cnf-testsuite.yml deploy_with_chart=false") + ShellCmd.new_cnf_setup("cnf-config=./sample-cnfs/k8s-multiple-deployments/cnf-testsuite.yml") result = ShellCmd.run_testsuite("readiness verbose", cmd_prefix: "LOG_LEVEL=debug") result[:status].success?.should be_true (/(PASSED).*(Helm readiness probe)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/k8s-multiple-deployments/cnf-testsuite.yml deploy_with_chart=false ") + ShellCmd.new_cnf_cleanup() end end it "'readiness' should fail when readinessProbe is not set", tags: ["readiness"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_coredns_bad_liveness/cnf-testsuite.yml verbose skip_wait_for_install") + ShellCmd.new_cnf_setup("cnf-config=./sample-cnfs/sample_coredns_bad_liveness/cnf-testsuite.yml verbose skip_wait_for_install") result = ShellCmd.run_testsuite("readiness verbose") result[:status].success?.should be_true (/(FAILED).*(No readinessProbe found)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("sample_coredns_bad_liveness_cleanup") + ShellCmd.new_cnf_cleanup() end end it "'rolling_update' should pass when valid version is given", tags: ["rolling_update"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_rolling/cnf-testsuite.yml verbose") + ShellCmd.new_cnf_setup("cnf-config=./sample-cnfs/sample_rolling/cnf-testsuite.yml verbose") result = ShellCmd.run_testsuite("rolling_update verbose") result[:status].success?.should be_true (/Passed/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample_rolling/cnf-testsuite.yml") + ShellCmd.new_cnf_cleanup() end end it "'rolling_update' should fail when invalid version is given", tags: ["rolling_update"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_rolling_invalid_version/cnf-testsuite.yml deploy_with_chart=false") + ShellCmd.new_cnf_setup("cnf-config=./sample-cnfs/sample_rolling_invalid_version/cnf-testsuite.yml") result = ShellCmd.run_testsuite("rolling_update verbose") result[:status].success?.should be_true (/Failed/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample_rolling_invalid_version/cnf-testsuite.yml deploy_with_chart=false") + ShellCmd.new_cnf_cleanup() end end it "'rolling_downgrade' should pass when valid version is given", tags: ["rolling_downgrade"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_rolling/cnf-testsuite.yml verbose skip_wait_for_install") + ShellCmd.new_cnf_setup("cnf-config=./sample-cnfs/sample_rolling/cnf-testsuite.yml verbose skip_wait_for_install") retry_limit = 5 retries = 1 result = ShellCmd.run_testsuite("rolling_downgrade verbose") @@ -99,51 +99,51 @@ describe CnfTestSuite do result[:status].success?.should be_true (/Passed/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample_rolling/cnf-testsuite.yml") + ShellCmd.new_cnf_cleanup() end end it "'rolling_downgrade' should fail when invalid version is given", tags: ["rolling_downgrade"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_rolling_invalid_version/cnf-testsuite.yml deploy_with_chart=false") + ShellCmd.new_cnf_setup("cnf-config=./sample-cnfs/sample_rolling_invalid_version/cnf-testsuite.yml") result = ShellCmd.run_testsuite("rolling_downgrade verbose") result[:status].success?.should be_true (/Failed/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample_rolling_invalid_version/cnf-testsuite.yml deploy_with_chart=false") + ShellCmd.new_cnf_cleanup() end end it "'rolling_version_change' should pass when valid version is given", tags: ["rolling_version_change"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_rolling/cnf-testsuite.yml verbose skip_wait_for_install") + ShellCmd.new_cnf_setup("cnf-config=./sample-cnfs/sample_rolling/cnf-testsuite.yml verbose skip_wait_for_install") result = ShellCmd.run_testsuite("rolling_version_change verbose") result[:status].success?.should be_true (/Passed/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample_rolling/cnf-testsuite.yml") + ShellCmd.new_cnf_cleanup() end end it "'rolling_version_change' should fail when invalid version is given", tags: ["rolling_version_change"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_rolling_invalid_version/cnf-testsuite.yml deploy_with_chart=false") + ShellCmd.new_cnf_setup("cnf-config=./sample-cnfs/sample_rolling_invalid_version/cnf-testsuite.yml") result = ShellCmd.run_testsuite("rolling_version_change verbose") result[:status].success?.should be_true (/Failed/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample_rolling_invalid_version/cnf-testsuite.yml deploy_with_chart=false") + ShellCmd.new_cnf_cleanup() end end it "'rollback' should pass ", tags: ["rollback"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_rolling/cnf-testsuite.yml verbose skip_wait_for_install") + ShellCmd.new_cnf_setup("cnf-config=./sample-cnfs/sample_rolling/cnf-testsuite.yml verbose skip_wait_for_install") result = ShellCmd.run_testsuite("rollback verbose") result[:status].success?.should be_true (/Passed/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample_rolling/cnf-testsuite.yml") + ShellCmd.new_cnf_cleanup() end end @@ -151,223 +151,223 @@ describe CnfTestSuite do it "'nodeport_not_used' should fail when a node port is being used", tags: ["nodeport_not_used"] do begin - ShellCmd.cnf_setup("cnf-path=sample-cnfs/sample_nodeport deploy_with_chart=false") + ShellCmd.new_cnf_setup("cnf-path=sample-cnfs/sample_nodeport") result = ShellCmd.run_testsuite("nodeport_not_used verbose") result[:status].success?.should be_true (/(FAILED).*(NodePort is being used)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=sample-cnfs/sample_nodeport deploy_with_chart=false") + ShellCmd.new_cnf_cleanup() end end it "'nodeport_not_used' should pass when a node port is not being used", tags: ["nodeport_not_used"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml verbose skip_wait_for_install") + ShellCmd.new_cnf_setup("cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml verbose skip_wait_for_install") result = ShellCmd.run_testsuite("nodeport_not_used verbose") result[:status].success?.should be_true (/(PASSED).*(NodePort is not used)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml") + ShellCmd.new_cnf_cleanup() end end it "'hostport_not_used' should fail when a node port is being used", tags: ["hostport_not_used"] do begin - ShellCmd.cnf_setup("cnf-path=sample-cnfs/sample_hostport deploy_with_chart=false") + ShellCmd.new_cnf_setup("cnf-path=sample-cnfs/sample_hostport") result = ShellCmd.run_testsuite("hostport_not_used verbose") result[:status].success?.should be_true (/(FAILED).*(HostPort is being used)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=sample-cnfs/sample_hostport deploy_with_chart=false") + ShellCmd.new_cnf_cleanup() end end it "'hostport_not_used' should pass when a node port is not being used", tags: ["hostport_not_used"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml verbose skip_wait_for_install") + ShellCmd.new_cnf_setup("cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml verbose skip_wait_for_install") result = ShellCmd.run_testsuite("hostport_not_used verbose") result[:status].success?.should be_true (/(PASSED).*(HostPort is not used)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml") + ShellCmd.new_cnf_cleanup() end end it "'ip_addresses' should pass when no uncommented ip addresses are found in helm chart source", tags: ["ip_addresses"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample-coredns-cnf-source/cnf-testsuite.yml verbose skip_wait_for_install") + ShellCmd.new_cnf_setup("cnf-config=./sample-cnfs/sample-coredns-cnf-source/cnf-testsuite.yml verbose skip_wait_for_install") result = ShellCmd.run_testsuite("ip_addresses verbose") result[:status].success?.should be_true (/(PASSED).*(No IP addresses found)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("sample_coredns_source_cleanup verbose") + ShellCmd.new_cnf_cleanup() end end it "'hardcoded_ip_addresses_in_k8s_runtime_configuration' should fail when a hardcoded ip is found in the K8s configuration", tags: ["ip_addresses"] do begin - ShellCmd.cnf_setup("cnf-path=sample-cnfs/sample_coredns_hardcoded_ips deploy_with_chart=false") + ShellCmd.new_cnf_setup("cnf-path=sample-cnfs/sample_coredns_hardcoded_ips") result = ShellCmd.run_testsuite("hardcoded_ip_addresses_in_k8s_runtime_configuration verbose", cmd_prefix: "LOG_LEVEL=info") result[:status].success?.should be_true (/(FAILED).*(Hard-coded IP addresses found in the runtime K8s configuration)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=sample-cnfs/sample_coredns_hardcoded_ips deploy_with_chart=false") + ShellCmd.new_cnf_cleanup() end end it "'hardcoded_ip_addresses_in_k8s_runtime_configuration' should pass when no ip addresses are found in the K8s configuration", tags: ["ip_addresses"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml verbose skip_wait_for_install") + ShellCmd.new_cnf_setup("cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml verbose skip_wait_for_install") result = ShellCmd.run_testsuite("hardcoded_ip_addresses_in_k8s_runtime_configuration verbose") result[:status].success?.should be_true (/(PASSED).*(No hard-coded IP addresses found in the runtime K8s configuration)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml") + ShellCmd.new_cnf_cleanup() end end it "'secrets_used' should pass when secrets are provided as volumes and used by a container", tags: ["secrets"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_secret_volume/cnf-testsuite.yml verbose") + ShellCmd.new_cnf_setup("cnf-config=./sample-cnfs/sample_secret_volume/cnf-testsuite.yml verbose") result = ShellCmd.run_testsuite("secrets_used verbose") result[:status].success?.should be_true (/(PASSED).*(Secrets defined and used)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=sample-cnfs/sample_secret_volume verbose") + ShellCmd.new_cnf_cleanup() end end it "'secrets_used' should be skipped when secrets are provided as volumes and not mounted by a container", tags: ["secrets"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_unmounted_secret_volume/cnf-testsuite.yml verbose skip_wait_for_install") + ShellCmd.new_cnf_setup("cnf-config=./sample-cnfs/sample_unmounted_secret_volume/cnf-testsuite.yml verbose skip_wait_for_install") result = ShellCmd.run_testsuite("secrets_used verbose") result[:status].success?.should be_true (/(SKIPPED).*(Secrets not used)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=sample-cnfs/sample_unmounted_secret_volume verbose") + ShellCmd.new_cnf_cleanup() end end it "'secrets_used' should pass when secrets are provided as environment variables and used by a container", tags: ["secrets"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_secret_env/cnf-testsuite.yml verbose") + ShellCmd.new_cnf_setup("cnf-config=./sample-cnfs/sample_secret_env/cnf-testsuite.yml verbose") result = ShellCmd.run_testsuite("secrets_used verbose") result[:status].success?.should be_true (/(PASSED).*(Secrets defined and used)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=sample-cnfs/sample_secret_env verbose") + ShellCmd.new_cnf_cleanup() end end it "'secrets_used' should skip when secrets are not referenced as environment variables by a container", tags: ["secrets"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_secret_env_no_ref/cnf-testsuite.yml skip_wait_for_install verbose") + ShellCmd.new_cnf_setup("cnf-config=./sample-cnfs/sample_secret_env_no_ref/cnf-testsuite.yml skip_wait_for_install verbose") result = ShellCmd.run_testsuite("secrets_used verbose") result[:status].success?.should be_true (/(SKIPPED).*(Secrets not used)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=sample-cnfs/sample_secret_env verbose") + ShellCmd.new_cnf_cleanup() end end it "'secrets_used' should be skipped when no secret volumes are mounted or no container secrets are provided (secrets ignored)`", tags: ["secrets"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml verbose skip_wait_for_install") + ShellCmd.new_cnf_setup("cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml verbose skip_wait_for_install") result = ShellCmd.run_testsuite("secrets_used verbose") result[:status].success?.should be_true (/(SKIPPED).*(Secrets not used)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=sample-cnfs/sample_coredns verbose") + ShellCmd.new_cnf_cleanup() end end it "'immutable_configmap' fail with some mutable configmaps in container env or volume mount", tags: ["immutable_configmap"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/ndn-mutable-configmap deploy_with_chart=false") + ShellCmd.new_cnf_setup("cnf-config=./sample-cnfs/ndn-mutable-configmap") result = ShellCmd.run_testsuite("immutable_configmap verbose") result[:status].success?.should be_true (/(FAILED).*(Found mutable configmap)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/ndn-mutable-configmap deploy_with_chart=false") + ShellCmd.new_cnf_cleanup() end end it "'immutable_configmap' pass with all immutable configmaps in container env or volume mounts", tags: ["immutable_configmap"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/ndn-immutable-configmap deploy_with_chart=false") + ShellCmd.new_cnf_setup("cnf-config=./sample-cnfs/ndn-immutable-configmap") result = ShellCmd.run_testsuite("immutable_configmap verbose") result[:status].success?.should be_true (/(PASSED).*(All volume or container mounted configmaps immutable)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/ndn-immutable-configmap deploy_with_chart=false") + ShellCmd.new_cnf_cleanup() end end it "'require_labels' should fail if a cnf does not have the app.kubernetes.io/name label", tags: ["require_labels"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_nonroot/cnf-testsuite.yml") + ShellCmd.new_cnf_setup("cnf-config=./sample-cnfs/sample_nonroot/cnf-testsuite.yml") result = ShellCmd.run_testsuite("require_labels verbose") result[:status].success?.should be_true (/(FAILED).*(Pods should have the app.kubernetes.io\/name label)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample_nonroot/cnf-testsuite.yml") + ShellCmd.new_cnf_cleanup() end end it "'require_labels' should pass if a cnf has the app.kubernetes.io/name label", tags: ["require_labels"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml") + ShellCmd.new_cnf_setup("cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml") result = ShellCmd.run_testsuite("require_labels verbose") result[:status].success?.should be_true (/(PASSED).*(Pods have the app.kubernetes.io\/name label)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml") + ShellCmd.new_cnf_cleanup() end end it "'default_namespace' should fail if a cnf creates resources in the default namespace", tags: ["default_namespace"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_coredns_default_namespace") + ShellCmd.new_cnf_setup("cnf-config=./sample-cnfs/sample_coredns_default_namespace") result = ShellCmd.run_testsuite("default_namespace verbose") result[:status].success?.should be_true (/(FAILED).*(Resources are created in the default namespace)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample_coredns_default_namespace") + ShellCmd.new_cnf_cleanup() KubectlClient::Utils.wait_for_terminations() end end it "'default_namespace' should pass if a cnf does not create resources in the default namespace", tags: ["default_namespace"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_latest_tag") + ShellCmd.new_cnf_setup("cnf-config=./sample-cnfs/sample_latest_tag") result = ShellCmd.run_testsuite("default_namespace verbose") result[:status].success?.should be_true (/(PASSED).*(default namespace is not being used)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample_latest_tag") + ShellCmd.new_cnf_cleanup() KubectlClient::Utils.wait_for_terminations() end end it "'latest_tag' should fail if a cnf has containers that use images with the latest tag", tags: ["latest_tag"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_latest_tag") + ShellCmd.new_cnf_setup("cnf-config=./sample-cnfs/sample_latest_tag") result = ShellCmd.run_testsuite("latest_tag verbose") result[:status].success?.should be_true (/(FAILED).*(Container images are using the latest tag)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample_latest_tag") + ShellCmd.new_cnf_cleanup() end end it "'latest_tag' should pass if a cnf does not have containers that use images with the latest tag", tags: ["latest_tag"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_nonroot") + ShellCmd.new_cnf_setup("cnf-config=./sample-cnfs/sample_nonroot") result = ShellCmd.run_testsuite("latest_tag verbose") result[:status].success?.should be_true (/(PASSED).*(Container images are not using the latest tag)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample_nonroot") + ShellCmd.new_cnf_cleanup() end end @@ -378,7 +378,7 @@ describe CnfTestSuite do result[:status].success?.should be_false (/You must install a CNF first./ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample_nonroot") + ShellCmd.new_cnf_cleanup() end end end diff --git a/spec/workload/installability_spec.cr b/spec/workload/installability_spec.cr index daf15e524..272e4d41b 100644 --- a/spec/workload/installability_spec.cr +++ b/spec/workload/installability_spec.cr @@ -9,12 +9,12 @@ describe CnfTestSuite do end it "'helm_deploy' should fail on a manifest CNF", tags: ["helm"] do - ShellCmd.cnf_setup("cnf-path=./sample-cnfs/k8s-non-helm") + ShellCmd.new_cnf_setup("cnf-path=./sample-cnfs/k8s-non-helm") result = ShellCmd.run_testsuite("helm_deploy verbose") result[:status].success?.should be_true (/(FAILED).*(Helm deploy failed)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=./sample-cnfs/k8s-non-helm verbose") + ShellCmd.new_cnf_cleanup() end it "'helm_deploy' should fail if command is not supplied cnf-config argument", tags: ["helm"] do @@ -24,47 +24,47 @@ describe CnfTestSuite do end it "'helm_chart_valid' should pass on a good helm chart", tags: ["helm"] do - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml verbose") + ShellCmd.new_cnf_setup("cnf-config=./sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml verbose") result = ShellCmd.run_testsuite("helm_chart_valid verbose") result[:status].success?.should be_true (/Lint Passed/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml verbose") + ShellCmd.new_cnf_cleanup() end it "'helm_chart_valid' should fail on a bad helm chart", tags: ["helm"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample-bad_helm_coredns-cnf/cnf-testsuite.yml verbose skip_wait_for_install", expect_failure: true) + ShellCmd.new_cnf_setup("cnf-config=./sample-cnfs/sample-bad_helm_coredns-cnf/cnf-testsuite.yml verbose skip_wait_for_install", expect_failure: true) result = ShellCmd.run_testsuite("helm_chart_valid") result[:status].success?.should be_true (/Lint Failed/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml verbose") + ShellCmd.new_cnf_cleanup() end end it "'helm_chart_published' should pass on a good helm chart repo", tags: ["helm_chart_published"] do begin - ShellCmd.cnf_setup("cnf-path=sample-cnfs/sample-coredns-cnf") + ShellCmd.new_cnf_setup("cnf-path=sample-cnfs/sample-coredns-cnf") result = ShellCmd.run_testsuite("helm_chart_published") result[:status].success?.should be_true (/(PASSED).*(Published Helm Chart Found)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=sample-cnfs/sample-coredns-cnf") + ShellCmd.new_cnf_cleanup() end end it "'helm_chart_published' should fail on a bad helm chart repo", tags: ["helm_chart_published"] do begin result = ShellCmd.run("helm search repo stable/coredns", force_output: true) - ShellCmd.cnf_setup("cnf-path=sample-cnfs/sample-bad-helm-repo skip_wait_for_install", expect_failure: true) + ShellCmd.new_cnf_setup("cnf-path=sample-cnfs/sample-bad-helm-repo skip_wait_for_install", expect_failure: true) result = ShellCmd.run("helm search repo stable/coredns", force_output: true) result = ShellCmd.run_testsuite("helm_chart_published verbose") result[:status].success?.should be_true (/(FAILED).*(Published Helm Chart Not Found)/ =~ result[:output]).should_not be_nil ensure result = ShellCmd.run("#{Helm::BinarySingleton.helm} repo remove badrepo") - result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=sample-cnfs/sample-bad-helm-repo") + ShellCmd.new_cnf_cleanup() end end end diff --git a/spec/workload/microservice_spec.cr b/spec/workload/microservice_spec.cr index 95da12792..a44cb5525 100644 --- a/spec/workload/microservice_spec.cr +++ b/spec/workload/microservice_spec.cr @@ -18,56 +18,52 @@ describe "Microservice" do it "'shared_database' should be skipped no MariaDB containers are found", tags: ["shared_database"] do begin - ShellCmd.cnf_setup("cnf-path=sample-cnfs/sample_coredns/cnf-testsuite.yml") + ShellCmd.new_cnf_setup("cnf-path=sample-cnfs/sample_coredns/cnf-testsuite.yml") result = ShellCmd.run_testsuite("shared_database") result[:status].success?.should be_true (/(N\/A).*(No MariaDB containers were found)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=sample-cnfs/sample_coredns/cnf-testsuite.yml") - result[:status].success?.should be_true + ShellCmd.new_cnf_cleanup() end end it "'shared_database' should pass if no database is used by two microservices", tags: ["shared_database"] do begin - ShellCmd.cnf_setup("cnf-path=sample-cnfs/sample-statefulset-cnf/cnf-testsuite.yml") + ShellCmd.new_cnf_setup("cnf-path=sample-cnfs/sample-statefulset-cnf/cnf-testsuite.yml") result = ShellCmd.run_testsuite("shared_database") result[:status].success?.should be_true (/(PASSED).*(No shared database found)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=sample-cnfs/sample-statefulset-cnf/cnf-testsuite.yml") - result[:status].success?.should be_true + ShellCmd.new_cnf_cleanup() end end it "'shared_database' should pass if one service connects to a database but other non-service connections are made to the database", tags: ["shared_database"] do begin - ShellCmd.cnf_setup("cnf-path=sample-cnfs/sample-multi-db-connections-exempt/cnf-testsuite.yml") + ShellCmd.new_cnf_setup("cnf-path=sample-cnfs/sample-multi-db-connections-exempt/cnf-testsuite.yml") result = ShellCmd.run_testsuite("shared_database") result[:status].success?.should be_true (/(PASSED).*(No shared database found)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=sample-cnfs/sample-multi-db-connections-exempt/cnf-testsuite.yml") - result[:status].success?.should be_true + ShellCmd.new_cnf_cleanup() end end it "'shared_database' should fail if two services on the cluster connect to the same database", tags: ["shared_database2"] do begin - ShellCmd.cnf_setup("cnf-path=sample-cnfs/ndn-multi-db-connections-fail/cnf-testsuite.yml") + ShellCmd.new_cnf_setup("cnf-path=sample-cnfs/ndn-multi-db-connections-fail/cnf-testsuite.yml") result = ShellCmd.run_testsuite("shared_database") result[:status].success?.should be_true (/(FAILED).*(Found a shared database)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=sample-cnfs/ndn-multi-db-connections-fail/cnf-testsuite.yml") - result[:status].success?.should be_true + ShellCmd.new_cnf_cleanup() KubectlClient::Delete.command("pvc data-test-mariadb-0 -n wordpress") end end it "'shared_database' should pass if two services on the cluster connect to the same database but they are not in the helm chart of the cnf", tags: ["shared_database"] do begin - ShellCmd.cnf_setup("cnf-path=sample-cnfs/sample_coredns") + ShellCmd.new_cnf_setup("cnf-path=sample-cnfs/sample_coredns") Helm.install("multi-db sample-cnfs/ndn-multi-db-connections-fail/wordpress/") KubectlClient::Get.resource_wait_for_install(kind: "Deployment", resource_name: "multi-db-wordpress", wait_count: 180, namespace: "default") KubectlClient::Get.resource_wait_for_install(kind: "Deployment", resource_name: "multi-db-wordpress2", wait_count: 180, namespace: "default") @@ -82,68 +78,62 @@ describe "Microservice" do ensure Helm.delete("multi-db") KubectlClient::Delete.command("pvc data-multi-db-mariadb-0") - result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=sample-cnfs/ndn-multi-db-connections-fail/cnf-testsuite.yml") - result[:status].success?.should be_true + ShellCmd.new_cnf_cleanup() end end it "'single_process_type' should pass if the containers in the cnf have only one process type", tags: ["process_check"] do begin - ShellCmd.cnf_setup("cnf-path=sample-cnfs/sample_coredns") + ShellCmd.new_cnf_setup("cnf-path=sample-cnfs/sample_coredns") result = ShellCmd.run_testsuite("single_process_type verbose") result[:status].success?.should be_true (/(PASSED).*(Only one process type used)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=sample-cnfs/sample_coredns") - result[:status].success?.should be_true + ShellCmd.new_cnf_cleanup() end end it "'single_process_type' should fail if the containers in the cnf have more than one process type", tags: ["process_check"] do begin - ShellCmd.cnf_setup("cnf-path=sample-cnfs/k8s-multiple-processes") + ShellCmd.new_cnf_setup("cnf-path=sample-cnfs/k8s-multiple-processes") result = ShellCmd.run_testsuite("single_process_type verbose") result[:status].success?.should be_true (/(FAILED).*(More than one process type used)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=sample-cnfs/k8s-multiple-processes") - result[:status].success?.should be_true + ShellCmd.new_cnf_cleanup() end end it "'single_process_type' should fail if the containers in the cnf have more than one process type and in a pod", tags: ["process_check"] do begin - ShellCmd.cnf_setup("cnf-path=sample-cnfs/sample-multiple-processes") + ShellCmd.new_cnf_setup("cnf-path=sample-cnfs/sample-multiple-processes") result = ShellCmd.run_testsuite("single_process_type verbose") result[:status].success?.should be_true (/(FAILED).*(More than one process type used)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=sample-cnfs/sample-multiple-processes") - result[:status].success?.should be_true + ShellCmd.new_cnf_cleanup() end end it "'reasonable_startup_time' should pass if the cnf has a reasonable startup time(helm_directory)", tags: ["reasonable_startup_time"] do begin - ShellCmd.cnf_setup("cnf-path=sample-cnfs/sample_coredns") + ShellCmd.new_cnf_setup("cnf-path=sample-cnfs/sample_coredns") result = ShellCmd.run_testsuite("reasonable_startup_time verbose") result[:status].success?.should be_true (/(PASSED).*(CNF had a reasonable startup time)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=sample-cnfs/sample_coredns") - result[:status].success?.should be_true + ShellCmd.new_cnf_cleanup() end end it "'reasonable_startup_time' should fail if the cnf doesn't has a reasonable startup time(helm_directory)", tags: ["reasonable_startup_time"] do - ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample_envoy_slow_startup/cnf-testsuite.yml force=true") + ShellCmd.new_cnf_setup("cnf-config=sample-cnfs/sample_envoy_slow_startup/cnf-testsuite.yml") begin result = ShellCmd.run_testsuite("reasonable_startup_time verbose") result[:status].success?.should be_true (/(FAILED).*(CNF had a startup time of)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample_envoy_slow_startup/cnf-testsuite.yml force=true") - result[:status].success?.should be_true + ShellCmd.new_cnf_cleanup() end end @@ -157,60 +147,58 @@ describe "Microservice" do #else cnf = "./sample-cnfs/sample-coredns-cnf" #end - ShellCmd.cnf_setup("cnf-path=#{cnf}") + ShellCmd.new_cnf_setup("cnf-path=#{cnf}") result = ShellCmd.run_testsuite("reasonable_image_size verbose") result[:status].success?.should be_true (/Image size is good/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=#{cnf}") + ShellCmd.new_cnf_cleanup() end it "'reasonable_image_size' should fail if image is larger than 5gb", tags: ["reasonable_image_size"] do - ShellCmd.cnf_setup("cnf-path=./sample-cnfs/ndn-reasonable-image-size skip_wait_for_install") + ShellCmd.new_cnf_setup("cnf-path=./sample-cnfs/ndn-reasonable-image-size skip_wait_for_install") result = ShellCmd.run_testsuite("reasonable_image_size verbose") result[:status].success?.should be_true (/Image size too large/ =~ result[:output]).should_not be_nil end it "'specialized_init_system' should fail if pods do not use specialized init systems", tags: ["specialized_init_system"] do - ShellCmd.cnf_setup("cnf-path=./sample-cnfs/sample-coredns-cnf") + ShellCmd.new_cnf_setup("cnf-path=./sample-cnfs/sample-coredns-cnf") result = ShellCmd.run_testsuite("specialized_init_system") result[:status].success?.should be_true (/Containers do not use specialized init systems/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=./sample-cnfs/sample-coredns-cnf force=true") + ShellCmd.new_cnf_cleanup() end it "'specialized_init_system' should pass if pods use specialized init systems", tags: ["specialized_init_system"] do - ShellCmd.cnf_setup("cnf-path=./sample-cnfs/sample-init-systems") + ShellCmd.new_cnf_setup("cnf-path=./sample-cnfs/sample-init-systems") result = ShellCmd.run_testsuite("specialized_init_system") result[:status].success?.should be_true (/Containers use specialized init systems/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=./sample-cnfs/sample-init-systems force=true") + ShellCmd.new_cnf_cleanup() end it "'service_discovery' should pass if any containers in the cnf are exposed as a service", tags: ["service_discovery"] do begin - ShellCmd.cnf_setup("cnf-path=sample-cnfs/sample_coredns") + ShellCmd.new_cnf_setup("cnf-path=sample-cnfs/sample_coredns") result = ShellCmd.run_testsuite("service_discovery verbose") result[:status].success?.should be_true (/(PASSED).*(Some containers exposed as a service)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=sample-cnfs/sample_coredns") - result[:status].success?.should be_true + ShellCmd.new_cnf_cleanup() end end it "'service_discovery' should fail if no containers in the cnf are exposed as a service", tags: ["service_discovery"] do begin - ShellCmd.cnf_setup("cnf-path=./sample-cnfs/sample-ndn-privileged") + ShellCmd.new_cnf_setup("cnf-path=./sample-cnfs/sample-ndn-privileged") result = ShellCmd.run_testsuite("service_discovery verbose") result[:status].success?.should be_true (/(FAILED).*(No containers exposed as a service)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=./sample-cnfs/sample-ndn-privileged") - result[:status].success?.should be_true + ShellCmd.new_cnf_cleanup() end end @@ -221,13 +209,12 @@ describe "Microservice" do #todo 3. Collect all signals sent, if SIGKILL is captured, application fails test because it doesn't exit child processes cleanly #todo 3. Collect all signals sent, if SIGTERM is captured, application pass test because it exits child processes cleanly #todo 4. Make sure that threads are not counted as new processes. A thread does not get a signal (sigterm or sigkill) - ShellCmd.cnf_setup("cnf-path=./sample-cnfs/sample_good_signal_handling/") + ShellCmd.new_cnf_setup("cnf-path=./sample-cnfs/sample_good_signal_handling/") result = ShellCmd.run_testsuite("sig_term_handled verbose") result[:status].success?.should be_true (/(PASSED).*(Sig Term handled)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=./sample-cnfs/sample_good_signal_handling/") - result[:status].success?.should be_true + ShellCmd.new_cnf_cleanup() end end @@ -238,13 +225,12 @@ describe "Microservice" do #todo 3. Collect all signals sent, if SIGKILL is captured, application fails test because it doesn't exit child processes cleanly #todo 3. Collect all signals sent, if SIGTERM is captured, application pass test because it exits child processes cleanly #todo 4. Make sure that threads are not counted as new processes. A thread does not get a signal (sigterm or sigkill) - ShellCmd.cnf_setup("cnf-path=./sample-cnfs/sample_bad_signal_handling/") + ShellCmd.new_cnf_setup("cnf-path=./sample-cnfs/sample_bad_signal_handling/") result = ShellCmd.run_testsuite("sig_term_handled verbose") result[:status].success?.should be_true (/(FAILED).*(Sig Term not handled)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=./sample-cnfs/sample_bad_signal_handling/") - result[:status].success?.should be_true + ShellCmd.new_cnf_cleanup() end end @@ -255,7 +241,7 @@ describe "Microservice" do #todo 3. Collect all signals sent, if SIGKILL is captured, application fails test because it doesn't exit child processes cleanly #todo 3. Collect all signals sent, if SIGTERM is captured, application pass test because it exits child processes cleanly #todo 4. Make sure that threads are not counted as new processes. A thread does not get a signal (sigterm or sigkill) - ShellCmd.cnf_setup("cnf-path=./sample-cnfs/sample_good_signal_handling_tini/") + ShellCmd.new_cnf_setup("cnf-path=./sample-cnfs/sample_good_signal_handling_tini/") # Workaround to wait using kubectl because Jenkins pod takes a LONG time to start. result = ShellCmd.run("kubectl wait --for=condition=ready=True pod/jenkins-0 -n cnfspace --timeout=500s", force_output: true) @@ -264,33 +250,30 @@ describe "Microservice" do result[:status].success?.should be_true (/(PASSED).*(Sig Term handled)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=./sample-cnfs/sample_good_signal_handling_tini/") - result[:status].success?.should be_true + ShellCmd.new_cnf_cleanup() end end it "'zombie_handled' should pass if a zombie is succesfully reaped by PID 1", tags: ["zombie"] do begin - ShellCmd.cnf_setup("cnf-path=./sample-cnfs/sample_good_zombie_handling/") + ShellCmd.new_cnf_setup("cnf-path=./sample-cnfs/sample_good_zombie_handling/") result = ShellCmd.run_testsuite("zombie_handled verbose") result[:status].success?.should be_true (/(PASSED).*(Zombie handled)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=./sample-cnfs/sample_good_zombie_handling/") - result[:status].success?.should be_true + ShellCmd.new_cnf_cleanup() end end it "'zombie_handled' should failed if a zombie is not succesfully reaped by PID 1", tags: ["zombie"] do begin - ShellCmd.cnf_setup("cnf-path=./sample-cnfs/sample-bad-zombie/") + ShellCmd.new_cnf_setup("cnf-path=./sample-cnfs/sample-bad-zombie/") result = ShellCmd.run_testsuite("zombie_handled verbose") result[:status].success?.should be_true (/(FAILED).*(Zombie not handled)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=./sample-cnfs/sample-bad-zombie/") - result[:status].success?.should be_true + ShellCmd.new_cnf_cleanup() end end end diff --git a/spec/workload/observability_spec.cr b/spec/workload/observability_spec.cr index ad7671f83..dd192430f 100644 --- a/spec/workload/observability_spec.cr +++ b/spec/workload/observability_spec.cr @@ -12,28 +12,28 @@ describe "Observability" do it "'log_output' should pass with a cnf that outputs logs to stdout", tags: ["observability"] do begin - ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") + ShellCmd.new_cnf_setup("cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") result = ShellCmd.run_testsuite("log_output verbose") result[:status].success?.should be_true (/(PASSED).*(Resources output logs to stdout and stderr)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") + ShellCmd.new_cnf_cleanup() end end it "'log_output' should fail with a cnf that does not output logs to stdout", tags: ["observability"] do begin - ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample_no_logs/cnf-testsuite.yml") + ShellCmd.new_cnf_setup("cnf-config=sample-cnfs/sample_no_logs/cnf-testsuite.yml") result = ShellCmd.run_testsuite("log_output verbose") result[:status].success?.should be_true (/(FAILED).*(Resources do not output logs to stdout and stderr)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample_no_logs/cnf-testsuite.yml") + ShellCmd.new_cnf_cleanup() end end it "'prometheus_traffic' should pass if there is prometheus traffic", tags: ["observability"] do - ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample-prom-pod-discovery/cnf-testsuite.yml") + ShellCmd.new_cnf_setup("cnf-config=sample-cnfs/sample-prom-pod-discovery/cnf-testsuite.yml") helm = Helm::BinarySingleton.helm Log.info { "Add prometheus helm repo" } @@ -49,26 +49,26 @@ describe "Observability" do test_result = ShellCmd.run_testsuite("prometheus_traffic") (/(PASSED).*(Your cnf is sending prometheus traffic)/ =~ test_result[:output]).should_not be_nil ensure - ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample-prom-pod-discovery/cnf-testsuite.yml") + ShellCmd.new_cnf_cleanup() result = ShellCmd.run("#{helm} delete prometheus", "helm_delete_prometheus") result[:status].success?.should be_true end it "'prometheus_traffic' should skip if there is no prometheus installed", tags: ["observability"] do - ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") + ShellCmd.new_cnf_setup("cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") helm = Helm::BinarySingleton.helm result = ShellCmd.run("#{helm} delete prometheus", force_output: true) result = ShellCmd.run_testsuite("prometheus_traffic") (/(SKIPPED).*(Prometheus server not found)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") + ShellCmd.new_cnf_cleanup() end it "'prometheus_traffic' should fail if the cnf is not registered with prometheus", tags: ["observability"] do - ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") + ShellCmd.new_cnf_setup("cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") Log.info { "Installing prometheus server" } helm = Helm::BinarySingleton.helm result = ShellCmd.run("helm repo add prometheus-community https://prometheus-community.github.io/helm-charts", force_output: true) @@ -80,13 +80,13 @@ describe "Observability" do result = ShellCmd.run_testsuite("prometheus_traffic") (/(FAILED).*(Your cnf is not sending prometheus traffic)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") + ShellCmd.new_cnf_cleanup() result = ShellCmd.run("#{helm} delete prometheus", force_output: true) result[:status].success?.should be_true end it "'open_metrics' should fail if there is not a valid open metrics response from the cnf", tags: ["observability"] do - ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample-prom-pod-discovery/cnf-testsuite.yml") + ShellCmd.new_cnf_setup("cnf-config=sample-cnfs/sample-prom-pod-discovery/cnf-testsuite.yml") result = ShellCmd.run("helm repo add prometheus-community https://prometheus-community.github.io/helm-charts", force_output: true) Log.info { "Installing prometheus server" } helm = Helm::BinarySingleton.helm @@ -98,13 +98,13 @@ describe "Observability" do result = ShellCmd.run_testsuite("open_metrics") (/(FAILED).*(Your cnf's metrics traffic is not OpenMetrics compatible)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample-prom-pod-discovery/cnf-testsuite.yml") + ShellCmd.new_cnf_cleanup() result = ShellCmd.run("#{helm} delete prometheus", force_output: true) result[:status].success?.should be_true end it "'open_metrics' should pass if there is a valid open metrics response from the cnf", tags: ["observability"] do - ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample-openmetrics/cnf-testsuite.yml") + ShellCmd.new_cnf_setup("cnf-config=sample-cnfs/sample-openmetrics/cnf-testsuite.yml") result = ShellCmd.run("helm repo add prometheus-community https://prometheus-community.github.io/helm-charts", force_output: true) Log.info { "Installing prometheus server" } helm = Helm::BinarySingleton.helm @@ -116,36 +116,36 @@ describe "Observability" do result = ShellCmd.run_testsuite("open_metrics") (/(PASSED).*(Your cnf's metrics traffic is OpenMetrics compatible)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample-openmetrics/cnf-testsuite.yml") + ShellCmd.new_cnf_cleanup() result = ShellCmd.run("#{helm} delete prometheus", force_output: true) result[:status].success?.should be_true end it "'routed_logs' should pass if cnfs logs are captured by fluentd bitnami", tags: ["observability"] do - ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") + ShellCmd.new_cnf_setup("cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") result = ShellCmd.run_testsuite("install_fluentdbitnami") result = ShellCmd.run_testsuite("routed_logs") (/(PASSED).*(Your CNF's logs are being captured)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") + ShellCmd.new_cnf_cleanup() result = ShellCmd.run_testsuite("uninstall_fluentdbitnami") result[:status].success?.should be_true end it "'routed_logs' should pass if cnfs logs are captured by fluentbit", tags: ["observability"] do - ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample-fluentbit") + ShellCmd.new_cnf_setup("cnf-config=sample-cnfs/sample-fluentbit") result = ShellCmd.run_testsuite("install_fluentbit") result = ShellCmd.run_testsuite("routed_logs") (/(PASSED).*(Your CNF's logs are being captured)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample-fluentbit") + ShellCmd.new_cnf_cleanup() result = ShellCmd.run_testsuite("uninstall_fluentbit") result[:status].success?.should be_true end it "'routed_logs' should fail if cnfs logs are not captured", tags: ["observability"] do - ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") + ShellCmd.new_cnf_setup("cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") Helm.helm_repo_add("bitnami","https://charts.bitnami.com/bitnami") #todo #helm install --values ./override.yml fluentd ./fluentd Helm.install("--values ./spec/fixtures/fluentd-values-bad.yml -n #{TESTSUITE_NAMESPACE} fluentd bitnami/fluentd") @@ -155,7 +155,7 @@ describe "Observability" do result = ShellCmd.run_testsuite("routed_logs") (/(FAILED).*(Your CNF's logs are not being captured)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") + ShellCmd.new_cnf_cleanup() result = ShellCmd.run_testsuite("uninstall_fluentd") result[:status].success?.should be_true end @@ -164,11 +164,11 @@ describe "Observability" do Log.info { "Installing Jaeger " } JaegerManager.install - ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") + ShellCmd.new_cnf_setup("cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") result = ShellCmd.run_testsuite("tracing") (/(FAILED).*(Tracing not used)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") + ShellCmd.new_cnf_cleanup() JaegerManager.uninstall KubectlClient::Get.resource_wait_for_uninstall("Statefulset", "jaeger-cassandra") KubectlClient::Get.resource_wait_for_uninstall("Deployment", "jaeger-collector") @@ -180,11 +180,11 @@ describe "Observability" do Log.info { "Installing Jaeger " } JaegerManager.install - ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample-tracing/cnf-testsuite.yml") + ShellCmd.new_cnf_setup("cnf-config=sample-cnfs/sample-tracing/cnf-testsuite.yml") result = ShellCmd.run_testsuite("tracing") (/(PASSED).*(Tracing used)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample-tracing/cnf-testsuite.yml") + ShellCmd.new_cnf_cleanup() JaegerManager.uninstall KubectlClient::Get.resource_wait_for_uninstall("Statefulset", "jaeger-cassandra") KubectlClient::Get.resource_wait_for_uninstall("Deployment", "jaeger-collector") diff --git a/spec/workload/operator_spec.cr b/spec/workload/operator_spec.cr index 8e57c6d3e..9216aed3b 100644 --- a/spec/workload/operator_spec.cr +++ b/spec/workload/operator_spec.cr @@ -23,12 +23,11 @@ describe "Operator" do Helm.install("operator --set olm.image.ref=quay.io/operator-framework/olm:v0.22.0 --set catalog.image.ref=quay.io/operator-framework/olm:v0.22.0 --set package.image.ref=quay.io/operator-framework/olm:v0.22.0 #{install_dir}/deploy/chart/") begin - ShellCmd.cnf_setup("cnf-path=./sample-cnfs/sample_operator", cmd_prefix: "LOG_LEVEL=info") + ShellCmd.new_cnf_setup("cnf-path=./sample-cnfs/sample_operator", cmd_prefix: "LOG_LEVEL=info") result = ShellCmd.run_testsuite("operator_installed", cmd_prefix: "LOG_LEVEL=info") (/(PASSED).*(Operator is installed)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=./sample-cnfs/sample_operator", cmd_prefix: "LOG_LEVEL=info") - result[:status].success?.should be_true + ShellCmd.new_cnf_cleanup() pods = KubectlClient::Get.pods_by_resource(KubectlClient::Get.deployment("catalog-operator", "operator-lifecycle-manager"), "operator-lifecycle-manager") + KubectlClient::Get.pods_by_resource(KubectlClient::Get.deployment("olm-operator", "operator-lifecycle-manager"), "operator-lifecycle-manager") + KubectlClient::Get.pods_by_resource(KubectlClient::Get.deployment("packageserver", "operator-lifecycle-manager"), "operator-lifecycle-manager") Helm.uninstall("operator") @@ -67,12 +66,11 @@ describe "Operator" do it "'operator_test' operator should not be found", tags: ["operator_test"] do begin - ShellCmd.cnf_setup("cnf-path=sample-cnfs/sample_coredns") + ShellCmd.new_cnf_setup("cnf-path=sample-cnfs/sample_coredns") result = ShellCmd.run_testsuite("operator_installed", cmd_prefix: "LOG_LEVEL=info") (/(N\/A).*(No Operators Found)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=sample-cnfs/sample_coredns") - result[:status].success?.should be_true + ShellCmd.new_cnf_cleanup() end end end diff --git a/spec/workload/registry_spec.cr b/spec/workload/registry_spec.cr index a88fa9616..340b5e014 100644 --- a/spec/workload/registry_spec.cr +++ b/spec/workload/registry_spec.cr @@ -42,23 +42,23 @@ describe "Private Registry: Image" do it "'reasonable_image_size' should pass if using local registry and a port", tags: ["private_registry_image"] do cnf="./sample-cnfs/sample_local_registry" - ShellCmd.cnf_setup("cnf-path=#{cnf}") + ShellCmd.new_cnf_setup("cnf-path=#{cnf}") result = ShellCmd.run_testsuite("reasonable_image_size verbose") result[:status].success?.should be_true (/Image size is good/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=#{cnf}") + ShellCmd.new_cnf_cleanup() end it "'reasonable_image_size' should pass if using local registry, a port and an org", tags: ["private_registry_image"] do cnf="./sample-cnfs/sample_local_registry_org_image" - ShellCmd.cnf_setup("cnf-path=#{cnf}") + ShellCmd.new_cnf_setup("cnf-path=#{cnf}") result = ShellCmd.run_testsuite("reasonable_image_size verbose") result[:status].success?.should be_true (/Image size is good/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=#{cnf}") + ShellCmd.new_cnf_cleanup() end after_all do @@ -91,12 +91,12 @@ describe "Private Registry: Rolling" do begin cnf="./sample-cnfs/sample_local_registry_rolling" - ShellCmd.cnf_setup("cnf-path=#{cnf}") + ShellCmd.new_cnf_setup("cnf-path=#{cnf}") result = ShellCmd.run_testsuite("rolling_update verbose") result[:status].success?.should be_true (/Passed/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=#{cnf} wait_count=0") + ShellCmd.new_cnf_cleanup() end end @@ -104,12 +104,12 @@ describe "Private Registry: Rolling" do begin cnf="./sample-cnfs/sample_local_registry_rolling" - ShellCmd.cnf_setup("cnf-path=#{cnf}") + ShellCmd.new_cnf_setup("cnf-path=#{cnf}") result = ShellCmd.run_testsuite("rolling_update verbose") result[:status].success?.should be_true (/Passed/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=#{cnf} wait_count=0") + ShellCmd.new_cnf_cleanup() end end @@ -117,12 +117,12 @@ describe "Private Registry: Rolling" do begin cnf="./sample-cnfs/sample_local_registry_rolling" - ShellCmd.cnf_setup("cnf-path=#{cnf}") + ShellCmd.new_cnf_setup("cnf-path=#{cnf}") result = ShellCmd.run_testsuite("rolling_version_change verbose") result[:status].success?.should be_true (/Passed/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-path=#{cnf} wait_count=0") + ShellCmd.new_cnf_cleanup() end end diff --git a/spec/workload/resilience/disk_fill_spec.cr b/spec/workload/resilience/disk_fill_spec.cr index 42fe68ce8..88c6b1ba0 100644 --- a/spec/workload/resilience/disk_fill_spec.cr +++ b/spec/workload/resilience/disk_fill_spec.cr @@ -14,13 +14,12 @@ describe "Resilience Disk Fill Chaos" do it "'disk_fill' A 'Good' CNF should not crash when disk fill occurs", tags: ["disk_fill"] do begin - ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml skip_wait_for_install") + ShellCmd.new_cnf_setup("cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml skip_wait_for_install") result = ShellCmd.run_testsuite("disk_fill verbose") result[:status].success?.should be_true (/(PASSED).*(disk_fill chaos test passed)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml wait_count=0") - result[:status].success?.should be_true + ShellCmd.new_cnf_cleanup() result = ShellCmd.run_testsuite("uninstall_litmus") result[:status].success?.should be_true end diff --git a/spec/workload/resilience/node_drain_spec.cr b/spec/workload/resilience/node_drain_spec.cr index 29d9fa5e1..f1dc03b6b 100644 --- a/spec/workload/resilience/node_drain_spec.cr +++ b/spec/workload/resilience/node_drain_spec.cr @@ -14,7 +14,7 @@ describe "Resilience Node Drain Chaos" do it "'node_drain' A 'Good' CNF should not crash when node drain occurs", tags: ["node_drain"] do begin - ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") + ShellCmd.new_cnf_setup("cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") result = ShellCmd.run_testsuite("node_drain verbose") result[:status].success?.should be_true if KubectlClient::Get.schedulable_nodes_list.size > 1 @@ -23,8 +23,7 @@ describe "Resilience Node Drain Chaos" do (/(SKIPPED).*(node_drain chaos test requires the cluster to have atleast two)/ =~ result[:output]).should_not be_nil end ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") - result[:status].success?.should be_true + ShellCmd.new_cnf_cleanup() result = ShellCmd.run_testsuite("uninstall_litmus") result[:status].success?.should be_true end diff --git a/spec/workload/resilience/pod_delete_spec.cr b/spec/workload/resilience/pod_delete_spec.cr index 80fdb13e2..8eae805de 100644 --- a/spec/workload/resilience/pod_delete_spec.cr +++ b/spec/workload/resilience/pod_delete_spec.cr @@ -14,13 +14,12 @@ describe "Resilience pod delete Chaos" do it "'pod_delete' A 'Good' CNF should not crash when pod delete occurs", tags: ["pod_delete"] do begin - ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") + ShellCmd.new_cnf_setup("cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") result = ShellCmd.run_testsuite("pod_delete verbose") result[:status].success?.should be_true (/(PASSED).*(pod_delete chaos test passed)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") - result[:status].success?.should be_true + ShellCmd.new_cnf_cleanup() result = ShellCmd.run_testsuite("uninstall_litmus") result[:status].success?.should be_true end diff --git a/spec/workload/resilience/pod_dns_error_spec.cr b/spec/workload/resilience/pod_dns_error_spec.cr index f7902182a..d08950c40 100644 --- a/spec/workload/resilience/pod_dns_error_spec.cr +++ b/spec/workload/resilience/pod_dns_error_spec.cr @@ -14,7 +14,7 @@ describe "Resilience pod dns error Chaos" do it "'pod_dns_error' A 'Good' CNF should not crash when pod dns error occurs", tags: ["pod_dns_error"] do begin - ShellCmd.cnf_setup("cnf-config=example-cnfs/envoy/cnf-testsuite.yml") + ShellCmd.new_cnf_setup("cnf-config=example-cnfs/envoy/cnf-testsuite.yml") result = ShellCmd.run_testsuite("pod_dns_error verbose") result[:status].success?.should be_true ((/(SKIPPED).*(pod_dns_error docker runtime not found)/) =~ result[:output] || @@ -24,8 +24,7 @@ describe "Resilience pod dns error Chaos" do # The ensure block will cleanup the CNF and the litmus installation. raise "Test failed with #{ex.message}" ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=example-cnfs/envoy/cnf-testsuite.yml") - result[:status].success?.should be_true + ShellCmd.new_cnf_cleanup() result = ShellCmd.run_testsuite("uninstall_litmus") result[:status].success?.should be_true end diff --git a/spec/workload/resilience/pod_io_stress_spec.cr b/spec/workload/resilience/pod_io_stress_spec.cr index f768409ce..522458e69 100644 --- a/spec/workload/resilience/pod_io_stress_spec.cr +++ b/spec/workload/resilience/pod_io_stress_spec.cr @@ -14,13 +14,12 @@ describe "Resilience pod delete Chaos" do it "'pod_io_stress' A 'Good' CNF should not crash when pod delete occurs", tags: ["pod_io_stress"] do begin - ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") + ShellCmd.new_cnf_setup("cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") result = ShellCmd.run_testsuite("pod_io_stress verbose") result[:status].success?.should be_true (/(PASSED).*(pod_io_stress chaos test passed)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") - result[:status].success?.should be_true + ShellCmd.new_cnf_cleanup() result = ShellCmd.run_testsuite("uninstall_litmus") result[:status].success?.should be_true end diff --git a/spec/workload/resilience/pod_memory_hog_spec.cr b/spec/workload/resilience/pod_memory_hog_spec.cr index 80ec62d0f..ab4f95326 100644 --- a/spec/workload/resilience/pod_memory_hog_spec.cr +++ b/spec/workload/resilience/pod_memory_hog_spec.cr @@ -14,13 +14,12 @@ describe "Resilience pod memory hog Chaos" do it "'pod_memory_hog' A 'Good' CNF should not crash when pod memory hog occurs", tags: ["pod_memory_hog"] do begin - ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") + ShellCmd.new_cnf_setup("cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") result = ShellCmd.run_testsuite("pod_memory_hog verbose") result[:status].success?.should be_true (/(PASSED).*(pod_memory_hog chaos test passed)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") - result[:status].success?.should be_true + ShellCmd.new_cnf_cleanup() result = ShellCmd.run_testsuite("uninstall_litmus") result[:status].success?.should be_true end diff --git a/spec/workload/resilience/pod_network_corruption_spec.cr b/spec/workload/resilience/pod_network_corruption_spec.cr index fbf8c23f3..53fa0984e 100644 --- a/spec/workload/resilience/pod_network_corruption_spec.cr +++ b/spec/workload/resilience/pod_network_corruption_spec.cr @@ -14,13 +14,12 @@ describe "Resilience Pod Network corruption Chaos" do it "'pod_network_corruption' A 'Good' CNF should not crash when network corruption occurs", tags: ["pod_network_corruption"] do begin - ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") + ShellCmd.new_cnf_setup("cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") result = ShellCmd.run_testsuite("pod_network_corruption verbose") result[:status].success?.should be_true (/(PASSED).*(pod_network_corruption chaos test passed)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") - result[:status].success?.should be_true + ShellCmd.new_cnf_cleanup() result = ShellCmd.run_testsuite("uninstall_litmus") result[:status].success?.should be_true end diff --git a/spec/workload/resilience/pod_network_duplication_spec.cr b/spec/workload/resilience/pod_network_duplication_spec.cr index de60e895a..3f8a77d02 100644 --- a/spec/workload/resilience/pod_network_duplication_spec.cr +++ b/spec/workload/resilience/pod_network_duplication_spec.cr @@ -14,13 +14,12 @@ describe "Resilience Pod Network duplication Chaos" do it "'pod_network_duplication' A 'Good' CNF should not crash when network duplication occurs", tags: ["pod_network_duplication"] do begin - ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") + ShellCmd.new_cnf_setup("cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") result = ShellCmd.run_testsuite("pod_network_duplication verbose") result[:status].success?.should be_true (/(PASSED).*(pod_network_duplication chaos test passed)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") - result[:status].success?.should be_true + ShellCmd.new_cnf_cleanup() result = ShellCmd.run_testsuite("uninstall_litmus") result[:status].success?.should be_true end diff --git a/spec/workload/resilience/pod_network_latency_spec.cr b/spec/workload/resilience/pod_network_latency_spec.cr index 24b455323..994f03520 100644 --- a/spec/workload/resilience/pod_network_latency_spec.cr +++ b/spec/workload/resilience/pod_network_latency_spec.cr @@ -14,13 +14,12 @@ describe "Resilience Pod Network Latency Chaos" do it "'pod_network_latency' A 'Good' CNF should not crash when network latency occurs", tags: ["pod_network_latency"] do begin - ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") + ShellCmd.new_cnf_setup("cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") result = ShellCmd.run_testsuite("pod_network_latency verbose") result[:status].success?.should be_true (/(PASSED).*(pod_network_latency chaos test passed)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") - result[:status].success?.should be_true + ShellCmd.new_cnf_cleanup() result = ShellCmd.run_testsuite("uninstall_litmus") result[:status].success?.should be_true end diff --git a/spec/workload/security_spec.cr b/spec/workload/security_spec.cr index 5c25eb71b..4fa5f6231 100644 --- a/spec/workload/security_spec.cr +++ b/spec/workload/security_spec.cr @@ -6,18 +6,17 @@ describe "Security" do it "'privileged_containers' should pass with a non-privileged cnf", tags: ["privileges"] do begin - ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample-statefulset-cnf/cnf-testsuite.yml") + ShellCmd.new_cnf_setup("cnf-config=sample-cnfs/sample-statefulset-cnf/cnf-testsuite.yml") result = ShellCmd.run_testsuite("privileged_containers verbose") result[:status].success?.should be_true (/No privileged containers/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample-statefulset-cnf/cnf-testsuite.yml") - Log.debug { result[:output] } + ShellCmd.new_cnf_cleanup() end end it "'privileged_containers' should fail on a non-whitelisted, privileged cnf", tags: ["privileges"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_privileged_cnf/cnf-testsuite.yml verbose skip_wait_for_install") + ShellCmd.new_cnf_setup("cnf-config=./sample-cnfs/sample_privileged_cnf/cnf-testsuite.yml verbose skip_wait_for_install") result = ShellCmd.run_testsuite("privileged_containers verbose") result[:status].success?.should be_true (/Found.*privileged containers.*/ =~ result[:output]).should_not be_nil @@ -28,7 +27,7 @@ describe "Security" do end it "'privileged_containers' should pass on a whitelisted, privileged cnf", tags: ["privileges"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_whitelisted_privileged_cnf/cnf-testsuite.yml verbose skip_wait_for_install") + ShellCmd.new_cnf_setup("cnf-config=./sample-cnfs/sample_whitelisted_privileged_cnf/cnf-testsuite.yml verbose skip_wait_for_install") result = ShellCmd.run_testsuite("privileged_containers verbose") result[:status].success?.should be_true (/Found.*privileged containers.*/ =~ result[:output]).should be_nil @@ -38,314 +37,314 @@ describe "Security" do end it "'privilege_escalation' should fail on a cnf that has escalated privileges", tags: ["privileges"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample-privilege-escalation/cnf-testsuite.yml") + ShellCmd.new_cnf_setup("cnf-config=./sample-cnfs/sample-privilege-escalation/cnf-testsuite.yml") result = ShellCmd.run_testsuite("privilege_escalation") result[:status].success?.should be_true (/(PASSED).*(No containers that allow privilege escalation were found)/ =~ result[:output]).should be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample-privilege-escalation/cnf-testsuite.yml") + ShellCmd.new_cnf_cleanup() end end it "'privilege_escalation' should pass on a cnf that does not have escalated privileges", tags: ["privileges"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample-nonroot-containers/cnf-testsuite.yml") + ShellCmd.new_cnf_setup("cnf-config=./sample-cnfs/sample-nonroot-containers/cnf-testsuite.yml") result = ShellCmd.run_testsuite("privilege_escalation") result[:status].success?.should be_true (/(PASSED).*(No containers that allow privilege escalation were found)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample-nonroot-containers/cnf-testsuite.yml") + ShellCmd.new_cnf_cleanup() end end it "'symlink_file_system' should pass on a cnf that does not allow a symlink attack", tags: ["capabilities"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample-privilege-escalation/cnf-testsuite.yml") + ShellCmd.new_cnf_setup("cnf-config=./sample-cnfs/sample-privilege-escalation/cnf-testsuite.yml") result = ShellCmd.run_testsuite("symlink_file_system") result[:status].success?.should be_true (/(PASSED).*(No containers allow a symlink attack)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample-privilege-escalation/cnf-testsuite.yml") + ShellCmd.new_cnf_cleanup() end end it "'insecure_capabilities' should pass on a cnf that does not have containers with insecure capabilities", tags: ["capabilities"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") + ShellCmd.new_cnf_setup("cnf-config=./sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") result = ShellCmd.run_testsuite("insecure_capabilities") result[:status].success?.should be_true (/(PASSED).*(Containers with insecure capabilities were not found)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") + ShellCmd.new_cnf_cleanup() end end it "'insecure_capabilities' should fail on a cnf that has containers with insecure capabilities", tags: ["security"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample-insecure-capabilities/cnf-testsuite.yml") + ShellCmd.new_cnf_setup("cnf-config=./sample-cnfs/sample-insecure-capabilities/cnf-testsuite.yml") result = ShellCmd.run_testsuite("insecure_capabilities") result[:status].success?.should be_true (/(PASSED).*(Containers with insecure capabilities were not found)/ =~ result[:output]).should be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample-insecure-capabilities/cnf-testsuite.yml") + ShellCmd.new_cnf_cleanup() end end it "'linux_hardening' should fail on a cnf that does not make use of security services", tags: ["capabilities"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample-coredns-cnf") + ShellCmd.new_cnf_setup("cnf-config=./sample-cnfs/sample-coredns-cnf") result = ShellCmd.run_testsuite("linux_hardening") result[:status].success?.should be_true (/(PASSED).*(Security services are being used to harden applications)/ =~ result[:output]).should be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample-coredns-cnf") + ShellCmd.new_cnf_cleanup() end end it "'application_credentials' should fail on a cnf that allows applications credentials in configuration files", tags: ["security"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample-appliciation-credentials/cnf-testsuite.yml") + ShellCmd.new_cnf_setup("cnf-config=./sample-cnfs/sample-appliciation-credentials/cnf-testsuite.yml") result = ShellCmd.run_testsuite("application_credentials") result[:status].success?.should be_true (/(FAILED).*(Found applications credentials in configuration files)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample-appliciation-credentials/cnf-testsuite.yml") + ShellCmd.new_cnf_cleanup() end end it "'host_network' should pass on a cnf that does not have a host network attached to pod", tags: ["security"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample-privilege-escalation/cnf-testsuite.yml") + ShellCmd.new_cnf_setup("cnf-config=./sample-cnfs/sample-privilege-escalation/cnf-testsuite.yml") result = ShellCmd.run_testsuite("host_network") result[:status].success?.should be_true (/(PASSED).*(No host network attached to pod)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample-privilege-escalation/cnf-testsuite.yml") + ShellCmd.new_cnf_cleanup() end end it "'service_account_mapping' should fail on a cnf that automatically maps the service account", tags: ["security"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample-service-accounts/cnf-testsuite.yml") + ShellCmd.new_cnf_setup("cnf-config=./sample-cnfs/sample-service-accounts/cnf-testsuite.yml") result = ShellCmd.run_testsuite("service_account_mapping") result[:status].success?.should be_true (/(FAILED).*(Service accounts automatically mapped)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample-service-accounts/cnf-testsuite.yml") + ShellCmd.new_cnf_cleanup() end end it "'cpu_limits' should pass on a cnf that has containers with cpu limits set", tags: ["security"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample-coredns-cnf") + ShellCmd.new_cnf_setup("cnf-config=./sample-cnfs/sample-coredns-cnf") result = ShellCmd.run_testsuite("cpu_limits") result[:status].success?.should be_true (/(PASSED).*(Containers have CPU limits set)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample-coredns-cnf") + ShellCmd.new_cnf_cleanup() end end it "'memory_limits' should pass on a cnf that has containers with memory limits set", tags: ["security"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample-coredns-cnf") + ShellCmd.new_cnf_setup("cnf-config=./sample-cnfs/sample-coredns-cnf") result = ShellCmd.run_testsuite("memory_limits") result[:status].success?.should be_true (/(PASSED).*(Containers have memory limits set)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample-coredns-cnf") + ShellCmd.new_cnf_cleanup() end end it "'ingress_egress_blocked' should fail on a cnf that has no ingress and egress traffic policy", tags: ["security"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample-coredns-cnf") + ShellCmd.new_cnf_setup("cnf-config=./sample-cnfs/sample-coredns-cnf") result = ShellCmd.run_testsuite("ingress_egress_blocked") result[:status].success?.should be_true (/(PASSED).*(Ingress and Egress traffic blocked on pods)/ =~ result[:output]).should be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample-coredns-cnf") + ShellCmd.new_cnf_cleanup() end end it "'host_pid_ipc_privileges' should pass on a cnf that does not have containers with host PID/IPC privileges", tags: ["capabilities"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample-coredns-cnf") + ShellCmd.new_cnf_setup("cnf-config=./sample-cnfs/sample-coredns-cnf") result = ShellCmd.run_testsuite("host_pid_ipc_privileges") result[:status].success?.should be_true (/(FAILED).*(Found containers with hostPID and hostIPC privileges)/ =~ result[:output]).should be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample-coredns-cnf") + ShellCmd.new_cnf_cleanup() end end it "'non_root_containers' should pass on a cnf that does not have containers running with root user or user with root group memberships", tags: ["security"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample-nonroot") + ShellCmd.new_cnf_setup("cnf-config=./sample-cnfs/sample-nonroot") result = ShellCmd.run_testsuite("non_root_containers") result[:status].success?.should be_true (/(FAILED).*(Found containers running with root user or user with root group membership)/ =~ result[:output]).should be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample-nonroot") + ShellCmd.new_cnf_cleanup() end end it "'non_root_containers' should fail on a cnf that has containers running with root user or user with root group memberships", tags: ["security"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample-coredns-cnf") + ShellCmd.new_cnf_setup("cnf-config=./sample-cnfs/sample-coredns-cnf") result = ShellCmd.run_testsuite("non_root_containers") result[:status].success?.should be_true (/(PASSED).*(Containers are running with non-root user with non-root group membership)/ =~ result[:output]).should be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample-coredns-cnf") + ShellCmd.new_cnf_cleanup() end end it "'immutable_file_systems' should fail when the cnf containers with mutable file systems", tags: ["security"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample-coredns-cnf") + ShellCmd.new_cnf_setup("cnf-config=./sample-cnfs/sample-coredns-cnf") result = ShellCmd.run_testsuite("immutable_file_systems") result[:status].success?.should be_true (/(PASSED).*(Containers have immutable file systems)/ =~ result[:output]).should be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample-coredns-cnf") + ShellCmd.new_cnf_cleanup() end end it "'immutable_file_systems' should pass when the cnf containers with immutable file systems", tags: ["security"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample-immutable-fs") + ShellCmd.new_cnf_setup("cnf-config=./sample-cnfs/sample-immutable-fs") result = ShellCmd.run_testsuite("immutable_file_systems") result[:status].success?.should be_true (/(PASSED).*(Containers have immutable file systems)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample-immutable-fs") + ShellCmd.new_cnf_cleanup() end end it "'hostpath_mounts' should pass when the cnf has no containers with hostPath mounts", tags: ["security"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample-coredns-cnf") + ShellCmd.new_cnf_setup("cnf-config=./sample-cnfs/sample-coredns-cnf") ClusterTools.uninstall result = ShellCmd.run_testsuite("hostpath_mounts") result[:status].success?.should be_true (/(FAILED).*(Found containers with hostPath mounts)/ =~ result[:output]).should be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample-coredns-cnf") + ShellCmd.new_cnf_cleanup() ClusterTools.install end end it "'hostpath_mounts' should fail when the cnf has containers with hostPath mounts", tags: ["security"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample-hostpath") + ShellCmd.new_cnf_setup("cnf-config=./sample-cnfs/sample-hostpath") ClusterTools.uninstall result = ShellCmd.run_testsuite("hostpath_mounts") result[:status].success?.should be_true (/(FAILED).*(Found containers with hostPath mounts)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample-hostpath") + ShellCmd.new_cnf_cleanup() ClusterTools.install end end it "'container_sock_mounts' should pass if a cnf has no pods that mount container engine socket", tags: ["container_sock_mounts"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml") + ShellCmd.new_cnf_setup("cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml") result = ShellCmd.run_testsuite("container_sock_mounts verbose") result[:status].success?.should be_true (/(PASSED).*(Container engine daemon sockets are not mounted as volumes)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml") + ShellCmd.new_cnf_cleanup() end end it "'container_sock_mounts' should fail if the CNF has pods with container engine sockets mounted", tags: ["container_sock_mounts"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_container_sock_mount/cnf-testsuite.yml") + ShellCmd.new_cnf_setup("cnf-config=./sample-cnfs/sample_container_sock_mount/cnf-testsuite.yml") result = ShellCmd.run_testsuite("container_sock_mounts verbose") result[:status].success?.should be_true (/(FAILED).*(Container engine daemon sockets are mounted as volumes)/ =~ result[:output]).should_not be_nil (/Unix socket is not allowed/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample_container_sock_mount/cnf-testsuite.yml") + ShellCmd.new_cnf_cleanup() end end it "'external_ips' should pass if a cnf has no services with external IPs", tags: ["external_ips"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml") + ShellCmd.new_cnf_setup("cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml") result = ShellCmd.run_testsuite("external_ips verbose") result[:status].success?.should be_true (/(PASSED).*(Services are not using external IPs)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample_coredns/cnf-testsuite.yml") + ShellCmd.new_cnf_cleanup() end end it "'external_ips' should fail if a cnf has services with external IPs", tags: ["external_ips"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_external_ips/cnf-testsuite.yml") + ShellCmd.new_cnf_setup("cnf-config=./sample-cnfs/sample_external_ips/cnf-testsuite.yml") result = ShellCmd.run_testsuite("external_ips verbose") result[:status].success?.should be_true (/(FAILED).*(Services are using external IPs)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample_external_ips/cnf-testsuite.yml") + ShellCmd.new_cnf_cleanup() end end it "'selinux_options' should fail if containers have custom selinux options that can be used for privilege escalations", tags: ["selinux_options"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_latest_tag") + ShellCmd.new_cnf_setup("cnf-config=./sample-cnfs/sample_latest_tag") result = ShellCmd.run_testsuite("selinux_options verbose") result[:status].success?.should be_true (/(FAILED).*(Pods are using custom SELinux options that can be used for privilege escalations)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample_latest_tag") + ShellCmd.new_cnf_cleanup() end end it "'selinux_options' should be skipped if containers do not use custom selinux options", tags: ["selinux_options"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_nonroot") + ShellCmd.new_cnf_setup("cnf-config=./sample-cnfs/sample_nonroot") result = ShellCmd.run_testsuite("selinux_options verbose") result[:status].success?.should be_true (/(N\/A).*(Pods are not using SELinux)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample_nonroot") + ShellCmd.new_cnf_cleanup() end end it "'selinux_options' should pass if containers do not have custom selinux options that can be used for privilege escalations", tags: ["selinux_options"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_valid_selinux_options") + ShellCmd.new_cnf_setup("cnf-config=./sample-cnfs/sample_valid_selinux_options") result = ShellCmd.run_testsuite("selinux_options verbose") result[:status].success?.should be_true (/(PASSED).*(Pods are not using custom SELinux options that can be used for privilege escalations)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample_valid_selinux_options") + ShellCmd.new_cnf_cleanup() end end it "'sysctls' should fail if Pods have restricted sysctls values", tags: ["sysctls"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_sysctls") + ShellCmd.new_cnf_setup("cnf-config=./sample-cnfs/sample_sysctls") result = ShellCmd.run_testsuite("sysctls verbose") result[:status].success?.should be_true (/(FAILED).*(Restricted values for are being used for sysctls)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample_sysctls") + ShellCmd.new_cnf_cleanup() end end it "'sysctls' should pass if Pods have allowed sysctls values", tags: ["sysctls"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_nonroot") + ShellCmd.new_cnf_setup("cnf-config=./sample-cnfs/sample_nonroot") result = ShellCmd.run_testsuite("sysctls verbose") result[:status].success?.should be_true (/(PASSED).*(No restricted values found for sysctls)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample_nonroot") + ShellCmd.new_cnf_cleanup() end end end diff --git a/spec/workload/state_spec.cr b/spec/workload/state_spec.cr index bf0e580f3..57aae4a41 100644 --- a/spec/workload/state_spec.cr +++ b/spec/workload/state_spec.cr @@ -14,12 +14,11 @@ describe "State" do it "'elastic_volumes' should fail if the cnf does not use volumes that are elastic volume", tags: ["elastic_volume"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample-elastic-volume/cnf-testsuite.yml", cmd_prefix: "LOG_LEVEL=info") + ShellCmd.new_cnf_setup("cnf-config=./sample-cnfs/sample-elastic-volume/cnf-testsuite.yml", cmd_prefix: "LOG_LEVEL=info") result = ShellCmd.run_testsuite("elastic_volumes verbose", cmd_prefix: "LOG_LEVEL=info") (/(PASSED).*(All used volumes are elastic)/ =~ result[:output]).should be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample-elastic-volume/cnf-testsuite.yml") - result[:status].success?.should be_true + ShellCmd.new_cnf_cleanup() end end @@ -27,12 +26,11 @@ describe "State" do # This CNF does not use any volumes except the ones that Kubernetes might mount by default (like the service account token) it "'elastic_volumes' should fail if the cnf does not use any elastic volumes", tags: ["elastic_volume"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample_nonroot", cmd_prefix: "LOG_LEVEL=info") + ShellCmd.new_cnf_setup("cnf-config=./sample-cnfs/sample_nonroot", cmd_prefix: "LOG_LEVEL=info") result = ShellCmd.run_testsuite("elastic_volumes verbose", cmd_prefix: "LOG_LEVEL=info") (/FAILED/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample_nonroot") - result[:status].success?.should be_true + ShellCmd.new_cnf_cleanup() end end @@ -40,24 +38,23 @@ describe "State" do begin Log.info { "Installing Mysql " } # todo make helm directories work with parameters - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample-mysql/cnf-testsuite.yml") + ShellCmd.new_cnf_setup("cnf-config=./sample-cnfs/sample-mysql/cnf-testsuite.yml") result = ShellCmd.run_testsuite("database_persistence", cmd_prefix: "LOG_LEVEL=info") (/(PASSED).*(CNF uses database with cloud-native persistence)/ =~ result[:output]).should_not be_nil ensure #todo fix cleanup for helm directory with parameters - ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample-mysql/cnf-testsuite.yml") + ShellCmd.new_cnf_cleanup() ShellCmd.run("kubectl delete pvc data-mysql-0", "delete_pvc") end end it "'elastic_volumes' should fail if the cnf doesn't use an elastic volume", tags: ["elastic_volume"] do begin - ShellCmd.cnf_setup("cnf-config=./sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml", cmd_prefix: "LOG_LEVEL=info") + ShellCmd.new_cnf_setup("cnf-config=./sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml", cmd_prefix: "LOG_LEVEL=info") result = ShellCmd.run_testsuite("elastic_volumes verbose", cmd_prefix: "LOG_LEVEL=info") (/(FAILED).*(Some of the used volumes are not elastic)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=./sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml") - result[:status].success?.should be_true + ShellCmd.new_cnf_cleanup() end end @@ -66,24 +63,22 @@ describe "State" do # update the helm parameter with a schedulable node for the pv chart schedulable_nodes = KubectlClient::Get.schedulable_nodes update_yml("sample-cnfs/sample-local-storage/worker-node-value.yml", "worker_node", "#{schedulable_nodes[0]}") - ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample-local-storage/cnf-testsuite.yml verbose") + ShellCmd.new_cnf_setup("cnf-config=sample-cnfs/sample-local-storage/cnf-testsuite.yml verbose") result = ShellCmd.run_testsuite("no_local_volume_configuration verbose") (/(FAILED).*(local storage configuration volumes found)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample-local-storage/cnf-testsuite.yml deploy_with_chart=false") + ShellCmd.new_cnf_cleanup() update_yml("sample-cnfs/sample-local-storage/worker-node-value.yml", "worker_node", "") - result[:status].success?.should be_true end end it "'no_local_volume_configuration' should pass if local storage configuration is not found", tags: ["no_local_volume_configuration"] do begin - ShellCmd.cnf_setup("cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml verbose") + ShellCmd.new_cnf_setup("cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml verbose") result = ShellCmd.run_testsuite("no_local_volume_configuration verbose") (/(PASSED).*(local storage configuration volumes not found)/ =~ result[:output]).should_not be_nil ensure - result = ShellCmd.run_testsuite("cnf_cleanup cnf-config=sample-cnfs/sample-coredns-cnf/cnf-testsuite.yml deploy_with_chart=false") - result[:status].success?.should be_true + ShellCmd.new_cnf_cleanup() end end end