From 4bc85f83df2851af5a6372894a4c0e916d9a2c00 Mon Sep 17 00:00:00 2001 From: Derek Nola Date: Mon, 9 Dec 2024 13:38:56 -0800 Subject: [PATCH 1/2] Split E2E Drone pipeline into 2 parallel chunks Signed-off-by: Derek Nola --- .drone.yml | 71 ++++++++++++++++++++++++++------ tests/e2e/scripts/cleanup_vms.sh | 4 +- 2 files changed, 61 insertions(+), 14 deletions(-) diff --git a/.drone.yml b/.drone.yml index 54482bcaafc6..dbaaaa7766a5 100644 --- a/.drone.yml +++ b/.drone.yml @@ -604,8 +604,27 @@ steps: - name: docker path: /var/run/docker.sock -- name: test-e2e +# For better parallelism, we run the E2E tests in a chunked directed acyclic graph +# First we cleanup any VMs that are older than 1h. Then, tests are split into 2 groups: +# - validatecluster, token, and upgradecluster +# - secretsencryption and splitserver + +- name: cleanup-vms + image: test-e2e + pull: never + depends_on: + - build-e2e-image + commands: + # Cleanup VMs that are older than 1h. Happens if a previous test panics or is canceled + - tests/e2e/scripts/cleanup_vms.sh + volumes: + - name: libvirt + path: /var/run/libvirt/ + +- name: test-block1 image: test-e2e + depends_on: + - cleanup-vms pull: never resources: cpu: 6000 @@ -617,8 +636,6 @@ steps: - mkdir -p dist/artifacts - cp /tmp/artifacts/* dist/artifacts/ - docker stop registry && docker rm registry - # Cleanup VMs that are older than 2h. Happens if a previous test panics or is canceled - - tests/e2e/scripts/cleanup_vms.sh - docker run -d -p 5000:5000 -e REGISTRY_PROXY_REMOTEURL=https://registry-1.docker.io --name registry registry:2 - | cd tests/e2e/validatecluster @@ -626,15 +643,10 @@ steps: go test -v -timeout=45m ./validatecluster_test.go -ci -local cp ./coverage.out /tmp/artifacts/validate-coverage.out - | - cd ../secretsencryption + cd ../token vagrant destroy -f - go test -v -timeout=30m ./secretsencryption_test.go -ci -local - cp ./coverage.out /tmp/artifacts/se-coverage.out - - | - cd ../splitserver - vagrant destroy -f - go test -v -timeout=30m ./splitserver_test.go -ci -local - cp ./coverage.out /tmp/artifacts/split-coverage.out + go test -v -timeout=30m ./token_test.go -ci -local + cp ./coverage.out /tmp/artifacts/token-coverage.out - | if [ "$DRONE_BUILD_EVENT" = "pull_request" ]; then cd ../upgradecluster @@ -652,6 +664,7 @@ steps: E2E_RELEASE_CHANNEL=$UPGRADE_CHANNEL go test -v -timeout=45m ./upgradecluster_test.go -ci -local -ginkgo.v cp ./coverage.out /tmp/artifacts/upgrade-coverage.out fi + - docker stop registry && docker rm registry volumes: @@ -662,16 +675,50 @@ steps: - name: cache path: /tmp/artifacts +- name: test-block2 + image: test-e2e + depends_on: + - cleanup-vms + pull: never + resources: + cpu: 6000 + memory: 10Gi + environment: + E2E_GOCOVER: 'true' + commands: + - mkdir -p dist/artifacts + - cp /tmp/artifacts/* dist/artifacts/ + - | + cd tests/e2e/secretsencryption + vagrant destroy -f + go test -v -timeout=30m ./secretsencryption_test.go -ci -local + cp ./coverage.out /tmp/artifacts/se-coverage.out + - | + cd ../splitserver + vagrant destroy -f + go test -v -timeout=30m ./splitserver_test.go -ci -local + cp ./coverage.out /tmp/artifacts/split-coverage.out + + volumes: + - name: libvirt + path: /var/run/libvirt/ + - name: cache + path: /tmp/artifacts + - name: upload to codecov image: robertstettner/drone-codecov + depends_on: + - test-block1 + - test-block2 settings: token: from_secret: codecov_token files: - /tmp/artifacts/validate-coverage.out + - /tmp/artifacts/upgrade-coverage.out - /tmp/artifacts/se-coverage.out - /tmp/artifacts/split-coverage.out - - /tmp/artifacts/upgrade-coverage.out + - /tmp/artifacts/token-coverage.out flags: - e2etests when: diff --git a/tests/e2e/scripts/cleanup_vms.sh b/tests/e2e/scripts/cleanup_vms.sh index f5c1b909c2cf..cf72d2d83102 100755 --- a/tests/e2e/scripts/cleanup_vms.sh +++ b/tests/e2e/scripts/cleanup_vms.sh @@ -15,7 +15,7 @@ for vm in $vms; do if [[ $vm =~ $time_regex ]]; then vm_time="${BASH_REMATCH[1]}" age=$((current_time - vm_time)) - if [ $age -gt 7200 ]; then + if [ $age -gt 3600 ]; then virsh destroy $vm virsh undefine $vm --remove-all-storage fi @@ -28,7 +28,7 @@ for vm in $vms; do if [[ $vm =~ $time_regex ]]; then vm_time="${BASH_REMATCH[1]}" age=$((current_time - vm_time)) - if [ $age -gt 7200 ]; then + if [ $age -gt 3600 ]; then virsh undefine $vm --remove-all-storage fi fi From f6aad7b83efd4c1235111fd5efd859fdb50fe444 Mon Sep 17 00:00:00 2001 From: Derek Nola Date: Tue, 10 Dec 2024 10:17:56 -0800 Subject: [PATCH 2/2] Drop E2E tail lines to 500 for Drone CI Signed-off-by: Derek Nola --- tests/e2e/secretsencryption/secretsencryption_test.go | 2 +- tests/e2e/snapshotrestore/snapshotrestore_test.go | 2 +- tests/e2e/splitserver/splitserver_test.go | 2 +- tests/e2e/token/token_test.go | 2 +- tests/e2e/upgradecluster/upgradecluster_test.go | 2 +- tests/e2e/validatecluster/validatecluster_test.go | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/e2e/secretsencryption/secretsencryption_test.go b/tests/e2e/secretsencryption/secretsencryption_test.go index 763e2f0ba381..0d60e92bc8d6 100644 --- a/tests/e2e/secretsencryption/secretsencryption_test.go +++ b/tests/e2e/secretsencryption/secretsencryption_test.go @@ -222,7 +222,7 @@ var _ = AfterEach(func() { var _ = AfterSuite(func() { if failed { - AddReportEntry("journald-logs", e2e.TailJournalLogs(1000, serverNodeNames)) + AddReportEntry("journald-logs", e2e.TailJournalLogs(500, serverNodeNames)) } else { Expect(e2e.GetCoverageReport(serverNodeNames)).To(Succeed()) } diff --git a/tests/e2e/snapshotrestore/snapshotrestore_test.go b/tests/e2e/snapshotrestore/snapshotrestore_test.go index 0f2fb8b9bd26..f07334c7beb1 100644 --- a/tests/e2e/snapshotrestore/snapshotrestore_test.go +++ b/tests/e2e/snapshotrestore/snapshotrestore_test.go @@ -307,7 +307,7 @@ var _ = AfterEach(func() { var _ = AfterSuite(func() { if failed { - AddReportEntry("journald-logs", e2e.TailJournalLogs(1000, append(serverNodeNames, agentNodeNames...))) + AddReportEntry("journald-logs", e2e.TailJournalLogs(500, append(serverNodeNames, agentNodeNames...))) } else { Expect(e2e.GetCoverageReport(append(serverNodeNames, agentNodeNames...))).To(Succeed()) } diff --git a/tests/e2e/splitserver/splitserver_test.go b/tests/e2e/splitserver/splitserver_test.go index 642dbc1592e3..cfe4c149841b 100644 --- a/tests/e2e/splitserver/splitserver_test.go +++ b/tests/e2e/splitserver/splitserver_test.go @@ -286,7 +286,7 @@ var _ = AfterSuite(func() { allNodes := append(cpNodeNames, etcdNodeNames...) allNodes = append(allNodes, agentNodeNames...) if failed { - AddReportEntry("journald-logs", e2e.TailJournalLogs(1000, allNodes)) + AddReportEntry("journald-logs", e2e.TailJournalLogs(500, allNodes)) } else { Expect(e2e.GetCoverageReport(allNodes)).To(Succeed()) } diff --git a/tests/e2e/token/token_test.go b/tests/e2e/token/token_test.go index 3b3c011d6ae7..f162923eb384 100644 --- a/tests/e2e/token/token_test.go +++ b/tests/e2e/token/token_test.go @@ -203,7 +203,7 @@ var _ = AfterEach(func() { var _ = AfterSuite(func() { if failed { - AddReportEntry("journald-logs", e2e.TailJournalLogs(1000, append(serverNodeNames, agentNodeNames...))) + AddReportEntry("journald-logs", e2e.TailJournalLogs(500, append(serverNodeNames, agentNodeNames...))) } else { Expect(e2e.GetCoverageReport(append(serverNodeNames, agentNodeNames...))).To(Succeed()) } diff --git a/tests/e2e/upgradecluster/upgradecluster_test.go b/tests/e2e/upgradecluster/upgradecluster_test.go index fab93a6bbd89..d343441d9e7a 100644 --- a/tests/e2e/upgradecluster/upgradecluster_test.go +++ b/tests/e2e/upgradecluster/upgradecluster_test.go @@ -385,7 +385,7 @@ var _ = AfterEach(func() { var _ = AfterSuite(func() { if failed { - AddReportEntry("journald-logs", e2e.TailJournalLogs(1000, append(serverNodeNames, agentNodeNames...))) + AddReportEntry("journald-logs", e2e.TailJournalLogs(500, append(serverNodeNames, agentNodeNames...))) } else { Expect(e2e.GetCoverageReport(append(serverNodeNames, agentNodeNames...))).To(Succeed()) } diff --git a/tests/e2e/validatecluster/validatecluster_test.go b/tests/e2e/validatecluster/validatecluster_test.go index 2c4807cce98d..d962e8bf9224 100644 --- a/tests/e2e/validatecluster/validatecluster_test.go +++ b/tests/e2e/validatecluster/validatecluster_test.go @@ -382,7 +382,7 @@ var _ = AfterEach(func() { var _ = AfterSuite(func() { if failed { - AddReportEntry("journald-logs", e2e.TailJournalLogs(1000, append(serverNodeNames, agentNodeNames...))) + AddReportEntry("journald-logs", e2e.TailJournalLogs(500, append(serverNodeNames, agentNodeNames...))) } else { Expect(e2e.GetCoverageReport(append(serverNodeNames, agentNodeNames...))).To(Succeed()) }