[WIP] Split E2E Drone pipeline into 2 parallel chunks

.drone.yml

@@ -604,8 +604,27 @@ steps:
   - name: docker
     path: /var/run/docker.sock
 
-- name: test-e2e
+# For better parallelism, we run the E2E tests in a chunked directed acyclic graph
+# First we cleanup any VMs that are older than 1h. Then, tests are split into 2 groups: 
+# - validatecluster, token, and upgradecluster
+# - secretsencryption and splitserver
+
+- name: cleanup-vms
+  image: test-e2e
+  pull: never
+  depends_on:
+    - build-e2e-image
+  commands:
+  # Cleanup VMs that are older than 1h. Happens if a previous test panics or is canceled
+  - tests/e2e/scripts/cleanup_vms.sh
+  volumes:
+  - name: libvirt
+    path: /var/run/libvirt/
+
+- name: test-block1
   image: test-e2e
+  depends_on:
+    - cleanup-vms
   pull: never
   resources:
     cpu: 6000
@@ -617,24 +636,17 @@ steps:
   - mkdir -p dist/artifacts
   - cp /tmp/artifacts/* dist/artifacts/
   - docker stop registry && docker rm registry
-  # Cleanup VMs that are older than 2h. Happens if a previous test panics or is canceled
-  - tests/e2e/scripts/cleanup_vms.sh
   - docker run -d -p 5000:5000 -e REGISTRY_PROXY_REMOTEURL=https://registry-1.docker.io --name registry registry:2
   - |
     cd tests/e2e/validatecluster
     vagrant destroy -f
     go test -v -timeout=45m ./validatecluster_test.go -ci -local
     cp ./coverage.out /tmp/artifacts/validate-coverage.out
   - |
-    cd ../secretsencryption
+    cd ../token
     vagrant destroy -f
-    go test -v -timeout=30m ./secretsencryption_test.go -ci -local
-    cp ./coverage.out /tmp/artifacts/se-coverage.out
-  - |
-    cd ../splitserver
-    vagrant destroy -f
-    go test -v -timeout=30m ./splitserver_test.go -ci -local
-    cp ./coverage.out /tmp/artifacts/split-coverage.out
+    go test -v -timeout=30m ./token_test.go -ci -local
+    cp ./coverage.out /tmp/artifacts/token-coverage.out
   - |
     if [ "$DRONE_BUILD_EVENT" = "pull_request" ]; then
       cd ../upgradecluster
@@ -652,6 +664,7 @@ steps:
       E2E_RELEASE_CHANNEL=$UPGRADE_CHANNEL go test -v -timeout=45m ./upgradecluster_test.go  -ci -local -ginkgo.v
       cp ./coverage.out /tmp/artifacts/upgrade-coverage.out
     fi
+
   - docker stop registry && docker rm registry
 
   volumes:
@@ -662,16 +675,50 @@ steps:
   - name: cache
     path: /tmp/artifacts
 
+- name: test-block2
+  image: test-e2e
+  depends_on:
+    - cleanup-vms
+  pull: never
+  resources:
+    cpu: 6000
+    memory: 10Gi
+  environment:
+    E2E_GOCOVER: 'true'
+  commands:
+  - mkdir -p dist/artifacts
+  - cp /tmp/artifacts/* dist/artifacts/
+  - |
+    cd tests/e2e/secretsencryption
+    vagrant destroy -f
+    go test -v -timeout=30m ./secretsencryption_test.go -ci -local
+    cp ./coverage.out /tmp/artifacts/se-coverage.out
+  - |
+    cd ../splitserver
+    vagrant destroy -f
+    go test -v -timeout=30m ./splitserver_test.go -ci -local
+    cp ./coverage.out /tmp/artifacts/split-coverage.out
+
+  volumes:
+  - name: libvirt
+    path: /var/run/libvirt/
+  - name: cache
+    path: /tmp/artifacts
+
 - name: upload to codecov
   image: robertstettner/drone-codecov
+  depends_on:
+    - test-block1
+    - test-block2
   settings:
     token: 
       from_secret: codecov_token
     files:
       - /tmp/artifacts/validate-coverage.out
+      - /tmp/artifacts/upgrade-coverage.out
       - /tmp/artifacts/se-coverage.out
       - /tmp/artifacts/split-coverage.out
-      - /tmp/artifacts/upgrade-coverage.out
+      - /tmp/artifacts/token-coverage.out
     flags: 
       - e2etests
   when:

tests/e2e/scripts/cleanup_vms.sh

@@ -15,7 +15,7 @@ for vm in $vms; do
     if [[ $vm =~ $time_regex ]]; then
         vm_time="${BASH_REMATCH[1]}"
         age=$((current_time - vm_time))
-        if [ $age -gt 7200 ]; then
+        if [ $age -gt 3600 ]; then
             virsh destroy $vm
             virsh undefine $vm --remove-all-storage
         fi
@@ -28,7 +28,7 @@ for vm in $vms; do
     if [[ $vm =~ $time_regex ]]; then
         vm_time="${BASH_REMATCH[1]}"
         age=$((current_time - vm_time))
-        if [ $age -gt 7200 ]; then
+        if [ $age -gt 3600 ]; then
             virsh undefine $vm  --remove-all-storage
         fi
     fi

tests/e2e/secretsencryption/secretsencryption_test.go

@@ -222,7 +222,7 @@ var _ = AfterEach(func() {
 
 var _ = AfterSuite(func() {
 	if failed {
-		AddReportEntry("journald-logs", e2e.TailJournalLogs(1000, serverNodeNames))
+		AddReportEntry("journald-logs", e2e.TailJournalLogs(500, serverNodeNames))
 	} else {
 		Expect(e2e.GetCoverageReport(serverNodeNames)).To(Succeed())
 	}

tests/e2e/snapshotrestore/snapshotrestore_test.go

@@ -307,7 +307,7 @@ var _ = AfterEach(func() {
 
 var _ = AfterSuite(func() {
 	if failed {
-		AddReportEntry("journald-logs", e2e.TailJournalLogs(1000, append(serverNodeNames, agentNodeNames...)))
+		AddReportEntry("journald-logs", e2e.TailJournalLogs(500, append(serverNodeNames, agentNodeNames...)))
 	} else {
 		Expect(e2e.GetCoverageReport(append(serverNodeNames, agentNodeNames...))).To(Succeed())
 	}

tests/e2e/splitserver/splitserver_test.go

@@ -286,7 +286,7 @@ var _ = AfterSuite(func() {
 	allNodes := append(cpNodeNames, etcdNodeNames...)
 	allNodes = append(allNodes, agentNodeNames...)
 	if failed {
-		AddReportEntry("journald-logs", e2e.TailJournalLogs(1000, allNodes))
+		AddReportEntry("journald-logs", e2e.TailJournalLogs(500, allNodes))
 	} else {
 		Expect(e2e.GetCoverageReport(allNodes)).To(Succeed())
 	}

tests/e2e/token/token_test.go

@@ -203,7 +203,7 @@ var _ = AfterEach(func() {
 
 var _ = AfterSuite(func() {
 	if failed {
-		AddReportEntry("journald-logs", e2e.TailJournalLogs(1000, append(serverNodeNames, agentNodeNames...)))
+		AddReportEntry("journald-logs", e2e.TailJournalLogs(500, append(serverNodeNames, agentNodeNames...)))
 	} else {
 		Expect(e2e.GetCoverageReport(append(serverNodeNames, agentNodeNames...))).To(Succeed())
 	}

tests/e2e/upgradecluster/upgradecluster_test.go

@@ -385,7 +385,7 @@ var _ = AfterEach(func() {
 
 var _ = AfterSuite(func() {
 	if failed {
-		AddReportEntry("journald-logs", e2e.TailJournalLogs(1000, append(serverNodeNames, agentNodeNames...)))
+		AddReportEntry("journald-logs", e2e.TailJournalLogs(500, append(serverNodeNames, agentNodeNames...)))
 	} else {
 		Expect(e2e.GetCoverageReport(append(serverNodeNames, agentNodeNames...))).To(Succeed())
 	}

tests/e2e/validatecluster/validatecluster_test.go

@@ -382,7 +382,7 @@ var _ = AfterEach(func() {
 
 var _ = AfterSuite(func() {
 	if failed {
-		AddReportEntry("journald-logs", e2e.TailJournalLogs(1000, append(serverNodeNames, agentNodeNames...)))
+		AddReportEntry("journald-logs", e2e.TailJournalLogs(500, append(serverNodeNames, agentNodeNames...)))
 	} else {
 		Expect(e2e.GetCoverageReport(append(serverNodeNames, agentNodeNames...))).To(Succeed())
 	}