Cherry Pick PR 8634 to 4.13 release branch

Cherry Pick PR red-hat-storage#8634 to 4.13 release branch Signed-off-by: Ramakrishnan Periyasamy <[email protected]>
ramkiperiy · Oct 9, 2023 · de7bdeb · de7bdeb
1 parent 9580be0
commit de7bdeb
Showing 1 changed file with 55 additions and 28 deletions.
diff --git a/tests/e2e/scale/test_pv_scale_and_respin_ceph_pods.py b/tests/e2e/scale/test_pv_scale_and_respin_ceph_pods.py
@@ -12,8 +12,13 @@
 from ocs_ci.ocs import constants, scale_lib
 from ocs_ci.helpers import helpers, disruption_helpers
 from ocs_ci.ocs.resources.objectconfigfile import ObjectConfFile
+from ocs_ci.framework.pytest_customization.marks import orange_squad
 from ocs_ci.framework.testlib import scale, E2ETest, ignore_leftovers
 from ocs_ci.framework.pytest_customization.marks import skipif_external_mode
+from ocs_ci.ocs.exceptions import (
+    PVCNotCreated,
+    PodNotCreated,
+)
 
 log = logging.getLogger(__name__)
 
@@ -24,6 +29,7 @@ class BasePvcCreateRespinCephPods(E2ETest):
     """
 
     kube_job_pvc_list, kube_job_pod_list = ([], [])
+    pod_count = 0
 
     def create_pvc_pod(self, obj_name, number_of_pvc, size):
         """
@@ -99,25 +105,30 @@ def create_pvc_pod(self, obj_name, number_of_pvc, size):
         lcl[f"rbd-rwx-pvc-{obj_name}"].create(namespace=self.namespace)
         lcl[f"cephfs-pvc-{obj_name}"].create(namespace=self.namespace)
 
-        # Check all the PVC reached Bound state
-        rbd_pvc_name = scale_lib.check_all_pvc_reached_bound_state_in_kube_job(
-            kube_job_obj=lcl[f"rbd-pvc-{obj_name}"],
-            namespace=self.namespace,
-            no_of_pvc=int(number_of_pvc),
-            timeout=60,
-        )
-        rbd_rwx_pvc_name = scale_lib.check_all_pvc_reached_bound_state_in_kube_job(
-            kube_job_obj=lcl[f"rbd-rwx-pvc-{obj_name}"],
-            namespace=self.namespace,
-            no_of_pvc=int(number_of_pvc),
-            timeout=60,
-        )
-        fs_pvc_name = scale_lib.check_all_pvc_reached_bound_state_in_kube_job(
-            kube_job_obj=lcl[f"cephfs-pvc-{obj_name}"],
-            namespace=self.namespace,
-            no_of_pvc=int(number_of_pvc * 2),
-            timeout=60,
-        )
+        try:
+            # Check all the PVC reached Bound state
+            rbd_pvc_name = scale_lib.check_all_pvc_reached_bound_state_in_kube_job(
+                kube_job_obj=lcl[f"rbd-pvc-{obj_name}"],
+                namespace=self.namespace,
+                no_of_pvc=int(number_of_pvc),
+                timeout=60,
+            )
+            rbd_rwx_pvc_name = scale_lib.check_all_pvc_reached_bound_state_in_kube_job(
+                kube_job_obj=lcl[f"rbd-rwx-pvc-{obj_name}"],
+                namespace=self.namespace,
+                no_of_pvc=int(number_of_pvc),
+                timeout=60,
+            )
+            fs_pvc_name = scale_lib.check_all_pvc_reached_bound_state_in_kube_job(
+                kube_job_obj=lcl[f"cephfs-pvc-{obj_name}"],
+                namespace=self.namespace,
+                no_of_pvc=int(number_of_pvc * 2),
+                timeout=60,
+            )
+
+        except Exception as ex:
+            log.error(f"All or some of the PVCs didn't Bound in expected time {ex}")
+            raise PVCNotCreated("PVC not up in expected time.")
 
         # Construct pod yaml file for kube_job
         pod_data_list = list()
@@ -153,13 +164,23 @@ def create_pvc_pod(self, obj_name, number_of_pvc, size):
         )
         lcl[f"pod-{obj_name}"].create(namespace=self.namespace)
 
-        # Check all the POD reached Running state
-        pod_running_list = scale_lib.check_all_pod_reached_running_state_in_kube_job(
-            kube_job_obj=lcl[f"pod-{obj_name}"],
-            namespace=self.namespace,
-            no_of_pod=len(pod_data_list),
-            timeout=90,
-        )
+        try:
+            # Check all the POD reached Running state
+            pod_running_list = (
+                scale_lib.check_all_pod_reached_running_state_in_kube_job(
+                kube_job_obj=lcl[f"pod-{obj_name}"],
+                namespace=self.namespace,
+                no_of_pod=len(pod_data_list),
+                timeout=90,
+                )
+            )
+
+        except Exception as ex:
+            log.error(
+                f"All or some of PODs didn't reach Running state in expected time {ex}"
+            )
+            raise PodNotCreated("Pod not running in expected time")
+
         self.pod_count = self.pod_count + len(pod_data_list)
 
         # Update list with all the kube_job object created, list will be used in cleanup
@@ -207,6 +228,7 @@ def cleanup(self):
 
 
 @scale
+@orange_squad
 @ignore_leftovers
 @skipif_external_mode
 @pytest.mark.parametrize(
@@ -237,8 +259,6 @@ class TestPVSTOcsCreatePVCsAndRespinCephPods(BasePvcCreateRespinCephPods):
     Class for PV scale Create Cluster with 1000 PVC, then Respin ceph pods parallel
     """
 
-    pod_count = 0
-
     @pytest.fixture()
     def setup_fixture(self, teardown_factory, request):
         proj_obj = helpers.create_project()
@@ -267,6 +287,13 @@ def test_pv_scale_out_create_pvcs_and_respin_ceph_pods(
             if scale_pod_count <= self.pod_count:
                 log.info(f"Create {scale_pod_count} pvc and pods")
                 break
+            elif iteration == 5:
+                # Section to terminate the while loop in-case environment failures.
+                log.error(
+                    "Breaking while loop since expected iteration already completed. "
+                    "There should be an issue with PVC or POD creation."
+                )
+                exit("Exiting code due to PVC POD creation issues")
             else:
                 iteration += 1
                 thread1 = threading.Thread(