From b30e211a86b5d165a3105be72e14990500be0916 Mon Sep 17 00:00:00 2001 From: Ramakrishnan Periyasamy Date: Mon, 9 Oct 2023 18:43:20 +0530 Subject: [PATCH] Cherry Pick PR-8634 for release 4.12 (#8650) * Cherry Pick PR-8634 for release 4.12 Cherry Pick PR https://github.com/red-hat-storage/ocs-ci/pull/8634 for 4.12 release branch Signed-off-by: Ramakrishnan Periyasamy --- .../test_pv_scale_and_respin_ceph_pods.py | 83 ++++++++++++------- 1 file changed, 55 insertions(+), 28 deletions(-) diff --git a/tests/e2e/scale/test_pv_scale_and_respin_ceph_pods.py b/tests/e2e/scale/test_pv_scale_and_respin_ceph_pods.py index e4ed588d38e..256acc1f9ea 100644 --- a/tests/e2e/scale/test_pv_scale_and_respin_ceph_pods.py +++ b/tests/e2e/scale/test_pv_scale_and_respin_ceph_pods.py @@ -12,8 +12,13 @@ from ocs_ci.ocs import constants, scale_lib from ocs_ci.helpers import helpers, disruption_helpers from ocs_ci.ocs.resources.objectconfigfile import ObjectConfFile +from ocs_ci.framework.pytest_customization.marks import orange_squad from ocs_ci.framework.testlib import scale, E2ETest, ignore_leftovers from ocs_ci.framework.pytest_customization.marks import skipif_external_mode +from ocs_ci.ocs.exceptions import ( + PVCNotCreated, + PodNotCreated, +) log = logging.getLogger(__name__) @@ -24,6 +29,7 @@ class BasePvcCreateRespinCephPods(E2ETest): """ kube_job_pvc_list, kube_job_pod_list = ([], []) + pod_count = 0 def create_pvc_pod(self, obj_name, number_of_pvc, size): """ @@ -99,25 +105,30 @@ def create_pvc_pod(self, obj_name, number_of_pvc, size): lcl[f"rbd-rwx-pvc-{obj_name}"].create(namespace=self.namespace) lcl[f"cephfs-pvc-{obj_name}"].create(namespace=self.namespace) - # Check all the PVC reached Bound state - rbd_pvc_name = scale_lib.check_all_pvc_reached_bound_state_in_kube_job( - kube_job_obj=lcl[f"rbd-pvc-{obj_name}"], - namespace=self.namespace, - no_of_pvc=int(number_of_pvc), - timeout=60, - ) - rbd_rwx_pvc_name = scale_lib.check_all_pvc_reached_bound_state_in_kube_job( - kube_job_obj=lcl[f"rbd-rwx-pvc-{obj_name}"], - namespace=self.namespace, - no_of_pvc=int(number_of_pvc), - timeout=60, - ) - fs_pvc_name = scale_lib.check_all_pvc_reached_bound_state_in_kube_job( - kube_job_obj=lcl[f"cephfs-pvc-{obj_name}"], - namespace=self.namespace, - no_of_pvc=int(number_of_pvc * 2), - timeout=60, - ) + try: + # Check all the PVC reached Bound state + rbd_pvc_name = scale_lib.check_all_pvc_reached_bound_state_in_kube_job( + kube_job_obj=lcl[f"rbd-pvc-{obj_name}"], + namespace=self.namespace, + no_of_pvc=int(number_of_pvc), + timeout=60, + ) + rbd_rwx_pvc_name = scale_lib.check_all_pvc_reached_bound_state_in_kube_job( + kube_job_obj=lcl[f"rbd-rwx-pvc-{obj_name}"], + namespace=self.namespace, + no_of_pvc=int(number_of_pvc), + timeout=60, + ) + fs_pvc_name = scale_lib.check_all_pvc_reached_bound_state_in_kube_job( + kube_job_obj=lcl[f"cephfs-pvc-{obj_name}"], + namespace=self.namespace, + no_of_pvc=int(number_of_pvc * 2), + timeout=60, + ) + + except Exception as ex: + log.error(f"All or some of the PVCs didn't Bound in expected time {ex}") + raise PVCNotCreated("PVC not up in expected time.") # Construct pod yaml file for kube_job pod_data_list = list() @@ -153,13 +164,23 @@ def create_pvc_pod(self, obj_name, number_of_pvc, size): ) lcl[f"pod-{obj_name}"].create(namespace=self.namespace) - # Check all the POD reached Running state - pod_running_list = scale_lib.check_all_pod_reached_running_state_in_kube_job( - kube_job_obj=lcl[f"pod-{obj_name}"], - namespace=self.namespace, - no_of_pod=len(pod_data_list), - timeout=90, - ) + try: + # Check all the POD reached Running state + pod_running_list = ( + scale_lib.check_all_pod_reached_running_state_in_kube_job( + kube_job_obj=lcl[f"pod-{obj_name}"], + namespace=self.namespace, + no_of_pod=len(pod_data_list), + timeout=90, + ) + ) + + except Exception as ex: + log.error( + f"All or some of PODs didn't reach Running state in expected time {ex}" + ) + raise PodNotCreated("Pod not running in expected time") + self.pod_count = self.pod_count + len(pod_data_list) # Update list with all the kube_job object created, list will be used in cleanup @@ -207,6 +228,7 @@ def cleanup(self): @scale +@orange_squad @ignore_leftovers @skipif_external_mode @pytest.mark.parametrize( @@ -237,8 +259,6 @@ class TestPVSTOcsCreatePVCsAndRespinCephPods(BasePvcCreateRespinCephPods): Class for PV scale Create Cluster with 1000 PVC, then Respin ceph pods parallel """ - pod_count = 0 - @pytest.fixture() def setup_fixture(self, teardown_factory, request): proj_obj = helpers.create_project() @@ -267,6 +287,13 @@ def test_pv_scale_out_create_pvcs_and_respin_ceph_pods( if scale_pod_count <= self.pod_count: log.info(f"Create {scale_pod_count} pvc and pods") break + elif iteration == 5: + # Section to terminate the while loop in-case environment failures. + log.error( + "Breaking while loop since expected iteration already completed. " + "There should be an issue with PVC or POD creation." + ) + exit("Exiting code due to PVC POD creation issues") else: iteration += 1 thread1 = threading.Thread(