From 3abe4fd2be306e2aad312cd0570569d3b8346f7c Mon Sep 17 00:00:00 2001 From: lrangine <19699092+lokeshrangineni@users.noreply.github.com> Date: Mon, 6 Jan 2025 22:50:51 -0500 Subject: [PATCH 1/8] Setting log messages to info so that we can see the log messages for troubleshooting. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> --- sdk/python/tests/integration/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/python/tests/integration/conftest.py b/sdk/python/tests/integration/conftest.py index 82f80b8992..b37e1cd31c 100644 --- a/sdk/python/tests/integration/conftest.py +++ b/sdk/python/tests/integration/conftest.py @@ -9,7 +9,7 @@ from tests.utils.auth_permissions_util import setup_permissions_on_keycloak logger = logging.getLogger(__name__) - +logger.setLevel(logging.INFO) @pytest.fixture(scope="session") def start_keycloak_server(): From ed0d7e6ea117bf8a7c2c0f18cec3816852008fa4 Mon Sep 17 00:00:00 2001 From: lrangine <19699092+lokeshrangineni@users.noreply.github.com> Date: Mon, 6 Jan 2025 23:01:03 -0500 Subject: [PATCH 2/8] Setting log messages to info so that we can see the log messages for troubleshooting. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> --- Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Makefile b/Makefile index 7962816921..99fe808344 100644 --- a/Makefile +++ b/Makefile @@ -107,6 +107,7 @@ test-python-unit: test-python-integration: python -m pytest --tb=short -v -n 8 --integration --color=yes --durations=10 --timeout=1200 --timeout_method=thread --dist loadgroup \ -k "(not snowflake or not test_historical_features_main)" \ + --log-cli-level=INFO \ sdk/python/tests test-python-integration-local: @@ -114,6 +115,7 @@ test-python-integration-local: FEAST_LOCAL_ONLINE_CONTAINER=True \ python -m pytest --tb=short -v -n 8 --color=yes --integration --durations=10 --timeout=1200 --timeout_method=thread --dist loadgroup \ -k "not test_lambda_materialization and not test_snowflake_materialization" \ + --log-cli-level=INFO \ sdk/python/tests test-python-integration-container: From b42b650126af26474704da8ded9bb428e2d2dd36 Mon Sep 17 00:00:00 2001 From: lrangine <19699092+lokeshrangineni@users.noreply.github.com> Date: Mon, 6 Jan 2025 23:08:09 -0500 Subject: [PATCH 3/8] Setting log messages to info so that we can see the log messages for troubleshooting. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> --- sdk/python/tests/integration/conftest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sdk/python/tests/integration/conftest.py b/sdk/python/tests/integration/conftest.py index b37e1cd31c..37111b299e 100644 --- a/sdk/python/tests/integration/conftest.py +++ b/sdk/python/tests/integration/conftest.py @@ -14,6 +14,7 @@ @pytest.fixture(scope="session") def start_keycloak_server(): logger.info("Starting keycloak instance") + print("Starting keycloak instance print") with KeycloakContainer("quay.io/keycloak/keycloak:24.0.1") as keycloak_container: setup_permissions_on_keycloak(keycloak_container.get_client()) yield keycloak_container.get_url() From 7ebcbf452e91460a093c195150a238b61b593430 Mon Sep 17 00:00:00 2001 From: lrangine <19699092+lokeshrangineni@users.noreply.github.com> Date: Mon, 6 Jan 2025 23:29:08 -0500 Subject: [PATCH 4/8] Setting log messages to info so that we can see the log messages for troubleshooting. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> --- sdk/python/tests/integration/conftest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sdk/python/tests/integration/conftest.py b/sdk/python/tests/integration/conftest.py index 37111b299e..4bdbc344d3 100644 --- a/sdk/python/tests/integration/conftest.py +++ b/sdk/python/tests/integration/conftest.py @@ -10,6 +10,7 @@ logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) +logger.propagate = True @pytest.fixture(scope="session") def start_keycloak_server(): From ee5e3f4b66d4de4ce4a39a91b6e1abe9b180b618 Mon Sep 17 00:00:00 2001 From: lrangine <19699092+lokeshrangineni@users.noreply.github.com> Date: Mon, 6 Jan 2025 23:38:56 -0500 Subject: [PATCH 5/8] Setting log messages to info so that we can see the log messages for troubleshooting. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 99fe808344..6a20bbfea6 100644 --- a/Makefile +++ b/Makefile @@ -107,7 +107,7 @@ test-python-unit: test-python-integration: python -m pytest --tb=short -v -n 8 --integration --color=yes --durations=10 --timeout=1200 --timeout_method=thread --dist loadgroup \ -k "(not snowflake or not test_historical_features_main)" \ - --log-cli-level=INFO \ + --log-cli-level=INFO -s \ sdk/python/tests test-python-integration-local: @@ -115,7 +115,7 @@ test-python-integration-local: FEAST_LOCAL_ONLINE_CONTAINER=True \ python -m pytest --tb=short -v -n 8 --color=yes --integration --durations=10 --timeout=1200 --timeout_method=thread --dist loadgroup \ -k "not test_lambda_materialization and not test_snowflake_materialization" \ - --log-cli-level=INFO \ + --log-cli-level=INFO -s \ sdk/python/tests test-python-integration-container: From 7da6e99a5c39437a24d71727a158e42b96868d20 Mon Sep 17 00:00:00 2001 From: lrangine <19699092+lokeshrangineni@users.noreply.github.com> Date: Tue, 7 Jan 2025 12:10:01 -0500 Subject: [PATCH 6/8] Managing the common state to avoid the keycloak spinning multiple times in the worker threads. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> --- sdk/python/tests/integration/conftest.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/sdk/python/tests/integration/conftest.py b/sdk/python/tests/integration/conftest.py index 4bdbc344d3..f6f621320b 100644 --- a/sdk/python/tests/integration/conftest.py +++ b/sdk/python/tests/integration/conftest.py @@ -1,4 +1,5 @@ import logging +from multiprocessing import Manager import pytest from testcontainers.keycloak import KeycloakContainer @@ -12,14 +13,24 @@ logger.setLevel(logging.INFO) logger.propagate = True +shared_state = Manager().dict() + @pytest.fixture(scope="session") def start_keycloak_server(): + # If the Keycloak instance is already started (in any worker), reuse it + if shared_state.get("keycloak_started", False): + return shared_state["keycloak_url"] logger.info("Starting keycloak instance") print("Starting keycloak instance print") with KeycloakContainer("quay.io/keycloak/keycloak:24.0.1") as keycloak_container: setup_permissions_on_keycloak(keycloak_container.get_client()) - yield keycloak_container.get_url() + shared_state["keycloak_started"] = True + shared_state["keycloak_url"] = keycloak_container.get_url() + yield shared_state["keycloak_url"] + # After the fixture is done, cleanup the shared state + del shared_state["keycloak_started"] + del shared_state["keycloak_url"] @pytest.fixture(scope="session") def mysql_server(): From 60a8e7776710936ca17e5befbf95d8da8a7f8375 Mon Sep 17 00:00:00 2001 From: lrangine <19699092+lokeshrangineni@users.noreply.github.com> Date: Tue, 7 Jan 2025 12:10:01 -0500 Subject: [PATCH 7/8] Managing the common state to avoid the keycloak spinning multiple times in the worker threads. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> --- sdk/python/tests/integration/conftest.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/sdk/python/tests/integration/conftest.py b/sdk/python/tests/integration/conftest.py index f6f621320b..1d36a3a2d8 100644 --- a/sdk/python/tests/integration/conftest.py +++ b/sdk/python/tests/integration/conftest.py @@ -1,4 +1,6 @@ import logging +import random +import time from multiprocessing import Manager import pytest @@ -17,6 +19,10 @@ @pytest.fixture(scope="session") def start_keycloak_server(): + # Add random sleep before checking the state to avoid concurrency issues + random_sleep_time = random.uniform(0, 2) # Random sleep time between 0 and 2 seconds + time.sleep(random_sleep_time) + # If the Keycloak instance is already started (in any worker), reuse it if shared_state.get("keycloak_started", False): return shared_state["keycloak_url"] From 6c522bcbf1c8d8ce0823f06063b59190383823fc Mon Sep 17 00:00:00 2001 From: lrangine <19699092+lokeshrangineni@users.noreply.github.com> Date: Tue, 7 Jan 2025 12:10:01 -0500 Subject: [PATCH 8/8] Managing the common state to avoid the keycloak spinning multiple times in the worker threads. Signed-off-by: lrangine <19699092+lokeshrangineni@users.noreply.github.com> --- sdk/python/tests/integration/conftest.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/sdk/python/tests/integration/conftest.py b/sdk/python/tests/integration/conftest.py index 1d36a3a2d8..267ccf08ac 100644 --- a/sdk/python/tests/integration/conftest.py +++ b/sdk/python/tests/integration/conftest.py @@ -19,15 +19,14 @@ @pytest.fixture(scope="session") def start_keycloak_server(): - # Add random sleep before checking the state to avoid concurrency issues - random_sleep_time = random.uniform(0, 2) # Random sleep time between 0 and 2 seconds + # Add random sleep between 0 and 2 before checking the state to avoid concurrency issues. + random_sleep_time = random.uniform(0, 2) time.sleep(random_sleep_time) # If the Keycloak instance is already started (in any worker), reuse it if shared_state.get("keycloak_started", False): return shared_state["keycloak_url"] logger.info("Starting keycloak instance") - print("Starting keycloak instance print") with KeycloakContainer("quay.io/keycloak/keycloak:24.0.1") as keycloak_container: setup_permissions_on_keycloak(keycloak_container.get_client()) shared_state["keycloak_started"] = True