test: expand test_app cov

nxbench/benchmarks/benchmark.py

@@ -478,7 +478,11 @@ async def main_benchmark(
         )
 
         for run_result in results:
-            if run_result is not None:
+            if isinstance(run_result, BaseException):
+                logger.error("A subflow raised an exception: %s", run_result)
+                continue
+
+            if isinstance(run_result, dict):
                 run_result["python_version"] = actual_python_version
                 bname = run_result.get("backend", "unknown")
                 run_result["backend_version"] = backend_version_map.get(

nxbench/viz/tests/test_app.py

@@ -1,5 +1,8 @@
+# test_app.py
+
 from unittest.mock import patch
 
+import pandas as pd
 import plotly.graph_objects as go
 import pytest
 
@@ -13,29 +16,33 @@
 @pytest.fixture
 def mock_load_and_prepare_data_return():
     """
-    Provide a mocked return value for load_and_prepare_data that includes a 3-level
-    MultiIndex: (algorithm, dataset, backend_full). This way, "dataset" and
-    "backend_full"
-    are legitimate levels in df_agg.
-
-    Returns
-    -------
-    tuple
-        (df, df_agg, group_columns, available_parcats_columns)
+    Provide a mocked return value for load_and_prepare_data that includes:
+      - All the columns referenced by `hover_data` in make_violin_figure
+      - "backend" (or "backend_full") so that the fallback dimension will work
+      - Enough columns so aggregator code can run
     """
-    import pandas as pd
-
     df = pd.DataFrame(
         {
             "algorithm": ["bfs", "bfs", "dfs"],
             "dataset": ["ds1", "ds2", "ds3"],
-            "backend_full": ["cpu", "gpu", "cpu"],
+            "backend_full": ["parallel", "cugraph", "graphblas"],
+            "backend": ["parallel", "cugraph", "graphblas"],
             "execution_time_with_preloading": [0.4, 0.5, 0.3],
             "execution_time": [0.6, 0.7, 0.4],
             "memory_used": [0.2, 0.1, 0.3],
+            "num_nodes_bin": [1000, 2000, 3000],
+            "num_edges_bin": [5000, 6000, 7000],
+            "is_directed": [False, True, False],
+            "is_weighted": [False, False, True],
+            "python_version": ["3.9", "3.9", "3.10"],
+            "cpu": ["Intel", "AMD", "Intel"],
+            "os": ["Linux", "Linux", "Windows"],
+            "num_thread": [1, 2, 4],
         }
     )
 
+    df["backend"] = df["backend_full"]
+
     index = pd.MultiIndex.from_tuples(
         [
             ("bfs", "ds1", "cpu"),
@@ -55,8 +62,7 @@ def mock_load_and_prepare_data_return():
     )
 
     group_columns = ["algorithm", "dataset", "backend_full"]
-
-    available_parcats_columns = ["dataset", "backend_full"]
+    available_parcats_columns = ["dataset", "backend_full"]  # for the parallel cat
 
     return (df, df_agg, group_columns, available_parcats_columns)
 
@@ -73,10 +79,7 @@ def mock_load_data_function(mock_load_and_prepare_data_return):
 
 
 def test_app_runs_without_crashing(mock_load_data_function):
-    """
-    Test that the Dash app can be instantiated without errors and without
-    starting the server.
-    """
+    """Ensure the app can be instantiated (server not actually run)."""
     try:
         run_server(debug=False, run=False)
     except Exception as e:
@@ -91,10 +94,10 @@ def test_app_runs_without_crashing(mock_load_data_function):
         "memory_used",
     ],
 )
-def test_make_parallel_categories_figure(color_by, mock_load_data_function):
+def test_make_parallel_categories_figure_basic(color_by, mock_load_data_function):
     """
     Test the logic function for building the parallel categories figure with various
-    color_by parameters.
+    color_by parameters under normal circumstances.
     """
     df, df_agg, group_columns, available_parcats_columns = (
         mock_load_data_function.return_value
@@ -106,56 +109,171 @@ def test_make_parallel_categories_figure(color_by, mock_load_data_function):
     fig, store_data = make_parallel_categories_figure(
         df, df_agg, group_columns, selected_algorithm, color_by, selected_dimensions
     )
+    assert isinstance(fig, go.Figure)
+    assert store_data is not None
 
-    # Basic checks
-    assert isinstance(fig, go.Figure), "Expected a Plotly Figure object"
-    assert store_data is not None, "Expected non-None store_data for hover info."
+    trace = fig.data[0]
+    assert isinstance(trace, go.Parcats)
 
-    # Check colorbar title logic
     if color_by == "execution_time":
         expected_title = "Execution Time (s)"
     elif color_by == "execution_time_with_preloading":
         expected_title = "Execution Time w/ Preloading (s)"
     else:
         expected_title = "Memory Used (GB)"
 
-    assert len(fig.data) > 0, "Figure has no data traces."
-    trace = fig.data[0]
-    assert isinstance(
-        trace, go.Parcats
-    ), "Expected the first trace to be a Parcats plot."
+    assert trace.line.colorbar.title.text == expected_title
 
-    actual_title = trace.line.colorbar.title.text
-    assert (
-        actual_title == expected_title
-    ), f"Colorbar title should be '{expected_title}', got '{actual_title}'"
 
+def test_make_parallel_categories_figure_preloading_column_missing(
+    mock_load_data_function,
+):
+    """If 'execution_time_with_preloading' is NOT in df.columns, fallback to
+    mean_execution_time.
+    """
+    df, df_agg, group_columns, _ = mock_load_data_function.return_value
+    df_no_preload = df.drop(columns=["execution_time_with_preloading"])
+
+    selected_algorithm = "bfs"
+    color_by = "execution_time_with_preloading"
+    selected_dimensions = ["dataset", "backend_full"]
 
-def test_make_violin_figure_no_data(mock_load_data_function):
-    """Test that make_violin_figure returns a figure with a
-    "No data available for the selected algorithm." annotation when given a
-    non-existent algorithm.
+    fig, store_data = make_parallel_categories_figure(
+        df_no_preload,
+        df_agg,
+        group_columns,
+        selected_algorithm,
+        color_by,
+        selected_dimensions,
+    )
+    assert isinstance(fig, go.Figure)
+    assert getattr(fig.data[0].line, "colorscale", None), "No colorscale found!"
 
-    Parameters
-    ----------
-    mock_load_data_function : pytest.fixture
-        Fixture that mocks the data loading, ensuring a consistent dataset.
+
+def test_make_parallel_categories_figure_preloading_agg_keyerror(
+    mock_load_data_function,
+):
+    """
+    If aggregator's .xs(...) fails for BFS, we fallback to mean_execution_time.
+    We'll force that by removing BFS from df_agg so `.xs('bfs')` triggers KeyError.
     """
+    df, df_agg, group_columns, _ = mock_load_data_function.return_value
+
+    df_agg_no_bfs = df_agg.drop(labels="bfs", level="algorithm")
+
+    selected_algorithm = "bfs"
+    color_by = "execution_time_with_preloading"
+    selected_dimensions = ["dataset", "backend_full"]
+
+    fig, store_data = make_parallel_categories_figure(
+        df,
+        df_agg_no_bfs,
+        group_columns,
+        selected_algorithm,
+        color_by,
+        selected_dimensions,
+    )
+
+    assert isinstance(fig, go.Figure)
+    assert not store_data, "We expect an empty store_data due to KeyError fallback."
+
+
+def test_make_violin_figure_empty_df(mock_load_data_function):
+    """If the .xs(...) yields an empty DataFrame, we get "No data available" figure."""
     df, df_agg, group_columns, available_parcats_columns = (
         mock_load_data_function.return_value
     )
+    df_agg_empty = df_agg.drop(labels="bfs", level="algorithm")
+
+    selected_algorithm = "bfs"
+    fig = make_violin_figure(
+        df,
+        df_agg_empty,
+        selected_algorithm,
+        "execution_time",
+        ["dataset", "backend_full"],
+    )
+    assert fig.layout.annotations
+    assert any("No data available" in ann["text"] for ann in fig.layout.annotations)
 
-    selected_algorithm = "fakealgo"
+
+@pytest.mark.parametrize(
+    ("color_by", "expected_y_metric"),
+    [
+        ("execution_time", "mean_execution_time"),
+        ("execution_time_with_preloading", "mean_execution_time_with_preloading"),
+        ("memory_used", "mean_memory_used"),
+    ],
+)
+def test_make_violin_figure_color_by(
+    color_by, expected_y_metric, mock_load_data_function
+):
+    df, df_agg, group_columns, available_parcats_columns = (
+        mock_load_data_function.return_value
+    )
+    df_agg = df_agg.reset_index()
+    df_agg["num_nodes_bin"] = [1000, 2000, 3000]
+    df_agg["num_edges_bin"] = [5000, 6000, 7000]
+    df_agg["is_directed"] = [False, True, False]
+    df_agg["is_weighted"] = [False, False, True]
+    df_agg["python_version"] = ["3.9", "3.9", "3.10"]
+    df_agg["cpu"] = ["Intel", "AMD", "Intel"]
+    df_agg["os"] = ["Linux", "Linux", "Windows"]
+    df_agg["num_thread"] = [1, 2, 4]
+    df_agg.set_index(["algorithm", "dataset", "backend_full"], inplace=True)
+
+    selected_algorithm = "bfs"
+    fig = make_violin_figure(
+        df,
+        df_agg,
+        selected_algorithm,
+        color_by,
+        ["dataset", "backend_full"],
+    )
+    assert isinstance(fig, go.Figure)
+
+
+def test_make_violin_figure_dimension_fallback(mock_load_data_function):
+    """
+    If the chosen dimension is missing, fallback to "backend" or "backend_full".
+    We'll just ensure it doesn't crash now that 'backend' exists.
+    """
+    df, df_agg, group_columns, available_parcats_columns = (
+        mock_load_data_function.return_value
+    )
+    df_agg = df_agg.reset_index()
+
+    df_agg["num_nodes_bin"] = [1000, 2000, 3000]
+    df_agg["num_edges_bin"] = [5000, 6000, 7000]
+    df_agg["is_directed"] = [False, True, False]
+    df_agg["is_weighted"] = [False, False, True]
+    df_agg["python_version"] = ["3.9", "3.9", "3.10"]
+    df_agg["cpu"] = ["Intel", "AMD", "Intel"]
+    df_agg["os"] = ["Linux", "Linux", "Windows"]
+    df_agg["num_thread"] = [1, 2, 4]
+
+    df_agg["backend"] = df_agg["backend_full"]
+
+    df_agg.set_index(["algorithm", "dataset", "backend_full"], inplace=True)
+
+    selected_algorithm = "bfs"
     color_by = "execution_time"
-    selected_dimensions = ["dataset", "backend_full"]
+    selected_dimensions = ["foo_dimension"]  # doesn't exist
 
     fig = make_violin_figure(
         df, df_agg, selected_algorithm, color_by, selected_dimensions
     )
+    assert isinstance(fig, go.Figure)
+    assert not fig.layout.annotations
 
-    assert fig.layout.annotations, "Expected annotations in the layout for no data."
 
-    expected_text = "No data available for the selected algorithm."
-    assert any(
-        expected_text in ann["text"] for ann in fig.layout.annotations
-    ), f"Could not find '{expected_text}' annotation in the figure."
+def test_make_violin_figure_no_data_for_algorithm(mock_load_data_function):
+    """If the algorithm doesn't exist, KeyError => "No data available" annotation."""
+    df, df_agg, group_columns, available_parcats_columns = (
+        mock_load_data_function.return_value
+    )
+    fig = make_violin_figure(
+        df, df_agg, "fakealgo", "execution_time", ["dataset", "backend_full"]
+    )
+    assert fig.layout.annotations
+    assert any("No data available" in ann["text"] for ann in fig.layout.annotations)