move benchmark targets to yaml file, and add mechanism for update

.github/workflows/benchmark.yml

@@ -4,7 +4,11 @@ name: benchmark
 on:
     workflow_dispatch:
     push:
-        branches: [ main, create-pull-request/patch ]
+        branches: [ main, create-pull-request/patch, benchmark ]
+
+    pull_request:
+      branches:
+        - main
 
 jobs:
   benchmark:
@@ -16,6 +20,12 @@ jobs:
         with:
           lfs: true
 
+      - uses: tibdex/github-app-token@v1
+        id: generate-token
+        with:
+          app_id: ${{ secrets.APP_ID }}
+          private_key: ${{ secrets.APP_PRIVATE_KEY }}
+
       # Use cached python and dependencies, install poetry
       - name: "Setup Python, Poetry and Dependencies"
         uses: packetcoders/action-setup-cache-python-poetry@main
@@ -27,3 +37,23 @@ jobs:
       - name: Run benchmakrs on python 3.8
         run: |
           poetry run pytest --full-trace --show-capture=no -sv benchmarks/benchmark_*.py
+
+      - name: Obtain git status
+        id: status
+        run: |
+          exec 5>&1
+          STATUS=$(git status|tee >(cat - >&5))
+          echo "STATUS<<EOF" >> $GITHUB_OUTPUT
+          echo "$STATUS" >> $GITHUB_OUTPUT
+          echo "EOF" >> $GITHUB_OUTPUT
+
+      # create pull request if necessary
+      - name: "Create Pull Request"
+        uses: peter-evans/create-pull-request@v5
+        if: ${{ contains(steps.status.outputs.STATUS, 'benchmark_targets.yaml')}}
+        with:
+          token: ${{ steps.generate-token.outputs.token }}
+          commit-message: Update benchmark targets
+          title: "Update benchmark targets"
+          body: |
+            Generated new benchmark targets.

benchmarks/benchmark_sed.py

@@ -10,6 +10,8 @@
 
 from sed import SedProcessor
 from sed.binning.binning import bin_dataframe
+from sed.core.config import load_config
+from sed.core.config import save_config
 
 package_dir = os.path.dirname(find_spec("sed").origin)
 
@@ -27,14 +29,10 @@
 dataframe = dask.dataframe.from_dask_array(array, columns=axes)
 
 
-target_artificial_1d = 2.68
-target_artificial_4d = 8.15
-target_inv_dfield = 6.1
-target_binning_1d = 16.6
-target_binning_4d = 22.0
+targets = load_config(package_dir + "/../benchmarks/benchmark_targets.yaml")
 
 
-def test_artificial_1d() -> None:
+def test_binning_1d() -> None:
     """Run a benchmark for 1d binning of artificial data"""
     bins_ = [1000]
     axes_ = ["t"]
@@ -50,10 +48,14 @@ def test_artificial_1d() -> None:
     )
     result = timer.repeat(5, number=1)
     print(result)
-    assert min(result) < target_artificial_1d
+    assert min(result) < targets["binning_1d"]
+    # update targets if substantial improvement occurs
+    if np.mean(result) < 0.9 * targets["binning_1d"]:
+        targets["binning_1d"] = float(np.mean(result) * 1.1)
+        save_config(targets, package_dir + "/../benchmarks/benchmark_targets.yaml")
 
 
-def test_artificial_4d() -> None:
+def test_binning_4d() -> None:
     """Run a benchmark for 4d binning of artificial data"""
     bins_ = [100, 100, 100, 100]
     axes_ = axes
@@ -69,7 +71,11 @@ def test_artificial_4d() -> None:
     )
     result = timer.repeat(5, number=1)
     print(result)
-    assert min(result) < target_artificial_4d
+    assert min(result) < targets["binning_4d"]
+    # update targets if substantial improvement occurs
+    if np.mean(result) < 0.9 * targets["binning_4d"]:
+        targets["binning_4d"] = float(np.mean(result) * 1.1)
+        save_config(targets, package_dir + "/../benchmarks/benchmark_targets.yaml")
 
 
 def test_splinewarp() -> None:
@@ -89,7 +95,11 @@ def test_splinewarp() -> None:
     )
     result = timer.repeat(5, number=1)
     print(result)
-    assert min(result) < target_inv_dfield
+    assert min(result) < targets["inv_dfield"]
+    # update targets if substantial improvement occurs
+    if np.mean(result) < 0.9 * targets["inv_dfield"]:
+        targets["inv_dfield"] = float(np.mean(result) * 1.1)
+        save_config(targets, package_dir + "/../benchmarks/benchmark_targets.yaml")
 
 
 def test_workflow_1d() -> None:
@@ -118,7 +128,11 @@ def test_workflow_1d() -> None:
     )
     result = timer.repeat(5, number=1)
     print(result)
-    assert min(result) < target_binning_1d
+    assert min(result) < targets["workflow_1d"]
+    # update targets if substantial improvement occurs
+    if np.mean(result) < 0.9 * targets["workflow_1d"]:
+        targets["workflow_1d"] = float(np.mean(result) * 1.1)
+        save_config(targets, package_dir + "/../benchmarks/benchmark_targets.yaml")
 
 
 def test_workflow_4d() -> None:
@@ -147,4 +161,8 @@ def test_workflow_4d() -> None:
     )
     result = timer.repeat(5, number=1)
     print(result)
-    assert min(result) < target_binning_4d
+    assert min(result) < targets["workflow_4d"]
+    # update targets if substantial improvement occurs
+    if np.mean(result) < 0.9 * targets["workflow_4d"]:
+        targets["workflow_4d"] = float(np.mean(result) * 1.1)
+        save_config(targets, package_dir + "/../benchmarks/benchmark_targets.yaml")

benchmarks/benchmark_targets.yaml

@@ -0,0 +1,5 @@
+binning_1d: 2.68
+binning_4d: 8.15
+inv_dfield: 6.1
+workflow_1d: 16.6
+workflow_4d: 22.0