Make the node hub CI/CD parallel for faster testing as well as having…

… more granular integration control (#710)

Currently the node hub CI/CD is sequential, which is not going to grow
if we have additional packages.

Making the node hub CI/CD parallel make it faster for testing as well as
having more granular integration control.

.github/workflows/node-hub-ci-cd.yml

@@ -11,9 +11,29 @@ on:
     types: [published]
 
 jobs:
-  ci:
+  find-jobs:
     runs-on: ubuntu-latest
+    name: Find Jobs
+    outputs:
+      folders: ${{ steps.jobs.outputs.folders }}
+    steps:
+      - uses: actions/checkout@v1
+
+      - id: jobs
+        uses: kmanimaran/list-folder-action@v4
+        with:
+          path: ./node-hub
 
+  ci:
+    runs-on: ubuntu-latest
+    needs: [find-jobs]
+    defaults:
+      run:
+        working-directory: node-hub/${{ matrix.folder }}
+    strategy:
+      matrix:
+        folder: ${{ fromJson(needs.find-jobs.outputs.folders )}}
+      fail-fast: false
     steps:
       - name: Checkout repository
         uses: actions/checkout@v2
@@ -53,11 +73,17 @@ jobs:
 
       - name: Run Linting and Tests
         run: |
-          chmod +x .github/workflows/node_hub_test.sh
-          .github/workflows/node_hub_test.sh
+          chmod +x ../../.github/workflows/node_hub_test.sh
+          ../../.github/workflows/node_hub_test.sh
 
   publish:
-    needs: [ci]
+    needs: [ci, find-jobs]
+    defaults:
+      run:
+        working-directory: node-hub/${{ matrix.folder }}
+    strategy:
+      matrix:
+        folder: ${{ fromJson(needs.find-jobs.outputs.folders )}}
     runs-on: ubuntu-latest
     if: github.event_name == 'release' && startsWith(github.ref, 'refs/tags/')
 
@@ -102,21 +128,20 @@ jobs:
           POETRY_PYPI_TOKEN_PYPI: ${{ secrets.PYPI_PASS }}
           CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
         run: |
-          for dir in node-hub/*/ ; do
-            if [ -d "$dir" ]; then
-              if [[ -f "$dir/Cargo.toml" && -f "$dir/pyproject.toml" ]]; then
-                echo "Publishing $dir using maturin..."
-                (cd "$dir" && poetry publish)
-              else
-                if [ -f "$dir/pyproject.toml" ]; then
-                  echo "Publishing $dir using Poetry..."
-                  (cd "$dir" && poetry publish --build)
-                fi
-              fi
-              
-              if [ -f "$dir/Cargo.toml" ]; then
-                echo "Publishing $dir using Cargo..."
-                (cd "$dir" && cargo publish)
-              fi
+          dir=$(pwd)
+          base_dir=$(basename "$dir")
+
+          if [[ -f "Cargo.toml" && -f "pyproject.toml" ]]; then
+            echo "Publishing $dir using maturin..."
+            poetry publish
+          else
+            if [ -f "pyproject.toml" ]; then
+              echo "Publishing $dir using Poetry..."
+              poetry publish --build
             fi
-          done
+          fi
+
+          if [ -f "Cargo.toml" ]; then
+            echo "Publishing $dir using Cargo..."
+            cargo publish
+          fi

.github/workflows/node_hub_test.sh

@@ -4,29 +4,29 @@ set -euo
 # List of ignored modules 
 ignored_folders=("dora-internvl" "dora-parler" "dora-keyboard" "dora-microphone" "terminal-input")
 
-for dir in node-hub/*/ ; do
-    # Get the base name of the directory (without the path)
-    base_dir=$(basename "$dir")
+# Get current working directory
+dir=$(pwd)
 
-    # Check if the directory name is in the ignored list
-    if [[ " ${ignored_folders[@]} " =~ " ${base_dir} " ]]; then
-        echo "Skipping $base_dir as there is a hf model fetching issue..."
-        continue
-    fi
+# Get the base name of the directory (without the path)
+base_dir=$(basename "$dir")
 
-    if [ -d "$dir" ]; then
-        if [ -f "$dir/Cargo.toml" ]; then
-            echo "Running build and tests for Rust project in $dir..."
-            (cd "$dir" && cargo build)
-            (cd "$dir" && cargo test)
-        else
-            if [ -f "$dir/pyproject.toml" ]; then
-            echo "Running linting and tests for Python project in $dir..."
-            (cd "$dir" && pip install .)
-            (cd "$dir" && poetry run black --check .)
-            (cd "$dir" && poetry run pylint --disable=C,R  --ignored-modules=cv2 **/*.py)
-            (cd "$dir" && poetry run pytest)
-            fi
-        fi   
-    fi
-done
+# Check if the directory name is in the ignored list
+if [[ " ${ignored_folders[@]} " =~ " ${base_dir} " ]]; then
+    echo "Skipping $base_dir as we cannot test it on the CI..."
+else
+    if [ -f "$dir/Cargo.toml" ]; then
+        echo "Running build and tests for Rust project in $dir..."
+        cargo check
+        cargo clippy
+        cargo build
+        cargo test
+    else
+        if [ -f "$dir/pyproject.toml" ]; then
+        echo "Running linting and tests for Python project in $dir..."
+        pip install .
+        poetry run black --check .
+        poetry run pylint --disable=C,R  --ignored-modules=cv2 **/*.py
+        poetry run pytest
+        fi
+    fi   
+fi

node-hub/dora-opus/pyproject.toml

@@ -17,6 +17,8 @@ numpy = "< 2.0.0"
 python = "^3.7"
 transformers = "^4.45"
 modelscope = "^1.18.1"
+sentencepiece = "^0.1.99"
+torch = "^2.2.0"
 
 [tool.poetry.scripts]
 dora-opus = "dora_opus.main:main"

node-hub/dora-pyorbbecksdk/pyproject.toml

@@ -10,6 +10,8 @@ packages = [{ include = "dora_pyorbbecksdk" }]
 [tool.poetry.dependencies]
 dora-rs = "^0.3.6"
 python = "^3.7"
+numpy = "< 2.0.0"
+opencv-python = ">= 4.1.1"
 
 
 [tool.poetry.scripts]

node-hub/dora-rerun/src/lib.rs

@@ -227,7 +227,7 @@ pub fn lib_main() -> Result<()> {
                         }
                     }
 
-                    update_visualization(&chain, &rec, &id, &positions)?;
+                    update_visualization(chain, &rec, &id, &positions)?;
                 } else {
                     println!("Could not find chain for {}", id);
                 }

node-hub/dora-rerun/src/urdf.rs

@@ -101,14 +101,14 @@ pub fn update_visualization(
     id: &str,
     positions: &[f32],
 ) -> Result<()> {
-    chain.set_joint_positions_clamped(&positions);
+    chain.set_joint_positions_clamped(positions);
 
     chain.update_transforms();
     chain.update_link_transforms();
 
     for link_name in chain.iter_links().map(|link| link.name.clone()) {
         let link = chain.find_link(&link_name).context("Could not find link")?;
-        let entity_path = get_entity_path(&link, &id);
+        let entity_path = get_entity_path(link, id);
         let link_to_world = link
             .world_transform()
             .context("Could not get world transform")?;