Use UV for the CI/CD (#757)

uv is an altermative to pip and offer better performance as well as being written in Rust it makes dora ecosystem in python with ruff fully rust based which increase the overall security and deployability of dora.
dora-rs · Jan 17, 2025 · c10eb15 · c10eb15
2 parents 9fdb074 + ce3c3b2
commit c10eb15
Show file tree

Hide file tree

Showing 36 changed files with 425 additions and 538 deletions.
diff --git a/.github/workflows/node-hub-ci-cd.yml b/.github/workflows/node-hub-ci-cd.yml
@@ -81,12 +81,11 @@ jobs:
         with:
           python-version: "3.10"
 
-      - name: Set up Poetry
+      - name: Set up UV
         if: runner.os == 'Linux' || github.event_name == 'workflow_dispatch' || (github.event_name == 'release' && startsWith(github.ref, 'refs/tags/'))
         run: |
-          curl -sSL https://install.python-poetry.org | python3 -
+          curl -LsSf https://astral.sh/uv/install.sh | sh
           echo "$HOME/.local/bin" >> $GITHUB_PATH
-          pip install ruff pytest
 
       - name: Set up Rust
         if: runner.os == 'Linux' || github.event_name == 'workflow_dispatch' || (github.event_name == 'release' && startsWith(github.ref, 'refs/tags/'))
@@ -139,9 +138,9 @@ jobs:
         with:
           python-version: "3.10"
 
-      - name: Set up Poetry
+      - name: Set up UV
         run: |
-          curl -sSL https://install.python-poetry.org | python3 -
+          curl -LsSf https://astral.sh/uv/install.sh | sh
           echo "$HOME/.local/bin" >> $GITHUB_PATH
 
       - name: Set up Rust
@@ -152,7 +151,7 @@ jobs:
       - name: Publish Projects
         env:
           MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_PASS }}
-          POETRY_PYPI_TOKEN_PYPI: ${{ secrets.PYPI_PASS }}
+          UV_PUBLISH_TOKEN: ${{ secrets.PYPI_PASS }}
           CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
         run: |
           dir=$(pwd)
@@ -165,7 +164,7 @@ jobs:
             if [[ -f "Cargo.toml" && -f "pyproject.toml" ]]; then
               echo "Publishing $dir using maturin..."
               if [[ "${{ runner.os }}" == "Linux" ]]; then
-                pip3 install "maturin[zig]"
+                pip install "maturin[zig]"
                 
                 ## The CI/CD is sequential to limit the number of workers used.
                 
@@ -181,13 +180,14 @@ jobs:
                 maturin publish --target armv7-unknown-linux-musleabihf --zig --skip-existing
 
               else
-                pip3 install maturin
+                pip install maturin
                 maturin publish  --skip-existing
               fi
             else
               if [ -f "pyproject.toml" ]; then
-                echo "Publishing $dir using Poetry..."
-                poetry publish --build  --skip-existing
+                echo "Publishing $dir using UV..."
+                uv build
+                uv publish
               fi
 
               if [ -f "Cargo.toml" ]; then

diff --git a/.github/workflows/node_hub_test.sh b/.github/workflows/node_hub_test.sh
@@ -22,7 +22,7 @@ else
         cargo build
         cargo test
 
-        pip3 install "maturin[zig]"
+        pip install "maturin[zig]"
         maturin build --zig
 
         # aarch64-unknown-linux-gnu
@@ -41,10 +41,13 @@ else
             cargo test
         else
             if [ -f "$dir/pyproject.toml" ]; then
-            echo "Running linting and tests for Python project in $dir..."
-            pip install .
-            ruff check .
-            poetry run pytest
+            echo "CI: Installing in $dir..."
+            uv venv
+            uv pip install .
+            echo "CI: Running Linting in $dir..."
+            uv run ruff check .
+            echo "CI: Running Pytest in $dir..."
+            uv run pytest
             fi
         fi 
     fi 

diff --git a/.gitignore b/.gitignore
@@ -129,7 +129,7 @@ ipython_config.py
 
 # PEP 582; used by e.g. github.com/David-OConnor/pyflow
 __pypackages__/
-
+.ruff_cache
 # Celery stuff
 celerybeat-schedule
 celerybeat.pid

diff --git a/apis/python/node/dora/__init__.py b/apis/python/node/dora/__init__.py
@@ -10,30 +10,31 @@
 from enum import Enum
 
 from .dora import *
-
 from .dora import (
     Node,
     Ros2Context,
+    Ros2Durability,
+    Ros2Liveliness,
     Ros2Node,
     Ros2NodeOptions,
-    Ros2Topic,
     Ros2Publisher,
+    Ros2QosPolicies,
     Ros2Subscription,
-    start_runtime,
-    __version__,
+    Ros2Topic,
     __author__,
-    Ros2QosPolicies,
-    Ros2Durability,
-    Ros2Liveliness,
+    __version__,
+    start_runtime,
 )
 
 
 class DoraStatus(Enum):
     """Dora status to indicate if operator `on_input` loop
      should be stopped.
+
     Args:
         Enum (u8): Status signaling to dora operator to
         stop or continue the operator.
+
     """
 
     CONTINUE = 0

diff --git a/apis/python/node/dora/cuda.py b/apis/python/node/dora/cuda.py
@@ -1,19 +1,18 @@
 import pyarrow as pa
 
-# To install pyarrow.cuda, run `conda install pyarrow "arrow-cpp-proc=*=cuda" -c conda-forge`
-import pyarrow.cuda as cuda
-
 # Make sure to install torch with cuda
 import torch
+from numba.cuda import to_device
 
 # Make sure to install numba with cuda
 from numba.cuda.cudadrv.devicearray import DeviceNDArray
-from numba.cuda import to_device
+
+# To install pyarrow.cuda, run `conda install pyarrow "arrow-cpp-proc=*=cuda" -c conda-forge`
+from pyarrow import cuda
 
 
 def torch_to_ipc_buffer(tensor: torch.TensorType) -> tuple[pa.array, dict]:
-    """
-    Converts a Pytorch tensor into a pyarrow buffer containing the IPC handle and its metadata.
+    """Converts a Pytorch tensor into a pyarrow buffer containing the IPC handle and its metadata.
 
     Example Use:
     ```python
@@ -34,8 +33,7 @@ def torch_to_ipc_buffer(tensor: torch.TensorType) -> tuple[pa.array, dict]:
 
 
 def ipc_buffer_to_ipc_handle(handle_buffer: pa.array) -> cuda.IpcMemHandle:
-    """
-    Converts a buffer containing a serialized handler into cuda IPC MemHandle.
+    """Converts a buffer containing a serialized handler into cuda IPC MemHandle.
 
     example use:
     ```python
@@ -57,8 +55,7 @@ def ipc_buffer_to_ipc_handle(handle_buffer: pa.array) -> cuda.IpcMemHandle:
 
 
 def cudabuffer_to_numba(buffer: cuda.CudaBuffer, metadata: dict) -> DeviceNDArray:
-    """
-    Converts a pyarrow CUDA buffer to numba.
+    """Converts a pyarrow CUDA buffer to numba.
 
     example use:
     ```python
@@ -74,7 +71,6 @@ def cudabuffer_to_numba(buffer: cuda.CudaBuffer, metadata: dict) -> DeviceNDArra
     numba_tensor = cudabuffer_to_numbda(cudabuffer, event["metadata"])
     ```
     """
-
     shape = metadata["shape"]
     strides = metadata["strides"]
     dtype = metadata["dtype"]
@@ -83,8 +79,7 @@ def cudabuffer_to_numba(buffer: cuda.CudaBuffer, metadata: dict) -> DeviceNDArra
 
 
 def cudabuffer_to_torch(buffer: cuda.CudaBuffer, metadata: dict) -> torch.Tensor:
-    """
-    Converts a pyarrow CUDA buffer to a torch tensor.
+    """Converts a pyarrow CUDA buffer to a torch tensor.
 
     example use:
     ```python
@@ -100,7 +95,6 @@ def cudabuffer_to_torch(buffer: cuda.CudaBuffer, metadata: dict) -> torch.Tensor
     torch_tensor = cudabuffer_to_torch(cudabuffer, event["metadata"])  # on cuda
     ```
     """
-
     device_arr = cudabuffer_to_numba(buffer, metadata)
     torch_tensor = torch.as_tensor(device_arr, device="cuda")
     return torch_tensor
diff --git a/apis/python/node/pyproject.toml b/apis/python/node/pyproject.toml
@@ -6,7 +6,13 @@ build-backend = "maturin"
 name = "dora-rs"
 dynamic = ["version"]
 # Install pyarrow at the same time of dora-rs
+requires-python = ">=3.7"
+license = { text = "MIT" }
+readme = "README.md"
 dependencies = ['pyarrow']
 
+[dependency-groups]
+dev = ["pytest >=8.1.1", "ruff >=0.9.1"]
+
 [tool.maturin]
 features = ["pyo3/extension-module"]
diff --git a/binaries/cli/pyproject.toml b/binaries/cli/pyproject.toml
@@ -5,8 +5,9 @@ build-backend = "maturin"
 [project]
 name = "dora-rs-cli"
 dynamic = ["version"]
-
 scripts = { "dora" = "dora_cli:py_main" }
+license = { text = "MIT" }
+requires-python = ">=3.8"
 dependencies = ['dora-rs']
 
 [tool.maturin]

diff --git a/binaries/cli/src/template/python/__node-name__/pyproject.toml b/binaries/cli/src/template/python/__node-name__/pyproject.toml
@@ -1,27 +1,16 @@
-[tool.poetry]
+[project]
 name = "__node-name__"
 version = "0.0.0"
-authors = ["author"]
+authors = [{ name = "Your Name", email = "[email protected]" }]
 description = "Node Name"
-license = "MIT License"
-homepage = "https://github.com/dora-rs/dora.git"
-documentation = "https://github.com/dora-rs/dora/blob/main/node-hub/__node-name__/README.md"
+license = { text = "MIT" }
 readme = "README.md"
-packages = [{ include = "__node_name__" }]
+requires-python = ">=3.8"
 
-[tool.poetry.dependencies]
-dora-rs = "^0.3.6"
-numpy = "< 2.0.0"
-pyarrow = ">= 15.0.0"
-python = "^3.7"
+dependencies = ["dora-rs >= 0.3.6"]
 
-[tool.poetry.dev-dependencies]
-pytest = ">= 6.3.4"
-ruff = ">= 0.9.1"
+[dependency-groups]
+dev = ["pytest >=8.1.1", "ruff >=0.9.1"]
 
-[tool.poetry.scripts]
+[project.scripts]
 __node-name__ = "__node_name__.main:main"
-
-[build-system]
-requires = ["poetry-core>=1.8.0"]
-build-backend = "poetry.core.masonry.api"
diff --git a/node-hub/README.md b/node-hub/README.md
@@ -27,36 +27,29 @@ def main():
 And then you will need to adapt the following `pyproject.toml` file:
 
 ```toml
-[tool.poetry]
+[project]
 name = "[name of the node e.g. video-encoder, with '-' to replace spaces]"
 version = "0.1"
-authors = ["[Pseudo/Name] <[email]>"]
+authors = [{ name = "[Pseudo/Name]", email = "[email]" }]
 description = "Dora Node for []"
 readme = "README.md"
+license = { text = "MIT" }
 
-packages = [
-    { include = "main.py", to = "[name of the node with '_' to replace spaces]" }
+dependencies = [
+    "dora-rs >= 0.3.8",
 ]
 
-[tool.poetry.dependencies]
-python = "^3.11"
-dora-rs = "0.3.5"
-... [add your dependencies here] ...
-
-[tool.poetry.scripts]
+[project.scripts]
 [name of the node with '-' to replace spaces] = "[name of the node with '_' to replace spaces].main:main"
 
-[build-system]
-requires = ["poetry-core>=1.0.0"]
-build-backend = "poetry.core.masonry.api"
 ```
 
 Finally, the README.md file should explicit all inputs/outputs of the node and how to configure it in the YAML file.
 
 # Example
 
 ```toml
-[tool.poetry]
+[project]
 name = "opencv-plot"
 version = "0.1"
 authors = [
@@ -65,22 +58,18 @@ authors = [
 ]
 description = "Dora Node for plotting data with OpenCV"
 readme = "README.md"
+license = { text = "MIT" }
+requires-python = ">=3.7"
 
-packages = [
-    { include = "main.py", to = "opencv_plot" }
+dependencies = [
+    "dora-rs >= 0.3.8",
 ]
+[dependency-groups]
+dev = ["pytest >=8.1.1", "ruff >=0.9.1"]
 
-[tool.poetry.dependencies]
-python = "^3.11"
-dora-rs = "^0.3.5"
-opencv-python = "^4.10.0.84"
-
-[tool.poetry.scripts]
+[project.scripts]
 opencv-plot = "opencv_plot.main:main"
 
-[build-system]
-requires = ["poetry-core>=1.0.0"]
-build-backend = "poetry.core.masonry.api"
 ```
 
 ## License

diff --git a/node-hub/dora-argotranslate/pyproject.toml b/node-hub/dora-argotranslate/pyproject.toml
@@ -1,25 +1,20 @@
-[tool.poetry]
+[project]
 name = "dora-argotranslate"
 version = "0.3.8"
 description = "Dora Node for Text translating using Argostranslate"
-readme = "README.md"
 authors = [
-    "Haixuan Xavier Tao <[email protected]>",
-    "Enzo Le Van <[email protected]>",
-    "Félix Huang <[email protected]>",
+    { name = "Haixuan Xavier Tao", email = "[email protected]" },
+    { name = "Enzo Le Van", email = "[email protected]" },
+    { name = "Félix Huang", email = "[email protected]" },
 ]
+license = { text = "MIT" }
+readme = "README.md"
+requires-python = ">=3.8"
 
-packages = [{ include = "dora_argotranslate" }]
+dependencies = ["dora-rs >= 0.3.6", "numpy < 2.0.0", "argostranslate >= 1.9.6"]
 
-[tool.poetry.dependencies]
-dora-rs = "^0.3.6"
-numpy = "< 2.0.0"
-python = "^3.7"
-argostranslate = "^1.9.6"
+[dependency-groups]
+dev = ["pytest >=8.1.1", "ruff >=0.9.1"]
 
-[tool.poetry.scripts]
+[project.scripts]
 dora-argotranslate = "dora_argotranslate.main:main"
-
-[build-system]
-requires = ["poetry-core>=1.8.0"]
-build-backend = "poetry.core.masonry.api"