From fb223934f2493d7e01e78e2e492f8dee2da5edcc Mon Sep 17 00:00:00 2001 From: sang Date: Thu, 25 Jul 2024 09:58:29 -0700 Subject: [PATCH 1/2] remove forward dag error msg. --- vllm/executor/ray_gpu_executor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vllm/executor/ray_gpu_executor.py b/vllm/executor/ray_gpu_executor.py index e4aaeaa24c1bc..37a943eb70394 100644 --- a/vllm/executor/ray_gpu_executor.py +++ b/vllm/executor/ray_gpu_executor.py @@ -393,7 +393,7 @@ def _compiled_ray_dag(self, enable_asyncio: bool): return forward_dag.experimental_compile(enable_asyncio=enable_asyncio) def __del__(self): - if self.forward_dag is not None: + if hasattr(self, "forward_dag") and self.forward_dag is not None: self.forward_dag.teardown() import ray for worker in self.workers: From 5426cbe3d051c1e01afcbaec09bb8e0456a5daa7 Mon Sep 17 00:00:00 2001 From: sang Date: Thu, 25 Jul 2024 09:59:45 -0700 Subject: [PATCH 2/2] better fix --- vllm/executor/ray_gpu_executor.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/vllm/executor/ray_gpu_executor.py b/vllm/executor/ray_gpu_executor.py index 37a943eb70394..564fa79acfd40 100644 --- a/vllm/executor/ray_gpu_executor.py +++ b/vllm/executor/ray_gpu_executor.py @@ -29,6 +29,7 @@ class RayGPUExecutor(DistributedGPUExecutor): uses_ray: bool = True def _init_executor(self) -> None: + self.forward_dag: Optional["ray.dag.CompiledDAG"] = None # If the env var is set, it uses the Ray's compiled DAG API # which optimizes the control plane overhead. # Run vLLM with VLLM_USE_RAY_COMPILED_DAG=1 to enable it. @@ -60,8 +61,6 @@ def _init_executor(self) -> None: # Create the parallel GPU workers. self._init_workers_ray(placement_group) - self.forward_dag: Optional["ray.dag.CompiledDAG"] = None - def _configure_ray_workers_use_nsight(self, ray_remote_kwargs) -> Dict[str, Any]: # If nsight profiling is enabled, we need to set the profiling @@ -393,7 +392,7 @@ def _compiled_ray_dag(self, enable_asyncio: bool): return forward_dag.experimental_compile(enable_asyncio=enable_asyncio) def __del__(self): - if hasattr(self, "forward_dag") and self.forward_dag is not None: + if self.forward_dag is not None: self.forward_dag.teardown() import ray for worker in self.workers: