Fix placeholder maps handling on V0

Signed-off-by: DarkLight1337 <[email protected]>

vllm/multimodal/base.py

@@ -326,26 +326,32 @@ def from_seq_group(
             src_ranges  = []
             dest_ranges = []
         """
-        if (not seq_group.multi_modal_data
-                or not seq_group.multi_modal_placeholders):
-            return seq_group.multi_modal_data, {}
+        seq_mm_data = seq_group.multi_modal_data
+        seq_mm_placeholders = seq_group.multi_modal_placeholders
 
-        mm_data = {**seq_group.multi_modal_data}
-        placeholder_maps: Dict[str, MultiModalPlaceholderMap] = defaultdict(
+        if not seq_mm_data or not seq_mm_placeholders:
+            return seq_mm_data, {}
+
+        mm_data = {**seq_mm_data}
+        placeholder_maps = defaultdict[str, MultiModalPlaceholderMap](
             MultiModalPlaceholderMap)
 
-        for (
-                modality,
-                placeholders,
-        ) in seq_group.multi_modal_placeholders.items():
-            mm_items = mm_data.pop(modality)
-            if not isinstance(mm_items, list):
-                mm_items = [mm_items]
+        for modality, placeholders in seq_mm_placeholders.items():
+            # For merged processor, data keys do not correspond to modalities
+            if isinstance(seq_mm_data, MultiModalKwargs):
+                mm_items = [seq_mm_data]
+            else:
+                mm_items = mm_data.pop(modality)
+                if not isinstance(mm_items, list):
+                    mm_items = [mm_items]
 
             if positions:
-                intersecting_items = placeholder_maps[
-                    modality].append_items_from_seq_group(
-                        positions, mm_items, placeholders)
+                intersecting_items = placeholder_maps[modality] \
+                    .append_items_from_seq_group(
+                        positions,
+                        mm_items,
+                        placeholders,
+                    )
 
                 if intersecting_items:
                     mm_data[modality] = intersecting_items

vllm/v1/engine/mm_input_mapper.py

@@ -23,8 +23,9 @@ def process_inputs(
         mm_data: MultiModalDataDict,
         mm_processor_kwargs: Optional[Dict[str, Any]],
     ) -> List[MultiModalKwargs]:
-        if self.mm_registry.has_processor(self.model_config):
-            return [MultiModalKwargs(mm_data)]  # Already processed
+        # Skip this redundant step if merged processor has been applied
+        if isinstance(mm_data, MultiModalKwargs):
+            return [mm_data]
 
         image_inputs = mm_data["image"]
         if not isinstance(image_inputs, list):