Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add image mimetypes #345

Closed
wants to merge 2 commits into from
Closed

add image mimetypes #345

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
96edf57
add image mimetypes
Ceceliachenen Jan 20, 2025
70b4b24
fix image type
Ceceliachenen Jan 20, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 10 additions & 2 deletions src/pai_rag/integrations/index/pai/multimodal/multimodal_retriever.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
from pai_rag.integrations.index.pai.local.local_bm25_index import LocalBm25IndexStore
from loguru import logger
import llama_index.core.instrumentation as instrument
from pai_rag.utils.constants import DEFAULT_IMAGE_MIMETYPE

dispatcher = instrument.get_dispatcher(__name__)

Expand Down Expand Up @@ -200,7 +201,10 @@ def _retrieve(
if image_url and image_url not in seen_image_urls:
integrated_image_nodes.append(
NodeWithScore(
node=ImageNode(image_url=image_url),
node=ImageNode(
image_url=image_url,
image_mimetype=DEFAULT_IMAGE_MIMETYPE,
),
score=node.score,
)
)
Expand Down Expand Up @@ -447,6 +451,7 @@ def _build_node_list_from_query_result(
node = ImageNode(
id_=node.id_,
image_url=node.metadata.get("image_url"),
image_mimetype=DEFAULT_IMAGE_MIMETYPE,
metadata=node.metadata,
)
query_result.nodes[i] = node
Expand Down Expand Up @@ -513,7 +518,10 @@ async def _aretrieve(self, query_bundle: QueryBundle) -> List[NodeWithScore]:
if image_url and image_url not in seen_image_urls:
integrated_image_nodes.append(
NodeWithScore(
node=ImageNode(image_url=image_url),
node=ImageNode(
image_url=image_url,
image_mimetype=DEFAULT_IMAGE_MIMETYPE,
),
score=node.score,
)
)
Expand Down
3 changes: 3 additions & 0 deletions src/pai_rag/integrations/nodeparsers/pai/pai_markdown_parser.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
build_markdown_tree,
TreeNode,
)
from pai_rag.utils.constants import DEFAULT_IMAGE_MIMETYPE


class ImageInfo(BaseModel):
Expand Down Expand Up @@ -74,6 +75,7 @@ def _format_tree_nodes(
image_node = ImageNode(
embedding=doc_node.embedding,
image_url=node.content,
image_mimetype=DEFAULT_IMAGE_MIMETYPE,
excluded_embed_metadata_keys=doc_node.excluded_embed_metadata_keys,
excluded_llm_metadata_keys=doc_node.excluded_llm_metadata_keys,
metadata_separator=doc_node.metadata_separator,
Expand Down Expand Up @@ -235,6 +237,7 @@ def traverse_tree(
image_node = ImageNode(
embedding=doc_node.embedding,
image_url=child.content,
image_mimetype=DEFAULT_IMAGE_MIMETYPE,
excluded_embed_metadata_keys=doc_node.excluded_embed_metadata_keys,
excluded_llm_metadata_keys=doc_node.excluded_llm_metadata_keys,
metadata_separator=doc_node.metadata_separator,
Expand Down
1 change: 0 additions & 1 deletion src/pai_rag/integrations/nodeparsers/pai/pai_node_parser.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -158,7 +158,6 @@ def get_nodes_from_documents(
text=doc_node.text,
metadata=doc_node.metadata,
image_url=doc_node.image_url,
image_path=doc_node.image_path,
image_mimetype=doc_node.image_mimetype,
)
)
Expand Down
6 changes: 5 additions & 1 deletion src/pai_rag/integrations/readers/pai_image_reader.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,11 +46,15 @@ def load_data(
}, # set public read to make image accessible
path_prefix="pairag/images/",
)
extension = file_path.suffix.replace(".", "")
mimetypes = f"image/{extension}"

extra_info["file_path"] = str(file_path)
extra_info["file_name"] = os.path.basename(file_path)
extra_info["image_url"] = image_url
image_doc = ImageDocument(image_url=image_url, extra_info=extra_info)
image_doc = ImageDocument(
image_url=image_url, image_mimetype=mimetypes, extra_info=extra_info
)
docs = [image_doc]
# docs = self.load_image_urls([image_url], extra_info=extra_info)
return docs
2 changes: 2 additions & 0 deletions src/pai_rag/utils/constants.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,3 +23,5 @@
DEFAULT_DATAFILE_DIR = "./data"

DEFAULT_DASHSCOPE_EMBEDDING_MODEL = "text-embedding-v2"

DEFAULT_IMAGE_MIMETYPE = "image/jpeg"
Loading