Remove duplicate logger from PaiPDFReader (#263)

* Remove duplicate logger from PaiPDFReader

* Add ModelScopeDownloader

* Use logger from loguru not logging

* Remove ak sk info

docs/agentic_rag.md

@@ -118,9 +118,7 @@ Python代码工具分为两个部分:代码和函数定义。
 ```python
 import requests
 import os
-import logging
-
-logger = logging.getLogger(__name__)
+from loguru import logger
 
 
 def get_place_weather(city: str) -> str:

example_data/function_tools/api-tool-with-intent-detection-for-travel-assistant/tools.py

@@ -1,8 +1,6 @@
 import requests
 import os
-import logging
-
-logger = logging.getLogger(__name__)
+from loguru import logger
 
 
 def get_place_weather(city: str) -> str:

src/pai_rag/app/api/agent_demo.py

@@ -1,11 +1,8 @@
 from datetime import datetime
 from fastapi import APIRouter
-import logging
 
 from pydantic import BaseModel
 
-logger = logging.getLogger(__name__)
-
 demo_router = APIRouter()
 
 

src/pai_rag/app/api/middleware.py

@@ -3,9 +3,7 @@
 from starlette.middleware.base import BaseHTTPMiddleware
 from asgi_correlation_id import CorrelationIdMiddleware
 import time
-import logging
-
-logger = logging.getLogger(__name__)
+from loguru import logger
 
 
 class CustomMiddleWare(BaseHTTPMiddleware):

src/pai_rag/app/api/query.py

@@ -15,14 +15,12 @@
     RetrievalQuery,
 )
 from fastapi.responses import StreamingResponse
-import logging
+from loguru import logger
 
 from pai_rag.integrations.nodeparsers.pai.pai_node_parser import (
     COMMON_FILE_PATH_FODER_NAME,
 )
 
-logger = logging.getLogger(__name__)
-
 router = APIRouter()
 
 

src/pai_rag/app/web/event_listeners.py

@@ -19,14 +19,15 @@
     HuggingFaceEmbeddingConfig,
 )
 from pai_rag.integrations.index.pai.vector_store_config import FaissVectorStoreConfig
+from loguru import logger
 
 
 def add_index(*components):
     component_args = dict(zip(index_related_component_keys, components))
     index_entry = components_to_index(**component_args)
     rag_client.add_index(index_entry)
     index_map = get_index_map()
-    print(f"Add index {index_entry.index_name} successfully")
+    logger.info(f"Add index {index_entry.index_name} successfully")
     return [
         gr.update(
             choices=list(index_map.indexes.keys()) + ["NEW"],
@@ -44,7 +45,7 @@ def update_index(*components):
     index_entry = components_to_index(**component_args)
     rag_client.update_index(index_entry)
     index_map = get_index_map()
-    print(f"Update index {index_entry.index_name} successfully")
+    logger.info(f"Update index {index_entry.index_name} successfully")
     return [
         gr.update(
             choices=list(index_map.indexes.keys()) + ["NEW"],

src/pai_rag/app/web/index_utils.py

@@ -301,7 +301,6 @@ def index_to_components(
     component_settings = index_to_components_settings(
         index_entry, index_list, is_new_index
     )
-    print("+++", index_entry.index_name)
     return [gr.update(**setting) for setting in component_settings.values()] + [
         gr.update(choices=index_list, value=index_entry.index_name),
         gr.update(choices=index_list, value=index_entry.index_name),

src/pai_rag/app/web/rag_client.py

@@ -8,6 +8,7 @@
 import html
 import mimetypes
 from http import HTTPStatus
+from loguru import logger
 from pai_rag.app.web.view_model import ViewModel
 from pai_rag.app.web.ui_constants import EMPTY_KNOWLEDGEBASE_MESSAGE
 from pai_rag.core.rag_config import RagConfig
@@ -425,7 +426,7 @@ def add_datasheet(
             if r.status_code != HTTPStatus.OK:
                 raise RagApiError(code=r.status_code, msg=response.message)
         except Exception as e:
-            print(f"add_datasheet failed: {e}")
+            logger.exception(f"add_datasheet failed: {e}")
         finally:
             file_obj.close()
 

src/pai_rag/app/web/tabs/settings_tab.py

@@ -4,12 +4,9 @@
 from pai_rag.app.web.utils import components_to_dict
 from pai_rag.app.web.index_utils import index_related_component_keys
 from pai_rag.app.web.tabs.vector_db_panel import create_vector_db_panel
-import logging
 import os
 import pai_rag.app.web.event_listeners as ev_listeners
 
-logger = logging.getLogger(__name__)
-
 DEFAULT_IS_INTERACTIVE = os.environ.get("PAIRAG_RAG__SETTING__interactive", "true")
 
 

src/pai_rag/app/web/webui.py

@@ -20,12 +20,10 @@
 )
 from pai_rag.app.web.tabs.model.index_info import get_index_map
 
-import logging
+from loguru import logger
 
 DEFAULT_IS_INTERACTIVE = os.environ.get("PAIRAG_RAG__SETTING__interactive", "true")
 
-logger = logging.getLogger("WebUILogger")
-
 
 def resume_ui():
     outputs = {}
@@ -131,6 +129,6 @@ def configure_webapp(app: FastAPI, web_url, rag_url=DEFAULT_LOCAL_URL) -> gr.Blo
     home = make_homepage()
     home.queue(concurrency_count=1, max_size=64)
     home._queue.set_url(web_url)
-    print(web_url)
+    logger.info(f"web_url: {web_url}")
     gr.mount_gradio_app(app, home, path="")
     return home

src/pai_rag/core/rag_application.py

@@ -25,14 +25,13 @@
     ImageNode,
 )
 import json
-import logging
+from loguru import logger
 import os
 from enum import Enum
 from uuid import uuid4
 
 DEFAULT_EMPTY_RESPONSE_GEN = "Empty Response"
 DEFAULT_RAG_INDEX_FILE = "localdata/default_rag_indexes.json"
-logger = logging.getLogger(__name__)
 
 
 def uuid_generator() -> str:
@@ -74,7 +73,6 @@ async def event_generator_async(
 class RagApplication:
     def __init__(self, config: RagConfig):
         self.name = "RagApplication"
-        self.logger = logging.getLogger(__name__)
         self.config = config
         index_manager.add_default_index(self.config)
 
@@ -147,7 +145,7 @@ async def aretrieve(self, query: RetrievalQuery) -> RetrievalResponse:
 
     async def aquery(self, query: RagQuery, chat_type: RagChatType = RagChatType.RAG):
         session_id = query.session_id or uuid_generator()
-        self.logger.debug(f"Get session ID: {session_id}.")
+        logger.debug(f"Get session ID: {session_id}.")
         session_config = self.config.model_copy()
         index_entry = index_manager.get_index_by_name(query.index_name)
         session_config.embedding = index_entry.embedding_config
@@ -168,12 +166,12 @@ async def aquery(self, query: RagQuery, chat_type: RagChatType = RagChatType.RAG
             chat_history=query.chat_history,
         )
         new_question = new_query_bundle.query_str
-        self.logger.info(f"Querying with question '{new_question}'.")
+        logger.info(f"Querying with question '{new_question}'.")
 
         if query.with_intent:
             intent_router = resolve_intent_router(session_config)
             intent = await intent_router.aselect(str_or_query_bundle=new_question)
-            self.logger.info(f"[IntentDetection] Routing query to {intent}.")
+            logger.info(f"[IntentDetection] Routing query to {intent}.")
             if intent == Intents.TOOL:
                 return await self.aquery_agent(query)
             elif intent == Intents.WEBSEARCH:
@@ -295,7 +293,7 @@ async def aquery_analysis(self, query: RagQuery):
             RagResponse
         """
         session_id = query.session_id or uuid_generator()
-        self.logger.debug(f"Get session ID: {session_id}.")
+        logger.debug(f"Get session ID: {session_id}.")
         if not query.question:
             return RagResponse(
                 answer="Empty query. Please input your question.", session_id=session_id

src/pai_rag/core/rag_config_manager.py

@@ -1,10 +1,11 @@
 from dynaconf import Dynaconf, loaders
 from dynaconf.utils.boxing import DynaBox
 
-import logging
+from loguru import logger
 import os
 
 from pai_rag.core.rag_config import RagConfig
+from pai_rag.utils.oss_utils import check_and_set_oss_auth
 
 # store config file generated from ui.
 GENERATED_CONFIG_FILE_NAME = "localdata/settings.snapshot.toml"
@@ -26,7 +27,7 @@ def from_snapshot(cls):
             )
             return cls(config)
         except Exception as error:
-            logging.critical("Read config file failed.")
+            logger.critical("Read config file failed.")
             raise error
 
     @classmethod
@@ -51,7 +52,7 @@ def from_file(cls, config_file):
             # `envvar_prefix` = export envvars with `export PAIRAG_FOO=bar`.
             # `settings_files` = Load these files in the order.
         except Exception as error:
-            logging.critical("Read config file failed.")
+            logger.critical("Read config file failed.")
             raise error
 
     def get_value(self) -> RagConfig:
@@ -61,6 +62,7 @@ def get_value(self) -> RagConfig:
     def update(self, new_value: Dynaconf):
         if self.config.get("rag", None):
             self.config.rag.update(new_value, merge=True)
+            check_and_set_oss_auth(self.config.rag)
 
     def persist(self):
         """Save configuration to file."""
@@ -72,5 +74,5 @@ def get_config_mtime(self):
         try:
             return os.path.getmtime(GENERATED_CONFIG_FILE_NAME)
         except Exception as ex:
-            print(f"Fail to read config mtime {ex}")
+            logger.critical(f"Fail to read config mtime {ex}")
             return -1

src/pai_rag/core/rag_data_loader.py

@@ -4,9 +4,7 @@
 from llama_index.core.ingestion import IngestionPipeline
 from pai_rag.integrations.nodeparsers.pai.pai_node_parser import PaiNodeParser
 from pai_rag.integrations.readers.pai.pai_data_reader import PaiDataReader
-import logging
-
-logger = logging.getLogger(__name__)
+from loguru import logger
 
 
 class RagDataLoader:

src/pai_rag/core/rag_index_manager.py

@@ -7,9 +7,7 @@
     PaiBaseEmbeddingConfig,
 )
 from pai_rag.integrations.index.pai.vector_store_config import BaseVectorStoreConfig
-import logging
-
-logger = logging.getLogger(__name__)
+from loguru import logger
 
 DEFAULT_INDEX_FILE = "localdata/default__rag__index.json"
 DEFAULT_INDEX_NAME = "default_index"

src/pai_rag/core/rag_module.py

@@ -29,9 +29,6 @@
 from pai_rag.integrations.llms.pai.pai_llm import PaiLlm
 from pai_rag.integrations.llms.pai.pai_multi_modal_llm import PaiMultiModalLlm
 from pai_rag.utils.oss_client import OssClient
-import logging
-
-logger = logging.getLogger(__name__)
 
 cls_cache = {}
 

src/pai_rag/core/rag_service.py

@@ -13,10 +13,9 @@
 )
 from openinference.instrumentation import using_attributes
 from typing import Dict, List
-import logging
+from loguru import logger
 
 TASK_STATUS_FILE = "__upload_task_status.tmp"
-logger = logging.getLogger(__name__)
 
 
 def trace_correlation_id(function):