Add release pipeline and fix some bugs (#137)

* Fix bug * Add release pipeline * Update * Update * Fix bug * Fix login * Fix empty tag * Update * Fix ui issue * Add base version tag * Fix specific version * Use pg hybrid retrieval directly * Fix image tag
aigc-apps · Aug 1, 2024 · 25e2ffb · 25e2ffb
1 parent b313fe8
commit 25e2ffb
Show file tree

Hide file tree

Showing 8 changed files with 132 additions and 43 deletions.
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
@@ -10,7 +10,6 @@ on:
 # Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds.
 env:
   REGISTRY: registry.cn-beijing.aliyuncs.com
-  REGISTRY_HZ: registry.cn-hangzhou.aliyuncs.com
 
 # There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu.
 jobs:
@@ -20,6 +19,12 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v4
 
+      - name: Extract version
+        run: |
+          pip install poetry
+          VERSION_TAG=$(poetry version --short)
+          echo "VERSION_TAG=$VERSION_TAG" >> $GITHUB_ENV
+
       # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here.
       - name: Login to ACR Beijing region
         uses: docker/login-action@v1
@@ -28,45 +33,30 @@ jobs:
           username: ${{ secrets.ACR_USER }}
           password: ${{ secrets.ACR_PASSWORD }}
 
-      - name: Login to ACR Hangzhou region
-        uses: docker/login-action@v1
-        with:
-          registry: ${{ env.REGISTRY_HZ }}
-          username: ${{ secrets.ACR_USER }}
-          password: ${{ secrets.ACR_PASSWORD }}
-
       - name: Build and push base image
         env:
-          IMAGE_TAG: 0.0.2
+          IMAGE_TAG: ${{env.VERSION_TAG}}
         run: |
           docker build -t ${{ env.REGISTRY }}/mybigpai/pairag:$IMAGE_TAG .
-          docker tag ${{ env.REGISTRY }}/mybigpai/pairag:$IMAGE_TAG ${{ env.REGISTRY_HZ }}/mybigpai/pairag:$IMAGE_TAG
           docker push ${{ env.REGISTRY }}/mybigpai/pairag:$IMAGE_TAG
-          docker push ${{ env.REGISTRY_HZ }}/mybigpai/pairag:$IMAGE_TAG
 
       - name: Build and push GPU image
         env:
-          IMAGE_TAG: 0.0.2_gpu
+          IMAGE_TAG: ${{env.VERSION_TAG}}-gpu
         run: |
           docker build -t ${{ env.REGISTRY }}/mybigpai/pairag:$IMAGE_TAG -f Dockerfile_gpu .
-          docker tag ${{ env.REGISTRY }}/mybigpai/pairag:$IMAGE_TAG ${{ env.REGISTRY_HZ }}/mybigpai/pairag:$IMAGE_TAG
           docker push ${{ env.REGISTRY }}/mybigpai/pairag:$IMAGE_TAG
-          docker push ${{ env.REGISTRY_HZ }}/mybigpai/pairag:$IMAGE_TAG
 
       - name: Build and push UI image
         env:
-          IMAGE_TAG: 0.0.2_ui
+          IMAGE_TAG: ${{env.VERSION_TAG}}-ui
         run: |
           docker build -t ${{ env.REGISTRY }}/mybigpai/pairag:$IMAGE_TAG -f Dockerfile_ui .
-          docker tag ${{ env.REGISTRY }}/mybigpai/pairag:$IMAGE_TAG ${{ env.REGISTRY_HZ }}/mybigpai/pairag:$IMAGE_TAG
           docker push ${{ env.REGISTRY }}/mybigpai/pairag:$IMAGE_TAG
-          docker push ${{ env.REGISTRY_HZ }}/mybigpai/pairag:$IMAGE_TAG
 
       - name: Build and push nginx image
         env:
-          IMAGE_TAG: 0.0.2_nginx
+          IMAGE_TAG: ${{env.VERSION_TAG}}-nginx
         run: |
           docker build -t ${{ env.REGISTRY }}/mybigpai/pairag:$IMAGE_TAG -f Dockerfile_nginx .
-          docker tag ${{ env.REGISTRY }}/mybigpai/pairag:$IMAGE_TAG ${{ env.REGISTRY_HZ }}/mybigpai/pairag:$IMAGE_TAG
           docker push ${{ env.REGISTRY }}/mybigpai/pairag:$IMAGE_TAG
-          docker push ${{ env.REGISTRY_HZ }}/mybigpai/pairag:$IMAGE_TAG
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -0,0 +1,81 @@
+#
+name: Release image
+
+# Configures this workflow to run every time a change is pushed to the branch called `release`.
+on:
+  workflow_dispatch:
+  push:
+    branches: ["main", "release_test"]
+
+# Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds.
+env:
+  REGISTRY: mybigpai-public-registry.cn-beijing.cr.aliyuncs.com
+
+# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu.
+jobs:
+  build-and-push-image:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+
+      - name: Extract version
+        run: |
+          pip install poetry
+          VERSION_TAG=$(poetry version --short)
+          SPECIFIC_VERSION_TAG="$VERSION_TAG-$(date +'%Y%m%d')"
+          echo "VERSION_TAG=$VERSION_TAG" >> $GITHUB_ENV
+          echo "SPECIFIC_VERSION_TAG=$SPECIFIC_VERSION_TAG" >> $GITHUB_ENV
+          echo "version:$SPECIFIC_VERSION_TAG\ncommit_id:$(git rev-parse HEAD)" >  __build_version.cfg
+
+      # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here.
+      - name: Login to ACR Hangzhou region
+        uses: docker/login-action@v1
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ secrets.ACR_USER }}
+          password: ${{ secrets.ACR_PUBLIC_PASSWORD }}
+
+      - name: Build and push base image
+        env:
+          IMAGE_TAG: ${{env.VERSION_TAG}}
+          SPECIFIC_IMAGE_TAG: ${{env.SPECIFIC_VERSION_TAG}}
+        run: |
+          docker build -t ${{ env.REGISTRY }}/mybigpai/pairag:${{ env.IMAGE_TAG }} .
+          docker push ${{ env.REGISTRY }}/mybigpai/pairag:${{ env.IMAGE_TAG }}
+          docker tag ${{ env.REGISTRY }}/mybigpai/pairag:${{ env.IMAGE_TAG }} ${{ env.REGISTRY }}/mybigpai/pairag:${{ env.SPECIFIC_IMAGE_TAG }}
+          docker push ${{ env.REGISTRY }}/mybigpai/pairag:${{ env.SPECIFIC_IMAGE_TAG }}
+
+      - name: Build and push GPU image
+        env:
+          IMAGE_TAG: ${{env.VERSION_TAG}}-gpu
+          SPECIFIC_IMAGE_TAG: ${{env.SPECIFIC_VERSION_TAG}}-gpu
+        run: |
+          docker build -t ${{ env.REGISTRY }}/mybigpai/pairag:${{ env.IMAGE_TAG }} -f Dockerfile_gpu .
+          docker push ${{ env.REGISTRY }}/mybigpai/pairag:${{ env.IMAGE_TAG }}
+          docker tag ${{ env.REGISTRY }}/mybigpai/pairag:${{ env.IMAGE_TAG }} ${{ env.REGISTRY }}/mybigpai/pairag:${{ env.SPECIFIC_IMAGE_TAG }}
+          docker push ${{ env.REGISTRY }}/mybigpai/pairag:${{ env.SPECIFIC_IMAGE_TAG }}
+
+      - name: Build and push UI image
+        env:
+          IMAGE_TAG: ${{env.VERSION_TAG}}-ui
+          SPECIFIC_IMAGE_TAG: ${{env.SPECIFIC_VERSION_TAG}}-ui
+        run: |
+          docker build -t ${{ env.REGISTRY }}/mybigpai/pairag:${{ env.IMAGE_TAG }} -f Dockerfile_ui .
+          docker push ${{ env.REGISTRY }}/mybigpai/pairag:${{ env.IMAGE_TAG }}
+          docker tag ${{ env.REGISTRY }}/mybigpai/pairag:${{ env.IMAGE_TAG }} ${{ env.REGISTRY }}/mybigpai/pairag:${{ env.SPECIFIC_IMAGE_TAG }}
+          docker push ${{ env.REGISTRY }}/mybigpai/pairag:${{ env.SPECIFIC_IMAGE_TAG }}
+
+      - name: Build and push nginx image
+        env:
+          IMAGE_TAG: ${{env.VERSION_TAG}}-nginx
+          SPECIFIC_IMAGE_TAG: ${{env.SPECIFIC_VERSION_TAG}}-nginx
+        run: |
+          docker build -t ${{ env.REGISTRY }}/mybigpai/pairag:${{ env.IMAGE_TAG }} -f Dockerfile_nginx .
+          docker push ${{ env.REGISTRY }}/mybigpai/pairag:${{ env.IMAGE_TAG }}
+          docker tag ${{ env.REGISTRY }}/mybigpai/pairag:${{ env.IMAGE_TAG }} ${{ env.REGISTRY }}/mybigpai/pairag:${{ env.SPECIFIC_IMAGE_TAG }}
+          docker push ${{ env.REGISTRY }}/mybigpai/pairag:${{ env.SPECIFIC_IMAGE_TAG }}
diff --git a/.gitignore b/.gitignore
@@ -224,3 +224,4 @@ output
 localdata/
 model_repository/
 *.tmp
+__*
diff --git a/README.md b/README.md
@@ -162,7 +162,7 @@ To make it easier to use and save time on environment installation, we also prov
 - CPU
 
   ```bash
-  docker pull mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/mybigpai/pairag:0.0.2
+  docker pull mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/mybigpai/pairag:0.1.0
 
   # -p (port) -v (mount embedding and rerank model directories) -e (set environment variables, if using Dashscope LLM/Embedding, need to be introduced) -w (number of workers, can be specified as the approximate number of CPU cores)
   docker run --name pai_rag \
@@ -171,13 +171,13 @@ To make it easier to use and save time on environment installation, we also prov
               -v /your_local_documents_path:/data \
               -e DASHSCOPE_API_KEY=${DASHSCOPE_API_KEY} \
               -d \
-              mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/mybigpai/pairag:0.0.2 gunicorn -b 0.0.0.0:8001 -w 16 -k uvicorn.workers.UvicornH11Worker pai_rag.main:app
+              mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/mybigpai/pairag:0.1.0 gunicorn -b 0.0.0.0:8001 -w 16 -k uvicorn.workers.UvicornH11Worker pai_rag.main:app
   ```
 
 - GPU
 
   ```bash
-  docker pull mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/mybigpai/pairag:0.0.2_gpu
+  docker pull mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/mybigpai/pairag:0.1.0-gpu
 
   # -p (port) -v (mount embedding and rerank model directories) -e (set environment variables, if using Dashscope LLM/Embedding, you need to introduce it) -w (number of workers, which can be specified as the approximate number of CPU cores)
   docker run --name pai_rag \
@@ -187,7 +187,7 @@ To make it easier to use and save time on environment installation, we also prov
               --gpus all \
               -e DASHSCOPE_API_KEY=${DASHSCOPE_API_KEY} \
               -d \
-              mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/mybigpai/pairag:0.0.2_gpu gunicorn -b 0.0.0.0:8001 -w 16 -k uvicorn.workers.UvicornH11Worker pai_rag.main:app
+              mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/mybigpai/pairag:0.1.0-gpu gunicorn -b 0.0.0.0:8001 -w 16 -k uvicorn.workers.UvicornH11Worker pai_rag.main:app
   ```
 
 2. Load Data
@@ -211,17 +211,17 @@ To make it easier to use and save time on environment installation, we also prov
    Linux:
 
 ```bash
-docker pull mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/mybigpai/pairag:0.0.2_ui
+docker pull mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/mybigpai/pairag:0.1.0-ui
 
-docker run --network host -d mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/mybigpai/pairag:0.0.2_ui
+docker run --network host -d mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/mybigpai/pairag:0.1.0-ui
 ```
 
 Mac/Windows:
 
 ```bash
-docker pull mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/mybigpai/pairag:0.0.2_ui
+docker pull mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/mybigpai/pairag:0.1.0-ui
 
-docker run -p 8002:8002 -d mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/mybigpai/pairag:0.0.2_ui pai_rag ui -p 8002 -c http://host.docker.internal:8001/
+docker run -p 8002:8002 -d mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/mybigpai/pairag:0.1.0_ui pai_rag ui -p 8002 -c http://host.docker.internal:8001/
 ```
 
 You can also open http://127.0.0.1:8002/ to configure the RAG service and upload local data.

diff --git a/README_zh.md b/README_zh.md
@@ -162,36 +162,36 @@ evaluation [-c src/pai_rag/config/settings.yaml] [-o False] [-t retrieval]
 - CPU
 
   ```bash
-  docker pull mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/mybigpai/pairag:0.0.2
+  docker pull mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/mybigpai/pairag:0.1.0
 
   # 启动: -p(端口) -v(挂载embedding和rerank模型目录) -e(设置环境变量，若使用Dashscope LLM/Embedding，需要引入) -w(worker数量，可以指定为近似cpu核数)
-  docker run -p 8001:8001 -v /huggingface:/huggingface -e DASHSCOPE_API_KEY=sk-xxxx -d mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/mybigpai/pairag:0.0.2 gunicorn -b 0.0.0.0:8001 -w 16 -k uvicorn.workers.UvicornH11Worker pai_rag.main:app
+  docker run -p 8001:8001 -v /huggingface:/huggingface -e DASHSCOPE_API_KEY=sk-xxxx -d mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/mybigpai/pairag:0.1.0 gunicorn -b 0.0.0.0:8001 -w 16 -k uvicorn.workers.UvicornH11Worker pai_rag.main:app
   ```
 
 - GPU
 
   ```bash
-  docker pull mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/mybigpai/pairag:0.0.2_gpu
+  docker pull mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/mybigpai/pairag:0.1.0-gpu
 
   # 启动: -p(端口) -v(挂载embedding和rerank模型目录) -e(设置环境变量，若使用Dashscope LLM/Embedding，需要引入) -w(worker数量，可以指定为近似cpu核数)
-  docker run -p 8001:8001 -v /huggingface:/huggingface --gpus all -e DASHSCOPE_API_KEY=sk-xxxx -d mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/mybigpai/pairag:0.0.2_gpu gunicorn -b 0.0.0.0:8001 -w 16 -k uvicorn.workers.UvicornH11Worker pai_rag.main:app
+  docker run -p 8001:8001 -v /huggingface:/huggingface --gpus all -e DASHSCOPE_API_KEY=sk-xxxx -d mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/mybigpai/pairag:0.1.0-gpu gunicorn -b 0.0.0.0:8001 -w 16 -k uvicorn.workers.UvicornH11Worker pai_rag.main:app
   ```
 
 2. 启动RAG WebUI
    Linux:
 
 ```bash
-docker pull mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/mybigpai/pairag:0.0.2_ui
+docker pull mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/mybigpai/pairag:0.1.0-ui
 
-docker run --network host -d mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/mybigpai/pairag:0.0.2_ui
+docker run --network host -d mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/mybigpai/pairag:0.1.0-ui
 ```
 
 Mac/Windows:
 
 ```bash
-docker pull mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/mybigpai/pairag:0.0.2_ui
+docker pull mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/mybigpai/pairag:0.1.0-ui
 
-docker run -p 8002:8002 -d mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/mybigpai/pairag:0.0.2_ui pai_rag ui -p 8002 -c http://host.docker.internal:8001/
+docker run -p 8002:8002 -d mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/mybigpai/pairag:0.1.0-ui pai_rag ui -p 8002 -c http://host.docker.internal:8001/
 ```
 
 ### 基于Dockerfile自行构建镜像

diff --git a/src/pai_rag/app/web/tabs/chat_tab.py b/src/pai_rag/app/web/tabs/chat_tab.py
@@ -149,7 +149,7 @@ def create_chat_tab() -> Dict[str, Any]:
                     def change_weight(change_weight):
                         return round(float(1 - change_weight), 2)
 
-                    vector_weight.change(
+                    vector_weight.input(
                         fn=change_weight,
                         inputs=vector_weight,
                         outputs=[keyword_weight],
@@ -178,13 +178,13 @@ def change_retrieval_mode(retrieval_mode):
                         else:
                             return {simple_reranker_col: gr.update(visible=False)}
 
-                    reranker_type.change(
+                    reranker_type.input(
                         fn=change_reranker_type,
                         inputs=reranker_type,
                         outputs=[simple_reranker_col, model_reranker_col],
                     )
 
-                    retrieval_mode.change(
+                    retrieval_mode.input(
                         fn=change_retrieval_mode,
                         inputs=retrieval_mode,
                         outputs=[simple_reranker_col],
@@ -232,7 +232,7 @@ def change_retrieval_mode(retrieval_mode):
                 )
                 text_qa_template = gr.Textbox(
                     label="prompt template",
-                    placeholder="",
+                    value="",
                     elem_id="text_qa_template",
                     lines=4,
                 )
@@ -266,7 +266,7 @@ def change_prompt_template(prm_type):
                     else:
                         return {text_qa_template: gr.update(value="", interactive=True)}
 
-                prm_type.change(
+                prm_type.input(
                     fn=change_prompt_template,
                     inputs=prm_type,
                     outputs=[text_qa_template],
@@ -304,7 +304,7 @@ def change_query_radio(query_type):
                         lc_col: gr.update(visible=True),
                     }
 
-            query_type.change(
+            query_type.input(
                 fn=change_query_radio,
                 inputs=query_type,
                 outputs=[vs_col, vec_model_argument, llm_col, model_argument, lc_col],

diff --git a/src/pai_rag/modules/index/bm25_index.py b/src/pai_rag/modules/index/bm25_index.py
@@ -22,7 +22,12 @@ def get_dependencies() -> List[str]:
 
     def _create_new_instance(self, new_params: Dict[str, Any]):
         index = new_params["IndexModule"]
-        if index.vectordb_type == "elasticsearch" or index.vectordb_type == "milvus":
+        if (
+            index.vectordb_type == "elasticsearch"
+            or index.vectordb_type == "milvus"
+            or index.vectordb_type == "postgresql"
+        ):
+            logger.info(f"Do not use local BM25 Index for {index.vectordb_type}.")
             return None
         else:
             logger.info("Using BM25 Index.")

diff --git a/src/pai_rag/modules/retriever/retriever.py b/src/pai_rag/modules/retriever/retriever.py
@@ -70,6 +70,18 @@ def _create_new_instance(self, new_params: Dict[str, Any]):
                     similarity_top_k=similarity_top_k,
                     vector_store_query_mode=VectorStoreQueryMode.TEXT_SEARCH,
                 )
+        elif index.vectordb_type == "postgresql":
+            if retrieval_mode == "embedding":
+                query_mode = VectorStoreQueryMode.DEFAULT
+            elif retrieval_mode == "keyword":
+                query_mode = VectorStoreQueryMode.TEXT_SEARCH
+            else:
+                query_mode = VectorStoreQueryMode.HYBRID
+            return MyVectorIndexRetriever(
+                index=index.vector_index,
+                similarity_top_k=similarity_top_k,
+                vector_store_query_mode=query_mode,
+            )
         else:
             vector_retriever = MyVectorIndexRetriever(
                 index=index.vector_index, similarity_top_k=similarity_top_k
-Original file line number
+Diff line change
@@ Expand Up / @@ -224,3 +224,4 @@ output @@
     localdata/
     model_repository/
     *.tmp
+    __*