v0.1.0-20240802 release

.github/workflows/ci.yml

@@ -0,0 +1,55 @@
+name: PAI-RAG CI Build
+
+on:
+  push:
+    # Sequence of patterns matched against refs/heads
+    branches:
+      - main
+      - feature
+      - "releases/**"
+
+permissions:
+  contents: read
+  pull-requests: write
+
+jobs:
+  build:
+    name: Build and Test
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python 3.10
+        # This is the version of the action for setting up Python, not the Python version.
+        uses: actions/setup-python@v5
+        with:
+          # Semantic version range syntax or exact version of a Python version
+          python-version: "3.10"
+          # Optional - x64 or x86 architecture, defaults to x64
+          architecture: "x64"
+
+      - name: Install Dependencies
+        run: |
+          python -m pip install --upgrade pip setuptools wheel
+          pip install poetry
+          poetry install
+        env:
+          POETRY_VIRTUALENVS_CREATE: false
+
+      - name: Install pre-commit
+        shell: bash
+        run: poetry run pip install pre-commit
+
+      - name: Run Linter
+        shell: bash
+        run: poetry run make lint
+
+      - name: Run Tests
+        run: |
+          make coveragetest
+        env:
+          DASHSCOPE_API_KEY: ${{ secrets.TESTDASHSCOPEKEY }}
+          IS_PAI_RAG_CI_TEST: true
+          PAIRAG_RAG__embedding__source: "DashScope"
+          PAIRAG_RAG__llm__source: "DashScope"
+          PAIRAG_RAG__llm__name: "qwen-turbo"

.github/workflows/docker.yml

@@ -3,13 +3,13 @@ name: Create and publish a Docker image
 
 # Configures this workflow to run every time a change is pushed to the branch called `release`.
 on:
+  workflow_dispatch:
   push:
     branches: ["feature"]
 
 # Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds.
 env:
   REGISTRY: registry.cn-beijing.aliyuncs.com
-  REGISTRY_HZ: registry.cn-hangzhou.aliyuncs.com
 
 # There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu.
 jobs:
@@ -19,6 +19,12 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v4
 
+      - name: Extract version
+        run: |
+          pip install poetry
+          VERSION_TAG=$(poetry version --short)
+          echo "VERSION_TAG=$VERSION_TAG" >> $GITHUB_ENV
+
       # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here.
       - name: Login to ACR Beijing region
         uses: docker/login-action@v1
@@ -27,27 +33,30 @@ jobs:
           username: ${{ secrets.ACR_USER }}
           password: ${{ secrets.ACR_PASSWORD }}
 
-      - name: Login to ACR Hangzhou region
-        uses: docker/login-action@v1
-        with:
-          registry: ${{ env.REGISTRY_HZ }}
-          username: ${{ secrets.ACR_USER }}
-          password: ${{ secrets.ACR_PASSWORD }}
-
       - name: Build and push base image
         env:
-          IMAGE_TAG: 0.0.2
+          IMAGE_TAG: ${{env.VERSION_TAG}}
         run: |
           docker build -t ${{ env.REGISTRY }}/mybigpai/pairag:$IMAGE_TAG .
-          docker tag ${{ env.REGISTRY }}/mybigpai/pairag:$IMAGE_TAG ${{ env.REGISTRY_HZ }}/mybigpai/pairag:$IMAGE_TAG
           docker push ${{ env.REGISTRY }}/mybigpai/pairag:$IMAGE_TAG
-          docker push ${{ env.REGISTRY_HZ }}/mybigpai/pairag:$IMAGE_TAG
 
       - name: Build and push GPU image
         env:
-          IMAGE_TAG: 0.0.2_gpu
+          IMAGE_TAG: ${{env.VERSION_TAG}}-gpu
         run: |
           docker build -t ${{ env.REGISTRY }}/mybigpai/pairag:$IMAGE_TAG -f Dockerfile_gpu .
-          docker tag ${{ env.REGISTRY }}/mybigpai/pairag:$IMAGE_TAG ${{ env.REGISTRY_HZ }}/mybigpai/pairag:$IMAGE_TAG
           docker push ${{ env.REGISTRY }}/mybigpai/pairag:$IMAGE_TAG
-          docker push ${{ env.REGISTRY_HZ }}/mybigpai/pairag:$IMAGE_TAG
+
+      - name: Build and push UI image
+        env:
+          IMAGE_TAG: ${{env.VERSION_TAG}}-ui
+        run: |
+          docker build -t ${{ env.REGISTRY }}/mybigpai/pairag:$IMAGE_TAG -f Dockerfile_ui .
+          docker push ${{ env.REGISTRY }}/mybigpai/pairag:$IMAGE_TAG
+
+      - name: Build and push nginx image
+        env:
+          IMAGE_TAG: ${{env.VERSION_TAG}}-nginx
+        run: |
+          docker build -t ${{ env.REGISTRY }}/mybigpai/pairag:$IMAGE_TAG -f Dockerfile_nginx .
+          docker push ${{ env.REGISTRY }}/mybigpai/pairag:$IMAGE_TAG

.github/workflows/main.yml

@@ -1,4 +1,4 @@
-name: PAI-RAG CI
+name: PR Build
 
 on:
   pull_request:

.github/workflows/main_gpu.yml

@@ -0,0 +1,56 @@
+name: PR Build (GPU)
+
+on:
+  pull_request:
+    # Sequence of patterns matched against refs/heads
+    branches:
+      - main
+      - feature
+      - "releases/**"
+
+permissions:
+  contents: read
+  pull-requests: write
+
+jobs:
+  build:
+    name: Build and Test GPU version
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python 3.10
+        # This is the version of the action for setting up Python, not the Python version.
+        uses: actions/setup-python@v5
+        with:
+          # Semantic version range syntax or exact version of a Python version
+          python-version: "3.10"
+          # Optional - x64 or x86 architecture, defaults to x64
+          architecture: "x64"
+
+      - name: Install Dependencies
+        run: |
+          mv pyproject_gpu.toml pyproject.toml && rm poetry.lock
+          python -m pip install --upgrade pip setuptools wheel
+          pip install poetry
+          poetry install
+        env:
+          POETRY_VIRTUALENVS_CREATE: false
+
+      - name: Install pre-commit
+        shell: bash
+        run: poetry run pip install pre-commit
+
+      - name: Run Linter
+        shell: bash
+        run: poetry run make lint
+
+      - name: Run Tests
+        run: |
+          make coveragetest
+        env:
+          DASHSCOPE_API_KEY: ${{ secrets.TESTDASHSCOPEKEY }}
+          IS_PAI_RAG_CI_TEST: true
+          PAIRAG_RAG__embedding__source: "DashScope"
+          PAIRAG_RAG__llm__source: "DashScope"
+          PAIRAG_RAG__llm__name: "qwen-turbo"

.github/workflows/release.yml

@@ -0,0 +1,81 @@
+#
+name: Release image
+
+# Configures this workflow to run every time a change is pushed to the branch called `release`.
+on:
+  workflow_dispatch:
+  push:
+    branches: ["main", "release_test"]
+
+# Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds.
+env:
+  REGISTRY: mybigpai-public-registry.cn-beijing.cr.aliyuncs.com
+
+# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu.
+jobs:
+  build-and-push-image:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+
+      - name: Extract version
+        run: |
+          pip install poetry
+          VERSION_TAG=$(poetry version --short)
+          SPECIFIC_VERSION_TAG="$VERSION_TAG-$(date +'%Y%m%d')"
+          echo "VERSION_TAG=$VERSION_TAG" >> $GITHUB_ENV
+          echo "SPECIFIC_VERSION_TAG=$SPECIFIC_VERSION_TAG" >> $GITHUB_ENV
+          echo "version:$SPECIFIC_VERSION_TAG\ncommit_id:$(git rev-parse HEAD)" >  __build_version.cfg
+
+      # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here.
+      - name: Login to ACR Hangzhou region
+        uses: docker/login-action@v1
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ secrets.ACR_USER }}
+          password: ${{ secrets.ACR_PUBLIC_PASSWORD }}
+
+      - name: Build and push base image
+        env:
+          IMAGE_TAG: ${{env.VERSION_TAG}}
+          SPECIFIC_IMAGE_TAG: ${{env.SPECIFIC_VERSION_TAG}}
+        run: |
+          docker build -t ${{ env.REGISTRY }}/mybigpai/pairag:${{ env.IMAGE_TAG }} .
+          docker push ${{ env.REGISTRY }}/mybigpai/pairag:${{ env.IMAGE_TAG }}
+          docker tag ${{ env.REGISTRY }}/mybigpai/pairag:${{ env.IMAGE_TAG }} ${{ env.REGISTRY }}/mybigpai/pairag:${{ env.SPECIFIC_IMAGE_TAG }}
+          docker push ${{ env.REGISTRY }}/mybigpai/pairag:${{ env.SPECIFIC_IMAGE_TAG }}
+
+      - name: Build and push GPU image
+        env:
+          IMAGE_TAG: ${{env.VERSION_TAG}}-gpu
+          SPECIFIC_IMAGE_TAG: ${{env.SPECIFIC_VERSION_TAG}}-gpu
+        run: |
+          docker build -t ${{ env.REGISTRY }}/mybigpai/pairag:${{ env.IMAGE_TAG }} -f Dockerfile_gpu .
+          docker push ${{ env.REGISTRY }}/mybigpai/pairag:${{ env.IMAGE_TAG }}
+          docker tag ${{ env.REGISTRY }}/mybigpai/pairag:${{ env.IMAGE_TAG }} ${{ env.REGISTRY }}/mybigpai/pairag:${{ env.SPECIFIC_IMAGE_TAG }}
+          docker push ${{ env.REGISTRY }}/mybigpai/pairag:${{ env.SPECIFIC_IMAGE_TAG }}
+
+      - name: Build and push UI image
+        env:
+          IMAGE_TAG: ${{env.VERSION_TAG}}-ui
+          SPECIFIC_IMAGE_TAG: ${{env.SPECIFIC_VERSION_TAG}}-ui
+        run: |
+          docker build -t ${{ env.REGISTRY }}/mybigpai/pairag:${{ env.IMAGE_TAG }} -f Dockerfile_ui .
+          docker push ${{ env.REGISTRY }}/mybigpai/pairag:${{ env.IMAGE_TAG }}
+          docker tag ${{ env.REGISTRY }}/mybigpai/pairag:${{ env.IMAGE_TAG }} ${{ env.REGISTRY }}/mybigpai/pairag:${{ env.SPECIFIC_IMAGE_TAG }}
+          docker push ${{ env.REGISTRY }}/mybigpai/pairag:${{ env.SPECIFIC_IMAGE_TAG }}
+
+      - name: Build and push nginx image
+        env:
+          IMAGE_TAG: ${{env.VERSION_TAG}}-nginx
+          SPECIFIC_IMAGE_TAG: ${{env.SPECIFIC_VERSION_TAG}}-nginx
+        run: |
+          docker build -t ${{ env.REGISTRY }}/mybigpai/pairag:${{ env.IMAGE_TAG }} -f Dockerfile_nginx .
+          docker push ${{ env.REGISTRY }}/mybigpai/pairag:${{ env.IMAGE_TAG }}
+          docker tag ${{ env.REGISTRY }}/mybigpai/pairag:${{ env.IMAGE_TAG }} ${{ env.REGISTRY }}/mybigpai/pairag:${{ env.SPECIFIC_IMAGE_TAG }}
+          docker push ${{ env.REGISTRY }}/mybigpai/pairag:${{ env.SPECIFIC_IMAGE_TAG }}

.gitignore

@@ -222,3 +222,6 @@ output
 *.local.toml
 
 localdata/
+model_repository/
+*.tmp
+__*

.pre-commit-config.yaml

@@ -85,6 +85,8 @@ repos:
           (\/.*?\.[\w:]+)/poetry.lock
         args:
           [
+            "--exclude",
+            "./data/tokenization/qwen.tiktoken",
             "--ignore-words-list",
             "astroid,gallary,momento,narl,ot,rouge,nin,gere",
           ]

Dockerfile

@@ -13,6 +13,9 @@ COPY . .
 RUN poetry install && rm -rf $POETRY_CACHE_DIR
 
 FROM python:3.10-slim AS prod
+
+RUN rm -rf /etc/localtime && ln -s /usr/share/zoneinfo/Asia/Harbin  /etc/localtime
+
 ENV VIRTUAL_ENV=/app/.venv \
     PATH="/app/.venv/bin:$PATH"
 
@@ -21,4 +24,4 @@ RUN apt-get update && apt-get install -y libgl1 libglib2.0-0
 WORKDIR /app
 COPY . .
 COPY --from=builder ${VIRTUAL_ENV} ${VIRTUAL_ENV}
-ENTRYPOINT ["pai_rag", "run"]
+CMD ["pai_rag", "run"]

Dockerfile_gpu

@@ -15,6 +15,9 @@ RUN mv pyproject_gpu.toml pyproject.toml \
 RUN poetry install && rm -rf $POETRY_CACHE_DIR
 
 FROM python:3.10-slim AS prod
+
+RUN rm -rf /etc/localtime && ln -s /usr/share/zoneinfo/Asia/Harbin  /etc/localtime
+
 ENV VIRTUAL_ENV=/app/.venv \
     PATH="/app/.venv/bin:$PATH"
 
@@ -23,4 +26,4 @@ RUN apt-get update && apt-get install -y libgl1 libglib2.0-0
 WORKDIR /app
 COPY . .
 COPY --from=builder ${VIRTUAL_ENV} ${VIRTUAL_ENV}
-ENTRYPOINT ["pai_rag", "run"]
+CMD ["pai_rag", "run"]

Dockerfile_nginx

@@ -0,0 +1,3 @@
+FROM nginx:latest
+COPY ./nginx/default.conf etc/nginx/conf.d/default.conf
+COPY ./nginx/nginx.conf etc/nginx/nginx.conf

Dockerfile_ui

@@ -0,0 +1,27 @@
+FROM python:3.10-slim AS builder
+
+RUN pip3 install poetry
+
+ENV POETRY_NO_INTERACTION=1 \
+    POETRY_VIRTUALENVS_IN_PROJECT=1 \
+    POETRY_VIRTUALENVS_CREATE=1 \
+    POETRY_CACHE_DIR=/tmp/poetry_cache
+
+WORKDIR /app
+COPY . .
+
+RUN poetry install && rm -rf $POETRY_CACHE_DIR
+
+FROM python:3.10-slim AS prod
+
+RUN rm -rf /etc/localtime && ln -s /usr/share/zoneinfo/Asia/Harbin  /etc/localtime
+
+ENV VIRTUAL_ENV=/app/.venv \
+    PATH="/app/.venv/bin:$PATH"
+
+RUN apt-get update && apt-get install -y libgl1 libglib2.0-0
+
+WORKDIR /app
+COPY . .
+COPY --from=builder ${VIRTUAL_ENV} ${VIRTUAL_ENV}
+CMD ["pai_rag", "ui"]