Skip to content

Commit

Permalink
Merge branch 'dev' of https://github.com/janhq/cortex.cpp into feat/m…
Browse files Browse the repository at this point in the history 
…odel_estimation
  • Loading branch information
@sangjanai
sangjanai committed Dec 2, 2024
2 parents f2651ac + 9c95577 commit 22826b9
Show file tree
Hide file tree
Showing 90 changed files with 2,109 additions and 1,102 deletions.
118 changes: 83 additions & 35 deletions .github/workflows/cortex-cpp-quality-gate.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@ on:
types: [opened, synchronize, reopened, ready_for_review]
paths: ["engine/**", ".github/workflows/cortex-cpp-quality-gate.yml"]
workflow_dispatch:
schedule:
- cron: '0 22 * * *'

env:
LLM_MODEL_URL: https://delta.jan.ai/tinyllama-1.1b-chat-v0.3.Q2_K.gguf
Expand Down Expand Up @@ -149,6 +151,34 @@ jobs:
env:
GITHUB_TOKEN: ${{ secrets.PAT_SERVICE_ACCOUNT }}

- name: Run e2e tests
if: github.event_name == 'schedule' && runner.os != 'Windows' && github.event.pull_request.draft == false
run: |
cd engine
cp build/cortex build/cortex-nightly
cp build/cortex build/cortex-beta
python -m pip install --upgrade pip
python -m pip install -r e2e-test/requirements.txt
python e2e-test/cortex-llamacpp-e2e-nightly.py
rm build/cortex-nightly
rm build/cortex-beta
env:
GITHUB_TOKEN: ${{ secrets.PAT_SERVICE_ACCOUNT }}

- name: Run e2e tests
if: github.event_name == 'schedule' && runner.os == 'Windows' && github.event.pull_request.draft == false
run: |
cd engine
cp build/cortex.exe build/cortex-nightly.exe
cp build/cortex.exe build/cortex-beta.exe
python -m pip install --upgrade pip
python -m pip install -r e2e-test/requirements.txt
python e2e-test/cortex-llamacpp-e2e-nightly.py
rm build/cortex-nightly.exe
rm build/cortex-beta.exe
env:
GITHUB_TOKEN: ${{ secrets.PAT_SERVICE_ACCOUNT }}

- name: Pre-package
run: |
cd engine
Expand Down Expand Up @@ -188,40 +218,58 @@ jobs:
AWS_SECRET_ACCESS_KEY: "${{ secrets.MINIO_SECRET_ACCESS_KEY }}"
AWS_DEFAULT_REGION: "${{ secrets.MINIO_REGION }}"

# build-docker-and-test:
# runs-on: ubuntu-latest
# steps:
# - name: Getting the repo
# uses: actions/checkout@v3
# with:
# submodules: 'recursive'
build-docker-and-test:
runs-on: ubuntu-24-04-docker
steps:
- name: Getting the repo
uses: actions/checkout@v3
with:
submodules: 'recursive'

- name: Run Docker
if: github.event_name != 'schedule'
run: |
docker build \
--build-arg REMOTE_CACHE_URL="${{ secrets.MINIO_ENDPOINT }}/vcpkg-cache" \
--build-arg MINIO_ENDPOINT_URL="${{ secrets.MINIO_ENDPOINT }}" \
--build-arg MINIO_ACCESS_KEY="${{ secrets.MINIO_ACCESS_KEY_ID }}" \
--build-arg MINIO_SECRET_KEY="${{ secrets.MINIO_SECRET_ACCESS_KEY }}" \
-t menloltd/cortex:test -f docker/Dockerfile.cache .
docker run -it -d -p 3928:39281 --name cortex menloltd/cortex:test
sleep 20
# - name: Set up QEMU
# uses: docker/setup-qemu-action@v3
- name: Run Docker
if: github.event_name == 'schedule'
run: |
latest_prerelease=$(curl -s https://api.github.com/repos/cortexcpp/cortex.cpp/releases | jq -r '.[] | select(.prerelease == true) | .tag_name' | head -n 1)
echo "cortex.llamacpp latest release: $latest_prerelease"
docker build \
--build-arg REMOTE_CACHE_URL="${{ secrets.MINIO_ENDPOINT }}/vcpkg-cache" \
--build-arg MINIO_ENDPOINT_URL="${{ secrets.MINIO_ENDPOINT }}" \
--build-arg MINIO_ACCESS_KEY="${{ secrets.MINIO_ACCESS_KEY_ID }}" \
--build-arg MINIO_SECRET_KEY="${{ secrets.MINIO_SECRET_ACCESS_KEY }}" \
--build-arg CORTEX_CPP_VERSION="${latest_prerelease}" \
-t menloltd/cortex:test -f docker/Dockerfile.cache .
docker run -it -d -p 3928:39281 --name cortex menloltd/cortex:test
sleep 20
# - name: Set up Docker Buildx
# uses: docker/setup-buildx-action@v3

# - name: Run Docker
# run: |
# docker build -t menloltd/cortex:test -f docker/Dockerfile .
# docker run -it -d -p 3928:39281 --name cortex menloltd/cortex:test

# - name: use python
# uses: actions/setup-python@v5
# with:
# python-version: "3.10"

# - name: Run e2e tests
# run: |
# cd engine
# python -m pip install --upgrade pip
# python -m pip install -r e2e-test/requirements.txt
# pytest e2e-test/test_api_docker.py

# - name: Run Docker
# continue-on-error: true
# if: always()
# run: |
# docker stop cortex
# docker rm cortex
- name: use python
uses: actions/setup-python@v5
with:
python-version: "3.10"

- name: Run e2e tests
run: |
cd engine
python -m pip install --upgrade pip
python -m pip install -r e2e-test/requirements.txt
pytest e2e-test/test_api_docker.py
- name: Run Docker
continue-on-error: true
if: always()
run: |
docker logs cortex
docker stop cortex
docker rm cortex
echo "y\n" | docker system prune -af
70 changes: 30 additions & 40 deletions docker/Dockerfile
View file
Original file line number Diff line number Diff line change
@@ -1,16 +1,12 @@
FROM ubuntu:22.04 as base

FROM base as build

ARG CORTEX_CPP_VERSION=latest

ARG CMAKE_EXTRA_FLAGS=""
# Stage 1: Base dependencies (common stage)
FROM ubuntu:22.04 as common

ENV DEBIAN_FRONTEND=noninteractive

# Install dependencies
# Install common dependencies
RUN apt-get update && apt-get install -y --no-install-recommends \
ca-certificates \
software-properties-common \
curl \
wget \
jq \
Expand All @@ -20,71 +16,65 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
apt-get clean && \
rm -rf /var/lib/apt/lists/*

RUN apt-get update && apt-get install -y --no-install-recommends \
# Stage 2: Build dependencies and compilation
FROM common as build

# Install Dependencies
RUN wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | tee /etc/apt/trusted.gpg.d/kitware.gpg >/dev/null && \
apt-add-repository "deb https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" && \
apt-get update && \
apt-get install -y --no-install-recommends \
cmake \
make \
git \
uuid-dev \
lsb-release \
software-properties-common \
gpg \
zip \
unzip \
gcc \
g++ \
ninja-build \
pkg-config \
python3-pip \
openssl && \
pip3 install awscli && \
apt-get clean && \
rm -rf /var/lib/apt/lists/*

RUN wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | tee /etc/apt/trusted.gpg.d/kitware.gpg >/dev/null && \
apt-add-repository "deb https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" && \
apt-get update && \
apt-get install -y cmake && \
apt-get clean && \
rm -rf /var/lib/apt/lists/*
ARG CORTEX_CPP_VERSION=latest
ARG CMAKE_EXTRA_FLAGS=""

WORKDIR /app

# Copy source code
COPY ./engine /app/engine

COPY ./docs/static/openapi/cortex.json /app/docs/static/openapi/cortex.json

# Build project
# Configure vcpkg binary sources
RUN cd engine && make configure-vcpkg && make build CMAKE_EXTRA_FLAGS="-DCORTEX_CPP_VERSION=${CORTEX_CPP_VERSION} -DCMAKE_BUILD_TEST=OFF -DCMAKE_TOOLCHAIN_FILE=vcpkg/scripts/buildsystems/vcpkg.cmake ${CMAKE_EXTRA_FLAGS}"

FROM base as runtime

ENV DEBIAN_FRONTEND=noninteractive

# Install dependencies
RUN apt-get update && apt-get install -y --no-install-recommends \
ca-certificates \
curl \
wget \
jq \
tar \
openmpi-bin \
libopenmpi-dev && \
apt-get clean && \
rm -rf /var/lib/apt/lists/*

ARG CORTEX_LLAMACPP_VERSION=latest

# Stage 3: Runtime
FROM common as runtime
WORKDIR /app
COPY --from=build /app/engine/build/cortex /usr/local/bin/cortex
COPY --from=build /app/engine/build/cortex-server /usr/local/bin/cortex-server

COPY ./docker/download-cortex.llamacpp.sh /tmp/download-cortex.llamacpp.sh
COPY ./docker/entrypoint.sh /usr/local/bin/entrypoint.sh

# Get the latest version of the Cortex Llama
# Get the latest version of Cortex Llama
ARG CORTEX_LLAMACPP_VERSION=latest
RUN chmod +x /tmp/download-cortex.llamacpp.sh && /bin/bash /tmp/download-cortex.llamacpp.sh ${CORTEX_LLAMACPP_VERSION}

# Copy the entrypoint script
COPY ./docker/entrypoint.sh /usr/local/bin/entrypoint.sh

# Configure entrypoint
RUN chmod +x /usr/local/bin/entrypoint.sh

EXPOSE 39281

# Healthcheck
HEALTHCHECK --interval=300s --timeout=30s --start-period=10s --retries=3 \
CMD curl -f http://127.0.0.1:39281/healthz || exit 1

ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
121 changes: 121 additions & 0 deletions docker/Dockerfile.cache
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
# Stage 1: Base dependencies (common stage)
FROM ubuntu:22.04 as common

ENV DEBIAN_FRONTEND=noninteractive

# Install common dependencies
RUN apt-get update && apt-get install -y --no-install-recommends \
ca-certificates \
software-properties-common \
curl \
wget \
jq \
tar \
openmpi-bin \
libopenmpi-dev && \
apt-get clean && \
rm -rf /var/lib/apt/lists/*

# Stage 2: Build dependencies and compilation
FROM common as build

# Install Dependencies
RUN wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | tee /etc/apt/trusted.gpg.d/kitware.gpg >/dev/null && \
apt-add-repository "deb https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" && \
apt-get update && \
apt-get install -y --no-install-recommends \
cmake \
make \
git \
uuid-dev \
lsb-release \
gpg \
zip \
unzip \
gcc \
g++ \
ninja-build \
pkg-config \
python3-pip \
openssl && \
pip3 install awscli && \
apt-get clean && \
rm -rf /var/lib/apt/lists/*

ARG CORTEX_CPP_VERSION=latest
ARG CMAKE_EXTRA_FLAGS=""

WORKDIR /app

# Build arguments for remote cache
ARG REMOTE_CACHE_URL=""
ARG MINIO_ENDPOINT_URL=""
ARG MINIO_ACCESS_KEY=""
ARG MINIO_SECRET_KEY=""

# Configure cache
ENV LOCAL_CACHE_DIR="/vcpkg-cache"
RUN mkdir -p ${LOCAL_CACHE_DIR}

# Configure MinIO alias (if remote cache is provided)
RUN if [ -n "${REMOTE_CACHE_URL}" ]; then \
echo "Setting up MinIO for remote cache..." && \
aws configure set default.s3.signature_version s3v4 && \
aws configure set aws_access_key_id ${MINIO_ACCESS_KEY} && \
aws configure set aws_secret_access_key ${MINIO_SECRET_KEY} && \
aws configure set default.region us-east-1; \
else \
echo "No remote cache provided, using local fallback..."; \
fi

# Sync cache from MinIO (if remote cache is provided)
RUN if [ -n "${REMOTE_CACHE_URL}" ]; then \
echo "Downloading cache from MinIO..." && \
aws --endpoint-url=${MINIO_ENDPOINT_URL} s3 sync s3://vcpkg-cache ${LOCAL_CACHE_DIR}; \
else \
echo "No remote cache provided, skipping download."; \
fi

# Copy source code
COPY ./engine /app/engine
COPY ./docs/static/openapi/cortex.json /app/docs/static/openapi/cortex.json

# Build project
# Configure vcpkg binary sources
RUN export VCPKG_BINARY_SOURCES="files,${LOCAL_CACHE_DIR},readwrite;default"; \
cd engine && make configure-vcpkg && make build CMAKE_EXTRA_FLAGS="-DCORTEX_CPP_VERSION=${CORTEX_CPP_VERSION} -DCMAKE_BUILD_TEST=OFF -DCMAKE_TOOLCHAIN_FILE=vcpkg/scripts/buildsystems/vcpkg.cmake ${CMAKE_EXTRA_FLAGS}"

# Upload updated cache to MinIO (if remote cache is provided)
RUN if [ -n "${REMOTE_CACHE_URL}" ]; then \
echo "Uploading cache to MinIO..." && \
aws --endpoint-url=${MINIO_ENDPOINT_URL} s3 sync ${LOCAL_CACHE_DIR} s3://vcpkg-cache; \
else \
echo "No remote cache provided, skipping upload."; \
fi

# Stage 3: Runtime
FROM common as runtime

WORKDIR /app

# Copy built binaries from the build stage
COPY --from=build /app/engine/build/cortex /usr/local/bin/cortex
COPY --from=build /app/engine/build/cortex-server /usr/local/bin/cortex-server

COPY ./docker/download-cortex.llamacpp.sh /tmp/download-cortex.llamacpp.sh
COPY ./docker/entrypoint.sh /usr/local/bin/entrypoint.sh

# Get the latest version of Cortex Llama
ARG CORTEX_LLAMACPP_VERSION=latest
RUN chmod +x /tmp/download-cortex.llamacpp.sh && /bin/bash /tmp/download-cortex.llamacpp.sh ${CORTEX_LLAMACPP_VERSION}

# Configure entrypoint
RUN chmod +x /usr/local/bin/entrypoint.sh

EXPOSE 39281

# Healthcheck
HEALTHCHECK --interval=300s --timeout=30s --start-period=10s --retries=3 \
CMD curl -f http://127.0.0.1:39281/healthz || exit 1

ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
3 changes: 2 additions & 1 deletion docker/entrypoint.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,11 @@
echo "apiServerHost: 0.0.0.0" > /root/.cortexrc
echo "enableCors: true" >> /root/.cortexrc

# Install the engine
cortex engines install llama-cpp -s /opt/cortex.llamacpp
cortex engines list

# Start the cortex server

cortex start

# Keep the container running by tailing the log files
Expand Down
Loading

0 comments on commit 22826b9

Please sign in to comment.