Skip to content

Commit

Permalink
feat: add filter compatible for engine variant api (#1819)
Browse files Browse the repository at this point in the history
  • Loading branch information
@namchuai
namchuai authored Dec 23, 2024
1 parent 5414e02 commit a8b2503
Show file tree
Hide file tree
Showing 5 changed files with 139 additions and 11 deletions.
78 changes: 78 additions & 0 deletions docs/static/openapi/cortex.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2199,6 +2199,84 @@
"tags": ["Engines"]
}
},
"/v1/engines/{name}/releases/{version}": {
"get": {
"summary": "List variants for a specific engine version",
"description": "Lists all available variants (builds) for a specific version of an engine. Variants can include different CPU architectures (AVX, AVX2, AVX512), GPU support (CUDA, Vulkan), and operating systems (Windows, Linux, macOS).",
"parameters": [
{
"name": "name",
"in": "path",
"required": true,
"schema": {
"type": "string",
"enum": ["llama-cpp", "onnxruntime", "tensorrt-llm"],
"default": "llama-cpp"
},
"description": "The type of engine"
},
{
"name": "version",
"in": "path",
"required": true,
"schema": {
"type": "string"
},
"description": "The version of the engine"
},
{
"name": "show",
"in": "query",
"required": false,
"schema": {
"type": "string",
"enum": ["all", "compatible"],
"default": "all"
},
"description": "Filter the variants list. Use 'compatible' to show only variants compatible with the current system, or 'all' to show all available variants."
}
],
"responses": {
"200": {
"description": "Successfully retrieved variants list",
"content": {
"application/json": {
"schema": {
"type": "array",
"items": {
"type": "object",
"properties": {
"name": {
"type": "string",
"description": "The name of the variant, including OS, architecture, and capabilities",
"example": "linux-amd64-avx-cuda-11-7"
},
"created_at": {
"type": "string",
"format": "date-time",
"description": "Creation timestamp of the variant",
"example": "2024-11-13T04:51:16Z"
},
"size": {
"type": "integer",
"description": "Size of the variant in bytes",
"example": 151224604
},
"download_count": {
"type": "integer",
"description": "Number of times this variant has been downloaded",
"example": 0
}
}
}
}
}
}
}
},
"tags": ["Engines"]
}
},
"/v1/engines/{name}/releases/latest": {
"get": {
"summary": "Get latest release",
Expand Down
16 changes: 14 additions & 2 deletions engine/controllers/engines.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,8 @@ void Engines::GetEngineReleases(
void Engines::GetEngineVariants(
const HttpRequestPtr& req,
std::function<void(const HttpResponsePtr&)>&& callback,
const std::string& engine, const std::string& version) const {
const std::string& engine, const std::string& version,
std::optional<std::string> show) const {
if (engine.empty()) {
Json::Value res;
res["message"] = "Engine name is required";
Expand All @@ -140,7 +141,18 @@ void Engines::GetEngineVariants(
return;
}

auto result = engine_service_->GetEngineVariants(engine, version);
auto show_value = show.value_or("all");
if (show_value != "all" && show_value != "compatible") {
Json::Value res;
res["message"] = "Invalid show value. Can either be `all` or `compatible`";
auto resp = cortex_utils::CreateCortexHttpJsonResponse(res);
resp->setStatusCode(k400BadRequest);
callback(resp);
return;
}

auto result = engine_service_->GetEngineVariants(engine, version,
show_value == "compatible");

auto normalize_version = string_utils::RemoveSubstring(version, "v");
Json::Value releases(Json::arrayValue);
Expand Down
12 changes: 5 additions & 7 deletions engine/controllers/engines.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,13 +53,11 @@ class Engines : public drogon::HttpController<Engines, false> {
METHOD_ADD(Engines::GetEngineReleases, "/{1}/releases", Get);
ADD_METHOD_TO(Engines::GetEngineReleases, "/v1/engines/{1}/releases", Get);

METHOD_ADD(Engines::GetEngineVariants, "/{1}/releases/{2}", Get);
ADD_METHOD_TO(Engines::GetEngineVariants, "/v1/engines/{1}/releases/{2}",
Get);
ADD_METHOD_TO(Engines::GetEngineVariants,
"/v1/engines/{engine}/releases/{version}?show={show}", Get);

METHOD_ADD(Engines::GetLatestEngineVersion, "/{1}/releases/latest", Get);
ADD_METHOD_TO(Engines::GetLatestEngineVersion,
"/v1/engines/{1}/releases/latest", Get);
"/v1/engines/{engine}/releases/latest", Get);

METHOD_LIST_END

Expand All @@ -83,8 +81,8 @@ class Engines : public drogon::HttpController<Engines, false> {

void GetEngineVariants(const HttpRequestPtr& req,
std::function<void(const HttpResponsePtr&)>&& callback,
const std::string& engine,
const std::string& version) const;
const std::string& engine, const std::string& version,
std::optional<std::string> show) const;

void GetInstalledEngineVariants(
const HttpRequestPtr& req,
Expand Down
41 changes: 40 additions & 1 deletion engine/services/engine_service.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -482,7 +482,8 @@ EngineService::GetEngineReleases(const std::string& engine) const {

cpp::result<std::vector<EngineService::EngineVariant>, std::string>
EngineService::GetEngineVariants(const std::string& engine,
const std::string& version) const {
const std::string& version,
bool filter_compatible_only) const {
auto ne = NormalizeEngine(engine);
auto engine_release =
github_release_utils::GetReleaseByVersion("janhq", ne, version);
Expand All @@ -506,6 +507,44 @@ EngineService::GetEngineVariants(const std::string& engine,
return cpp::fail("No compatible variants found for " + engine);
}

if (filter_compatible_only) {
auto system_info = system_info_utils::GetSystemInfo();
compatible_variants.erase(
std::remove_if(compatible_variants.begin(), compatible_variants.end(),
[&system_info](const EngineVariant& variant) {
std::string name = variant.name;
std::transform(name.begin(), name.end(), name.begin(),
::tolower);

bool os_match = false;
if (system_info->os == "mac" &&
name.find("mac") != std::string::npos)
os_match = true;
if (system_info->os == "windows" &&
name.find("windows") != std::string::npos)
os_match = true;
if (system_info->os == "linux" &&
name.find("linux") != std::string::npos)
os_match = true;

bool arch_match = false;
if (system_info->arch == "arm64" &&
name.find("arm64") != std::string::npos)
arch_match = true;
if (system_info->arch == "amd64" &&
name.find("amd64") != std::string::npos)
arch_match = true;

return !(os_match && arch_match);
}),
compatible_variants.end());

if (compatible_variants.empty()) {
return cpp::fail("No compatible variants found for system " +
system_info->os + "/" + system_info->arch);
}
}

return compatible_variants;
}

Expand Down
3 changes: 2 additions & 1 deletion engine/services/engine_service.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,8 @@ class EngineService : public EngineServiceI {
const std::string& engine) const;

cpp::result<std::vector<EngineVariant>, std::string> GetEngineVariants(
const std::string& engine, const std::string& version) const;
const std::string& engine, const std::string& version,
bool filter_compatible_only = false) const;

cpp::result<DefaultEngineVariant, std::string> SetDefaultEngineVariant(
const std::string& engine, const std::string& version,
Expand Down

0 comments on commit a8b2503

Please sign in to comment.