From b03f1863f5f22a8bcea672cb105deb6d739de0ea Mon Sep 17 00:00:00 2001 From: Tempest <110401501+1-tempest@users.noreply.github.com> Date: Mon, 13 Jan 2025 19:40:10 -0600 Subject: [PATCH 01/42] Add additional variables to preload part ML models --- machine-learning/app/config.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/machine-learning/app/config.py b/machine-learning/app/config.py index 92799ac692ec1..b16667ae9598f 100644 --- a/machine-learning/app/config.py +++ b/machine-learning/app/config.py @@ -17,6 +17,10 @@ class PreloadModelData(BaseModel): clip: str | None = None facial_recognition: str | None = None + clip_textual: bool = True + clip_visual: bool = True + facial_recognition_detection: bool = True + facial_recognition_recognition: bool = True class MaxBatchSize(BaseModel): From 05fda4078aacd0b92e483afa98209d99e87e9552 Mon Sep 17 00:00:00 2001 From: Tempest <110401501+1-tempest@users.noreply.github.com> Date: Mon, 13 Jan 2025 21:11:10 -0600 Subject: [PATCH 02/42] Add additional variables to preload part ML models --- machine-learning/app/config.py | 35 ++++++++++++++++++++++++++++------ 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/machine-learning/app/config.py b/machine-learning/app/config.py index b16667ae9598f..489f75d4577e9 100644 --- a/machine-learning/app/config.py +++ b/machine-learning/app/config.py @@ -14,13 +14,36 @@ from uvicorn.workers import UvicornWorker +class ClipSettings(BaseModel): + model: str | None = None + textual: bool = True + visual: bool = True + + +class FacialRecognitionSettings(BaseModel): + model: str | None = None + recognition: bool = True + detection: bool = True + + class PreloadModelData(BaseModel): - clip: str | None = None - facial_recognition: str | None = None - clip_textual: bool = True - clip_visual: bool = True - facial_recognition_detection: bool = True - facial_recognition_recognition: bool = True + clip: ClipSettings = ClipSettings() + facialRecognition: FacialRecognitionSettings = FacialRecognitionSettings() + + # Define fallback environment variables + clip_model_fallback: str | None = Field(None, env="MACHINE_LEARNING_PRELOAD__CLIP") + facial_recognition_model_fallback: str | None = Field(None, env="MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION") + + # Root validator to use fallbacks + @root_validator(pre=True) + def set_models(cls, values): + # Set clip.model using fallback if clip.model is None + values['clip']['model'] = values.get('clip', {}).get('model') or values.get('clip_model_fallback') + + # Set facialRecognition.model using fallback if facialRecognition.model is None + values['facialRecognition']['model'] = values.get('facialRecognition', {}).get('model') or values.get('facial_recognition_model_fallback') + + return values class MaxBatchSize(BaseModel): From 41d74f066c9fdedbae825e9679a0c5d9b699f33b Mon Sep 17 00:00:00 2001 From: Tempest <110401501+1-tempest@users.noreply.github.com> Date: Mon, 13 Jan 2025 21:14:56 -0600 Subject: [PATCH 03/42] Add additional variables to preload part ML models --- machine-learning/app/main.py | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/machine-learning/app/main.py b/machine-learning/app/main.py index 684001b875e41..2f639c4228bf7 100644 --- a/machine-learning/app/main.py +++ b/machine-learning/app/main.py @@ -76,19 +76,23 @@ async def lifespan(_: FastAPI) -> AsyncGenerator[None, None]: async def preload_models(preload: PreloadModelData) -> None: log.info(f"Preloading models: {preload}") - if preload.clip is not None: - model = await model_cache.get(preload.clip, ModelType.TEXTUAL, ModelTask.SEARCH) - await load(model) - - model = await model_cache.get(preload.clip, ModelType.VISUAL, ModelTask.SEARCH) - await load(model) - - if preload.facial_recognition is not None: - model = await model_cache.get(preload.facial_recognition, ModelType.DETECTION, ModelTask.FACIAL_RECOGNITION) - await load(model) - - model = await model_cache.get(preload.facial_recognition, ModelType.RECOGNITION, ModelTask.FACIAL_RECOGNITION) - await load(model) + if preload.clip.model is not None: + if preload.clip.textual: + model = await model_cache.get(preload.clip, ModelType.TEXTUAL, ModelTask.SEARCH) + await load(model) + + if preload.clip.visual: + model = await model_cache.get(preload.clip, ModelType.VISUAL, ModelTask.SEARCH) + await load(model) + + if preload.facial_recognition.model is not None: + if preload.facial_recognition.detection: + model = await model_cache.get(preload.facial_recognition, ModelType.DETECTION, ModelTask.FACIAL_RECOGNITION) + await load(model) + + if preload.facial_recognition.recognition: + model = await model_cache.get(preload.facial_recognition, ModelType.RECOGNITION, ModelTask.FACIAL_RECOGNITION) + await load(model) def update_state() -> Iterator[None]: From f67c0518d3cf4d1174704f4f6d72ef4f0f7f2c06 Mon Sep 17 00:00:00 2001 From: Tempest <110401501+1-tempest@users.noreply.github.com> Date: Mon, 13 Jan 2025 21:16:20 -0600 Subject: [PATCH 04/42] Add additional variables to preload part ML models --- machine-learning/app/config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/machine-learning/app/config.py b/machine-learning/app/config.py index 489f75d4577e9..f2f1171ce6956 100644 --- a/machine-learning/app/config.py +++ b/machine-learning/app/config.py @@ -28,7 +28,7 @@ class FacialRecognitionSettings(BaseModel): class PreloadModelData(BaseModel): clip: ClipSettings = ClipSettings() - facialRecognition: FacialRecognitionSettings = FacialRecognitionSettings() + facial_recognition: FacialRecognitionSettings = FacialRecognitionSettings() # Define fallback environment variables clip_model_fallback: str | None = Field(None, env="MACHINE_LEARNING_PRELOAD__CLIP") @@ -41,7 +41,7 @@ def set_models(cls, values): values['clip']['model'] = values.get('clip', {}).get('model') or values.get('clip_model_fallback') # Set facialRecognition.model using fallback if facialRecognition.model is None - values['facialRecognition']['model'] = values.get('facialRecognition', {}).get('model') or values.get('facial_recognition_model_fallback') + values['facial_recognition']['model'] = values.get('facial_recognition', {}).get('model') or values.get('facial_recognition_model_fallback') return values From a5a9cc65f98995cf4b47ce784df5b72e9b0511d7 Mon Sep 17 00:00:00 2001 From: Tempest <110401501+1-tempest@users.noreply.github.com> Date: Mon, 13 Jan 2025 21:38:46 -0600 Subject: [PATCH 05/42] Add additional variables to preload part ML models --- docs/docs/install/environment-variables.md | 40 ++++++++++++---------- 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/docs/docs/install/environment-variables.md b/docs/docs/install/environment-variables.md index 1f34b5c6d00a4..1f4dff834d93b 100644 --- a/docs/docs/install/environment-variables.md +++ b/docs/docs/install/environment-variables.md @@ -148,24 +148,28 @@ Redis (Sentinel) URL example JSON before encoding: ## Machine Learning -| Variable | Description | Default | Containers | -| :-------------------------------------------------------- | :-------------------------------------------------------------------------------------------------- | :-----------------------------: | :--------------- | -| `MACHINE_LEARNING_MODEL_TTL` | Inactivity time (s) before a model is unloaded (disabled if \<= 0) | `300` | machine learning | -| `MACHINE_LEARNING_MODEL_TTL_POLL_S` | Interval (s) between checks for the model TTL (disabled if \<= 0) | `10` | machine learning | -| `MACHINE_LEARNING_CACHE_FOLDER` | Directory where models are downloaded | `/cache` | machine learning | -| `MACHINE_LEARNING_REQUEST_THREADS`\*1 | Thread count of the request thread pool (disabled if \<= 0) | number of CPU cores | machine learning | -| `MACHINE_LEARNING_MODEL_INTER_OP_THREADS` | Number of parallel model operations | `1` | machine learning | -| `MACHINE_LEARNING_MODEL_INTRA_OP_THREADS` | Number of threads for each model operation | `2` | machine learning | -| `MACHINE_LEARNING_WORKERS`\*2 | Number of worker processes to spawn | `1` | machine learning | -| `MACHINE_LEARNING_HTTP_KEEPALIVE_TIMEOUT_S`\*3 | HTTP Keep-alive time in seconds | `2` | machine learning | -| `MACHINE_LEARNING_WORKER_TIMEOUT` | Maximum time (s) of unresponsiveness before a worker is killed | `120` (`300` if using OpenVINO) | machine learning | -| `MACHINE_LEARNING_PRELOAD__CLIP` | Name of a CLIP model to be preloaded and kept in cache | | machine learning | -| `MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION` | Name of a facial recognition model to be preloaded and kept in cache | | machine learning | -| `MACHINE_LEARNING_ANN` | Enable ARM-NN hardware acceleration if supported | `True` | machine learning | -| `MACHINE_LEARNING_ANN_FP16_TURBO` | Execute operations in FP16 precision: increasing speed, reducing precision (applies only to ARM-NN) | `False` | machine learning | -| `MACHINE_LEARNING_ANN_TUNING_LEVEL` | ARM-NN GPU tuning level (1: rapid, 2: normal, 3: exhaustive) | `2` | machine learning | -| `MACHINE_LEARNING_DEVICE_IDS`\*4 | Device IDs to use in multi-GPU environments | `0` | machine learning | -| `MACHINE_LEARNING_MAX_BATCH_SIZE__FACIAL_RECOGNITION` | Set the maximum number of faces that will be processed at once by the facial recognition model | None (`1` if using OpenVINO) | machine learning | +| Variable | Description | Default | Containers | +| :---------------------------------------------------------- | :-------------------------------------------------------------------------------------------------- | :-----------------------------: | :--------------- | +| `MACHINE_LEARNING_MODEL_TTL` | Inactivity time (s) before a model is unloaded (disabled if \<= 0) | `300` | machine learning | +| `MACHINE_LEARNING_MODEL_TTL_POLL_S` | Interval (s) between checks for the model TTL (disabled if \<= 0) | `10` | machine learning | +| `MACHINE_LEARNING_CACHE_FOLDER` | Directory where models are downloaded | `/cache` | machine learning | +| `MACHINE_LEARNING_REQUEST_THREADS`\*1 | Thread count of the request thread pool (disabled if \<= 0) | number of CPU cores | machine learning | +| `MACHINE_LEARNING_MODEL_INTER_OP_THREADS` | Number of parallel model operations | `1` | machine learning | +| `MACHINE_LEARNING_MODEL_INTRA_OP_THREADS` | Number of threads for each model operation | `2` | machine learning | +| `MACHINE_LEARNING_WORKERS`\*2 | Number of worker processes to spawn | `1` | machine learning | +| `MACHINE_LEARNING_HTTP_KEEPALIVE_TIMEOUT_S`\*3 | HTTP Keep-alive time in seconds | `2` | machine learning | +| `MACHINE_LEARNING_WORKER_TIMEOUT` | Maximum time (s) of unresponsiveness before a worker is killed | `120` (`300` if using OpenVINO) | machine learning | +| `MACHINE_LEARNING_PRELOAD__CLIP__MODEL` | Name of a CLIP model to be preloaded and kept in cache | | machine learning | +| `MACHINE_LEARNING_PRELOAD__CLIP__TEXTUAL` | Preloads the textual model | `True` | machine learning | +| `MACHINE_LEARNING_PRELOAD__CLIP__VISUAL` | Preloads the visual model | `True` | machine learning | +| `MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION__MODEL` | Name of a facial recognition model to be preloaded and kept in cache | | machine learning | +| `MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION__RECOGNITION` | Preloads the recognition model | `True` | machine learning | +| `MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION__DETECTION` | Preloads the detection model | `True` | machine learning | +| `MACHINE_LEARNING_ANN` | Enable ARM-NN hardware acceleration if supported | `True` | machine learning | +| `MACHINE_LEARNING_ANN_FP16_TURBO` | Execute operations in FP16 precision: increasing speed, reducing precision (applies only to ARM-NN) | `False` | machine learning | +| `MACHINE_LEARNING_ANN_TUNING_LEVEL` | ARM-NN GPU tuning level (1: rapid, 2: normal, 3: exhaustive) | `2` | machine learning | +| `MACHINE_LEARNING_DEVICE_IDS`\*4 | Device IDs to use in multi-GPU environments | `0` | machine learning | +| `MACHINE_LEARNING_MAX_BATCH_SIZE__FACIAL_RECOGNITION` | Set the maximum number of faces that will be processed at once by the facial recognition model | None (`1` if using OpenVINO) | machine learning | \*1: It is recommended to begin with this parameter when changing the concurrency levels of the machine learning service and then tune the other ones. From cf2795874f382f1ed7afddabe205d8475067cf7a Mon Sep 17 00:00:00 2001 From: Tempest <110401501+1-tempest@users.noreply.github.com> Date: Mon, 13 Jan 2025 21:52:11 -0600 Subject: [PATCH 06/42] Add additional variables to preload part ML models --- machine-learning/app/config.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/machine-learning/app/config.py b/machine-learning/app/config.py index f2f1171ce6956..a5400a47185d7 100644 --- a/machine-learning/app/config.py +++ b/machine-learning/app/config.py @@ -6,7 +6,7 @@ from socket import socket from gunicorn.arbiter import Arbiter -from pydantic import BaseModel +from pydantic import BaseModel, Field, root_validator from pydantic_settings import BaseSettings, SettingsConfigDict from rich.console import Console from rich.logging import RichHandler @@ -31,8 +31,12 @@ class PreloadModelData(BaseModel): facial_recognition: FacialRecognitionSettings = FacialRecognitionSettings() # Define fallback environment variables - clip_model_fallback: str | None = Field(None, env="MACHINE_LEARNING_PRELOAD__CLIP") - facial_recognition_model_fallback: str | None = Field(None, env="MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION") + clip_model_fallback: str | None = Field( + None, env="MACHINE_LEARNING_PRELOAD__CLIP" + ) + facial_recognition_model_fallback: str | None = Field( + None, env="MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION" + ) # Root validator to use fallbacks @root_validator(pre=True) From 38068e6c44bc4bf23f37296c851a47e42ee26b71 Mon Sep 17 00:00:00 2001 From: Tempest <110401501+1-tempest@users.noreply.github.com> Date: Mon, 13 Jan 2025 21:59:53 -0600 Subject: [PATCH 07/42] Add additional variables to preload part ML models --- machine-learning/app/config.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/machine-learning/app/config.py b/machine-learning/app/config.py index a5400a47185d7..5ec26a3c83cb4 100644 --- a/machine-learning/app/config.py +++ b/machine-learning/app/config.py @@ -45,7 +45,10 @@ def set_models(cls, values): values['clip']['model'] = values.get('clip', {}).get('model') or values.get('clip_model_fallback') # Set facialRecognition.model using fallback if facialRecognition.model is None - values['facial_recognition']['model'] = values.get('facial_recognition', {}).get('model') or values.get('facial_recognition_model_fallback') + values['facial_recognition']['model'] = ( + values.get('facial_recognition', {}).get('model') + or values.get('facial_recognition_model_fallback') + ) return values From c378afb5756d0e982af95f56d576d4df4ccf3031 Mon Sep 17 00:00:00 2001 From: Tempest <110401501+1-tempest@users.noreply.github.com> Date: Mon, 13 Jan 2025 22:01:43 -0600 Subject: [PATCH 08/42] Add additional variables to preload part ML models --- machine-learning/app/main.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/machine-learning/app/main.py b/machine-learning/app/main.py index 2f639c4228bf7..921dcfd1b7ca6 100644 --- a/machine-learning/app/main.py +++ b/machine-learning/app/main.py @@ -91,7 +91,11 @@ async def preload_models(preload: PreloadModelData) -> None: await load(model) if preload.facial_recognition.recognition: - model = await model_cache.get(preload.facial_recognition, ModelType.RECOGNITION, ModelTask.FACIAL_RECOGNITION) + model = await model_cache.get( + preload.facial_recognition, + ModelType.RECOGNITION, + ModelTask.FACIAL_RECOGNITION + ) await load(model) From 3bd2ec6dbbeed3e3d51c5143c817d25ba3978a02 Mon Sep 17 00:00:00 2001 From: Tempest <110401501+1-tempest@users.noreply.github.com> Date: Mon, 13 Jan 2025 22:06:17 -0600 Subject: [PATCH 09/42] Add additional variables to preload part ML models --- machine-learning/app/main.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/machine-learning/app/main.py b/machine-learning/app/main.py index 921dcfd1b7ca6..503bb33b6a552 100644 --- a/machine-learning/app/main.py +++ b/machine-learning/app/main.py @@ -76,18 +76,27 @@ async def lifespan(_: FastAPI) -> AsyncGenerator[None, None]: async def preload_models(preload: PreloadModelData) -> None: log.info(f"Preloading models: {preload}") + if preload.clip.model is not None: if preload.clip.textual: - model = await model_cache.get(preload.clip, ModelType.TEXTUAL, ModelTask.SEARCH) + model = await model_cache.get( + preload.clip, ModelType.TEXTUAL, ModelTask.SEARCH + ) await load(model) if preload.clip.visual: - model = await model_cache.get(preload.clip, ModelType.VISUAL, ModelTask.SEARCH) + model = await model_cache.get( + preload.clip, ModelType.VISUAL, ModelTask.SEARCH + ) await load(model) if preload.facial_recognition.model is not None: if preload.facial_recognition.detection: - model = await model_cache.get(preload.facial_recognition, ModelType.DETECTION, ModelTask.FACIAL_RECOGNITION) + model = await model_cache.get( + preload.facial_recognition, + ModelType.DETECTION, + ModelTask.FACIAL_RECOGNITION + ) await load(model) if preload.facial_recognition.recognition: @@ -99,6 +108,7 @@ async def preload_models(preload: PreloadModelData) -> None: await load(model) + def update_state() -> Iterator[None]: global active_requests, last_called active_requests += 1 From 85e17e4582a9901d91366548022028653267d4ad Mon Sep 17 00:00:00 2001 From: Tempest <110401501+1-tempest@users.noreply.github.com> Date: Mon, 13 Jan 2025 22:08:22 -0600 Subject: [PATCH 10/42] Update config.py --- machine-learning/app/config.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/machine-learning/app/config.py b/machine-learning/app/config.py index 5ec26a3c83cb4..35b34026bae57 100644 --- a/machine-learning/app/config.py +++ b/machine-learning/app/config.py @@ -41,10 +41,11 @@ class PreloadModelData(BaseModel): # Root validator to use fallbacks @root_validator(pre=True) def set_models(cls, values): - # Set clip.model using fallback if clip.model is None - values['clip']['model'] = values.get('clip', {}).get('model') or values.get('clip_model_fallback') + values['clip']['model'] = ( + values.get('clip', {}).get('model') + or values.get('clip_model_fallback') + ) - # Set facialRecognition.model using fallback if facialRecognition.model is None values['facial_recognition']['model'] = ( values.get('facial_recognition', {}).get('model') or values.get('facial_recognition_model_fallback') From 4a224213b1d60b06438043b32987e1abd7455664 Mon Sep 17 00:00:00 2001 From: Tempest <110401501+1-tempest@users.noreply.github.com> Date: Mon, 13 Jan 2025 22:12:35 -0600 Subject: [PATCH 11/42] Add additional variables to preload part ML models --- machine-learning/app/config.py | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/machine-learning/app/config.py b/machine-learning/app/config.py index 35b34026bae57..e6e2ba9073ca4 100644 --- a/machine-learning/app/config.py +++ b/machine-learning/app/config.py @@ -29,11 +29,9 @@ class FacialRecognitionSettings(BaseModel): class PreloadModelData(BaseModel): clip: ClipSettings = ClipSettings() facial_recognition: FacialRecognitionSettings = FacialRecognitionSettings() - + # Define fallback environment variables - clip_model_fallback: str | None = Field( - None, env="MACHINE_LEARNING_PRELOAD__CLIP" - ) + clip_model_fallback: str | None = Field(None, env="MACHINE_LEARNING_PRELOAD__CLIP") facial_recognition_model_fallback: str | None = Field( None, env="MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION" ) @@ -41,15 +39,13 @@ class PreloadModelData(BaseModel): # Root validator to use fallbacks @root_validator(pre=True) def set_models(cls, values): - values['clip']['model'] = ( - values.get('clip', {}).get('model') - or values.get('clip_model_fallback') + values["clip"]["model"] = values.get("clip", {}).get("model") or values.get( + "clip_model_fallback" ) - values['facial_recognition']['model'] = ( - values.get('facial_recognition', {}).get('model') - or values.get('facial_recognition_model_fallback') - ) + values["facial_recognition"]["model"] = values.get( + "facial_recognition", {} + ).get("model") or values.get("facial_recognition_model_fallback") return values @@ -136,7 +132,10 @@ def emit(self, record: logging.LogRecord) -> None: if record.exc_info is not None: tb = record.exc_info[2] while tb is not None: - if any(excluded in tb.tb_frame.f_code.co_filename for excluded in self.excluded): + if any( + excluded in tb.tb_frame.f_code.co_filename + for excluded in self.excluded + ): tb.tb_frame.f_locals["_rich_traceback_omit"] = True tb = tb.tb_next From 5679c4fd100a55363bb3d9fa2a1aec623148be55 Mon Sep 17 00:00:00 2001 From: Tempest <110401501+1-tempest@users.noreply.github.com> Date: Mon, 13 Jan 2025 22:13:48 -0600 Subject: [PATCH 12/42] Add additional variables to preload part ML models --- machine-learning/app/main.py | 37 +++++++++++++++++++++++------------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/machine-learning/app/main.py b/machine-learning/app/main.py index 503bb33b6a552..6d7b2d9b10064 100644 --- a/machine-learning/app/main.py +++ b/machine-learning/app/main.py @@ -58,8 +58,14 @@ async def lifespan(_: FastAPI) -> AsyncGenerator[None, None]: try: if settings.request_threads > 0: # asyncio is a huge bottleneck for performance, so we use a thread pool to run blocking code - thread_pool = ThreadPoolExecutor(settings.request_threads) if settings.request_threads > 0 else None - log.info(f"Initialized request thread pool with {settings.request_threads} threads.") + thread_pool = ( + ThreadPoolExecutor(settings.request_threads) + if settings.request_threads > 0 + else None + ) + log.info( + f"Initialized request thread pool with {settings.request_threads} threads." + ) if settings.model_ttl > 0 and settings.model_ttl_poll_s > 0: asyncio.ensure_future(idle_shutdown_task()) if settings.preload is not None: @@ -76,7 +82,7 @@ async def lifespan(_: FastAPI) -> AsyncGenerator[None, None]: async def preload_models(preload: PreloadModelData) -> None: log.info(f"Preloading models: {preload}") - + if preload.clip.model is not None: if preload.clip.textual: model = await model_cache.get( @@ -93,22 +99,21 @@ async def preload_models(preload: PreloadModelData) -> None: if preload.facial_recognition.model is not None: if preload.facial_recognition.detection: model = await model_cache.get( - preload.facial_recognition, - ModelType.DETECTION, - ModelTask.FACIAL_RECOGNITION + preload.facial_recognition, + ModelType.DETECTION, + ModelTask.FACIAL_RECOGNITION, ) await load(model) if preload.facial_recognition.recognition: model = await model_cache.get( - preload.facial_recognition, - ModelType.RECOGNITION, - ModelTask.FACIAL_RECOGNITION + preload.facial_recognition, + ModelType.RECOGNITION, + ModelTask.FACIAL_RECOGNITION, ) await load(model) - def update_state() -> Iterator[None]: global active_requests, last_called active_requests += 1 @@ -169,12 +174,16 @@ async def predict( return ORJSONResponse(response) -async def run_inference(payload: Image | str, entries: InferenceEntries) -> InferenceResponse: +async def run_inference( + payload: Image | str, entries: InferenceEntries +) -> InferenceResponse: outputs: dict[ModelIdentity, Any] = {} response: InferenceResponse = {} async def _run_inference(entry: InferenceEntry) -> None: - model = await model_cache.get(entry["name"], entry["type"], entry["task"], ttl=settings.model_ttl) + model = await model_cache.get( + entry["name"], entry["type"], entry["task"], ttl=settings.model_ttl + ) inputs = [payload] for dep in model.depends: try: @@ -228,7 +237,9 @@ def _load(model: InferenceModel) -> InferenceModel: try: return await run(_load, model) except (OSError, InvalidProtobuf, BadZipFile, NoSuchFile): - log.warning(f"Failed to load {model.model_type.replace('_', ' ')} model '{model.model_name}'. Clearing cache.") + log.warning( + f"Failed to load {model.model_type.replace('_', ' ')} model '{model.model_name}'. Clearing cache." + ) model.clear_cache() return await run(_load, model) From 8556c32931f3e0bbaf81f03bb9aaf6a645740d10 Mon Sep 17 00:00:00 2001 From: Jared Dueck Date: Mon, 13 Jan 2025 22:41:16 -0600 Subject: [PATCH 13/42] Apply formatting --- machine-learning/app/config.py | 19 ++++++------------- machine-learning/app/main.py | 30 +++++++----------------------- 2 files changed, 13 insertions(+), 36 deletions(-) diff --git a/machine-learning/app/config.py b/machine-learning/app/config.py index e6e2ba9073ca4..6415a71bbd488 100644 --- a/machine-learning/app/config.py +++ b/machine-learning/app/config.py @@ -32,20 +32,16 @@ class PreloadModelData(BaseModel): # Define fallback environment variables clip_model_fallback: str | None = Field(None, env="MACHINE_LEARNING_PRELOAD__CLIP") - facial_recognition_model_fallback: str | None = Field( - None, env="MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION" - ) + facial_recognition_model_fallback: str | None = Field(None, env="MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION") # Root validator to use fallbacks @root_validator(pre=True) def set_models(cls, values): - values["clip"]["model"] = values.get("clip", {}).get("model") or values.get( - "clip_model_fallback" - ) + values["clip"]["model"] = values.get("clip", {}).get("model") or values.get("clip_model_fallback") - values["facial_recognition"]["model"] = values.get( - "facial_recognition", {} - ).get("model") or values.get("facial_recognition_model_fallback") + values["facial_recognition"]["model"] = values.get("facial_recognition", {}).get("model") or values.get( + "facial_recognition_model_fallback" + ) return values @@ -132,10 +128,7 @@ def emit(self, record: logging.LogRecord) -> None: if record.exc_info is not None: tb = record.exc_info[2] while tb is not None: - if any( - excluded in tb.tb_frame.f_code.co_filename - for excluded in self.excluded - ): + if any(excluded in tb.tb_frame.f_code.co_filename for excluded in self.excluded): tb.tb_frame.f_locals["_rich_traceback_omit"] = True tb = tb.tb_next diff --git a/machine-learning/app/main.py b/machine-learning/app/main.py index 6d7b2d9b10064..64e7c339a678a 100644 --- a/machine-learning/app/main.py +++ b/machine-learning/app/main.py @@ -58,14 +58,8 @@ async def lifespan(_: FastAPI) -> AsyncGenerator[None, None]: try: if settings.request_threads > 0: # asyncio is a huge bottleneck for performance, so we use a thread pool to run blocking code - thread_pool = ( - ThreadPoolExecutor(settings.request_threads) - if settings.request_threads > 0 - else None - ) - log.info( - f"Initialized request thread pool with {settings.request_threads} threads." - ) + thread_pool = ThreadPoolExecutor(settings.request_threads) if settings.request_threads > 0 else None + log.info(f"Initialized request thread pool with {settings.request_threads} threads.") if settings.model_ttl > 0 and settings.model_ttl_poll_s > 0: asyncio.ensure_future(idle_shutdown_task()) if settings.preload is not None: @@ -85,15 +79,11 @@ async def preload_models(preload: PreloadModelData) -> None: if preload.clip.model is not None: if preload.clip.textual: - model = await model_cache.get( - preload.clip, ModelType.TEXTUAL, ModelTask.SEARCH - ) + model = await model_cache.get(preload.clip, ModelType.TEXTUAL, ModelTask.SEARCH) await load(model) if preload.clip.visual: - model = await model_cache.get( - preload.clip, ModelType.VISUAL, ModelTask.SEARCH - ) + model = await model_cache.get(preload.clip, ModelType.VISUAL, ModelTask.SEARCH) await load(model) if preload.facial_recognition.model is not None: @@ -174,16 +164,12 @@ async def predict( return ORJSONResponse(response) -async def run_inference( - payload: Image | str, entries: InferenceEntries -) -> InferenceResponse: +async def run_inference(payload: Image | str, entries: InferenceEntries) -> InferenceResponse: outputs: dict[ModelIdentity, Any] = {} response: InferenceResponse = {} async def _run_inference(entry: InferenceEntry) -> None: - model = await model_cache.get( - entry["name"], entry["type"], entry["task"], ttl=settings.model_ttl - ) + model = await model_cache.get(entry["name"], entry["type"], entry["task"], ttl=settings.model_ttl) inputs = [payload] for dep in model.depends: try: @@ -237,9 +223,7 @@ def _load(model: InferenceModel) -> InferenceModel: try: return await run(_load, model) except (OSError, InvalidProtobuf, BadZipFile, NoSuchFile): - log.warning( - f"Failed to load {model.model_type.replace('_', ' ')} model '{model.model_name}'. Clearing cache." - ) + log.warning(f"Failed to load {model.model_type.replace('_', ' ')} model '{model.model_name}'. Clearing cache.") model.clear_cache() return await run(_load, model) From 3970aa74e79b79db6c791218677d5bb7a15876eb Mon Sep 17 00:00:00 2001 From: Jared Dueck Date: Mon, 13 Jan 2025 22:46:29 -0600 Subject: [PATCH 14/42] minor update --- machine-learning/app/main.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/machine-learning/app/main.py b/machine-learning/app/main.py index 64e7c339a678a..7cbbea96eb6e9 100644 --- a/machine-learning/app/main.py +++ b/machine-learning/app/main.py @@ -79,17 +79,17 @@ async def preload_models(preload: PreloadModelData) -> None: if preload.clip.model is not None: if preload.clip.textual: - model = await model_cache.get(preload.clip, ModelType.TEXTUAL, ModelTask.SEARCH) + model = await model_cache.get(preload.clip.model, ModelType.TEXTUAL, ModelTask.SEARCH) await load(model) if preload.clip.visual: - model = await model_cache.get(preload.clip, ModelType.VISUAL, ModelTask.SEARCH) + model = await model_cache.get(preload.clip.model, ModelType.VISUAL, ModelTask.SEARCH) await load(model) if preload.facial_recognition.model is not None: if preload.facial_recognition.detection: model = await model_cache.get( - preload.facial_recognition, + preload.facial_recognition.model, ModelType.DETECTION, ModelTask.FACIAL_RECOGNITION, ) @@ -97,7 +97,7 @@ async def preload_models(preload: PreloadModelData) -> None: if preload.facial_recognition.recognition: model = await model_cache.get( - preload.facial_recognition, + preload.facial_recognition.model, ModelType.RECOGNITION, ModelTask.FACIAL_RECOGNITION, ) From 007db2b22338600220c0c39b0596056332875515 Mon Sep 17 00:00:00 2001 From: Jared Dueck Date: Mon, 13 Jan 2025 22:55:12 -0600 Subject: [PATCH 15/42] formatting --- machine-learning/app/config.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/machine-learning/app/config.py b/machine-learning/app/config.py index 6415a71bbd488..cbaad7e1aae65 100644 --- a/machine-learning/app/config.py +++ b/machine-learning/app/config.py @@ -31,8 +31,10 @@ class PreloadModelData(BaseModel): facial_recognition: FacialRecognitionSettings = FacialRecognitionSettings() # Define fallback environment variables - clip_model_fallback: str | None = Field(None, env="MACHINE_LEARNING_PRELOAD__CLIP") - facial_recognition_model_fallback: str | None = Field(None, env="MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION") + clip_model_fallback: str | None = Field(default=None, env="MACHINE_LEARNING_PRELOAD__CLIP") + facial_recognition_model_fallback: str | None = Field( + default=None, env="MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION" + ) # Root validator to use fallbacks @root_validator(pre=True) From 6c9d173a526f14cfb3c2c7e914c14ba099ff5918 Mon Sep 17 00:00:00 2001 From: Jared Dueck Date: Mon, 13 Jan 2025 22:59:35 -0600 Subject: [PATCH 16/42] root validator --- machine-learning/app/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/machine-learning/app/config.py b/machine-learning/app/config.py index cbaad7e1aae65..2fdea80ccca45 100644 --- a/machine-learning/app/config.py +++ b/machine-learning/app/config.py @@ -38,7 +38,7 @@ class PreloadModelData(BaseModel): # Root validator to use fallbacks @root_validator(pre=True) - def set_models(cls, values): + def set_models(cls, values: dict) -> dict: values["clip"]["model"] = values.get("clip", {}).get("model") or values.get("clip_model_fallback") values["facial_recognition"]["model"] = values.get("facial_recognition", {}).get("model") or values.get( From 6f7939069627bf1234ba6bc2c093c78948336a49 Mon Sep 17 00:00:00 2001 From: Jared Dueck Date: Mon, 13 Jan 2025 23:03:18 -0600 Subject: [PATCH 17/42] minor update --- machine-learning/app/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/machine-learning/app/config.py b/machine-learning/app/config.py index 2fdea80ccca45..916c154300882 100644 --- a/machine-learning/app/config.py +++ b/machine-learning/app/config.py @@ -38,7 +38,7 @@ class PreloadModelData(BaseModel): # Root validator to use fallbacks @root_validator(pre=True) - def set_models(cls, values: dict) -> dict: + def set_models(cls, values: dict[str, dict]) -> dict[str, dict]: values["clip"]["model"] = values.get("clip", {}).get("model") or values.get("clip_model_fallback") values["facial_recognition"]["model"] = values.get("facial_recognition", {}).get("model") or values.get( From 47d53b135609ff817a699ba669c8229dcaa8b26c Mon Sep 17 00:00:00 2001 From: Jared Dueck Date: Mon, 13 Jan 2025 23:08:05 -0600 Subject: [PATCH 18/42] minor update --- machine-learning/app/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/machine-learning/app/config.py b/machine-learning/app/config.py index 916c154300882..2fdea80ccca45 100644 --- a/machine-learning/app/config.py +++ b/machine-learning/app/config.py @@ -38,7 +38,7 @@ class PreloadModelData(BaseModel): # Root validator to use fallbacks @root_validator(pre=True) - def set_models(cls, values: dict[str, dict]) -> dict[str, dict]: + def set_models(cls, values: dict) -> dict: values["clip"]["model"] = values.get("clip", {}).get("model") or values.get("clip_model_fallback") values["facial_recognition"]["model"] = values.get("facial_recognition", {}).get("model") or values.get( From ef3b3036ef5e3b4853ec8452219f5597c7430555 Mon Sep 17 00:00:00 2001 From: Jared Dueck Date: Mon, 13 Jan 2025 23:18:33 -0600 Subject: [PATCH 19/42] minor update --- machine-learning/app/config.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/machine-learning/app/config.py b/machine-learning/app/config.py index 2fdea80ccca45..881ff43c3d34b 100644 --- a/machine-learning/app/config.py +++ b/machine-learning/app/config.py @@ -6,7 +6,7 @@ from socket import socket from gunicorn.arbiter import Arbiter -from pydantic import BaseModel, Field, root_validator +from pydantic import BaseModel, Field from pydantic_settings import BaseSettings, SettingsConfigDict from rich.console import Console from rich.logging import RichHandler @@ -36,16 +36,15 @@ class PreloadModelData(BaseModel): default=None, env="MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION" ) - # Root validator to use fallbacks - @root_validator(pre=True) - def set_models(cls, values: dict) -> dict: - values["clip"]["model"] = values.get("clip", {}).get("model") or values.get("clip_model_fallback") + def __init__(self, **kwargs): + super().__init__(**kwargs) - values["facial_recognition"]["model"] = values.get("facial_recognition", {}).get("model") or values.get( - "facial_recognition_model_fallback" - ) + # Replace the values with fallbacks if necessary + if not self.clip.model and self.clip_model_fallback: + self.clip.model = self.clip_model_fallback - return values + if not self.facial_recognition.model and self.facial_recognition_model_fallback: + self.facial_recognition.model = self.facial_recognition_model_fallback class MaxBatchSize(BaseModel): From b4ea6e92403dcd0d7c5baf788c9f821c6d374f3d Mon Sep 17 00:00:00 2001 From: Jared Dueck Date: Mon, 13 Jan 2025 23:24:44 -0600 Subject: [PATCH 20/42] change to support explicit models --- machine-learning/app/config.py | 20 +++----------- machine-learning/app/main.py | 48 ++++++++++++++++------------------ 2 files changed, 27 insertions(+), 41 deletions(-) diff --git a/machine-learning/app/config.py b/machine-learning/app/config.py index 881ff43c3d34b..0cb773a1fdd7b 100644 --- a/machine-learning/app/config.py +++ b/machine-learning/app/config.py @@ -15,15 +15,13 @@ class ClipSettings(BaseModel): - model: str | None = None - textual: bool = True - visual: bool = True + textual: str | None = None + visual: str | None = None class FacialRecognitionSettings(BaseModel): - model: str | None = None - recognition: bool = True - detection: bool = True + recognition: str | None = None + detection: str | None = None class PreloadModelData(BaseModel): @@ -36,16 +34,6 @@ class PreloadModelData(BaseModel): default=None, env="MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION" ) - def __init__(self, **kwargs): - super().__init__(**kwargs) - - # Replace the values with fallbacks if necessary - if not self.clip.model and self.clip_model_fallback: - self.clip.model = self.clip_model_fallback - - if not self.facial_recognition.model and self.facial_recognition_model_fallback: - self.facial_recognition.model = self.facial_recognition_model_fallback - class MaxBatchSize(BaseModel): facial_recognition: int | None = None diff --git a/machine-learning/app/main.py b/machine-learning/app/main.py index 7cbbea96eb6e9..bb62f5926b8ac 100644 --- a/machine-learning/app/main.py +++ b/machine-learning/app/main.py @@ -77,31 +77,29 @@ async def lifespan(_: FastAPI) -> AsyncGenerator[None, None]: async def preload_models(preload: PreloadModelData) -> None: log.info(f"Preloading models: {preload}") - if preload.clip.model is not None: - if preload.clip.textual: - model = await model_cache.get(preload.clip.model, ModelType.TEXTUAL, ModelTask.SEARCH) - await load(model) - - if preload.clip.visual: - model = await model_cache.get(preload.clip.model, ModelType.VISUAL, ModelTask.SEARCH) - await load(model) - - if preload.facial_recognition.model is not None: - if preload.facial_recognition.detection: - model = await model_cache.get( - preload.facial_recognition.model, - ModelType.DETECTION, - ModelTask.FACIAL_RECOGNITION, - ) - await load(model) - - if preload.facial_recognition.recognition: - model = await model_cache.get( - preload.facial_recognition.model, - ModelType.RECOGNITION, - ModelTask.FACIAL_RECOGNITION, - ) - await load(model) + if preload.clip.textual is not None: + model = await model_cache.get(preload.clip.model, ModelType.TEXTUAL, ModelTask.SEARCH) + await load(model) + + if preload.clip.visual is not None: + model = await model_cache.get(preload.clip.model, ModelType.VISUAL, ModelTask.SEARCH) + await load(model) + + if preload.facial_recognition.detection is not None: + model = await model_cache.get( + preload.facial_recognition.model, + ModelType.DETECTION, + ModelTask.FACIAL_RECOGNITION, + ) + await load(model) + + if preload.facial_recognition.recognition is not None: + model = await model_cache.get( + preload.facial_recognition.model, + ModelType.RECOGNITION, + ModelTask.FACIAL_RECOGNITION, + ) + await load(model) def update_state() -> Iterator[None]: From 12eff16276d24389feddd7013dd46210ddcb4306 Mon Sep 17 00:00:00 2001 From: Jared Dueck Date: Mon, 13 Jan 2025 23:27:49 -0600 Subject: [PATCH 21/42] minor update --- machine-learning/app/main.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/machine-learning/app/main.py b/machine-learning/app/main.py index bb62f5926b8ac..ee8881b6c604c 100644 --- a/machine-learning/app/main.py +++ b/machine-learning/app/main.py @@ -78,16 +78,16 @@ async def preload_models(preload: PreloadModelData) -> None: log.info(f"Preloading models: {preload}") if preload.clip.textual is not None: - model = await model_cache.get(preload.clip.model, ModelType.TEXTUAL, ModelTask.SEARCH) + model = await model_cache.get(preload.clip.textual, ModelType.TEXTUAL, ModelTask.SEARCH) await load(model) if preload.clip.visual is not None: - model = await model_cache.get(preload.clip.model, ModelType.VISUAL, ModelTask.SEARCH) + model = await model_cache.get(preload.clip.visual, ModelType.VISUAL, ModelTask.SEARCH) await load(model) if preload.facial_recognition.detection is not None: model = await model_cache.get( - preload.facial_recognition.model, + preload.facial_recognition.detection, ModelType.DETECTION, ModelTask.FACIAL_RECOGNITION, ) @@ -95,7 +95,7 @@ async def preload_models(preload: PreloadModelData) -> None: if preload.facial_recognition.recognition is not None: model = await model_cache.get( - preload.facial_recognition.model, + preload.facial_recognition.recognition, ModelType.RECOGNITION, ModelTask.FACIAL_RECOGNITION, ) From 233427a970016bcae78e26a9f24ef13dc981bffe Mon Sep 17 00:00:00 2001 From: Jared Dueck Date: Mon, 13 Jan 2025 23:35:04 -0600 Subject: [PATCH 22/42] minor change --- machine-learning/app/config.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/machine-learning/app/config.py b/machine-learning/app/config.py index 0cb773a1fdd7b..ec625d7781a0a 100644 --- a/machine-learning/app/config.py +++ b/machine-learning/app/config.py @@ -29,10 +29,8 @@ class PreloadModelData(BaseModel): facial_recognition: FacialRecognitionSettings = FacialRecognitionSettings() # Define fallback environment variables - clip_model_fallback: str | None = Field(default=None, env="MACHINE_LEARNING_PRELOAD__CLIP") - facial_recognition_model_fallback: str | None = Field( - default=None, env="MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION" - ) + clip_model_fallback: str | None = Field(None, env="MACHINE_LEARNING_PRELOAD__CLIP") + facial_recognition_model_fallback: str | None = Field(None, env="MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION") class MaxBatchSize(BaseModel): From 648851ec84fe5d35ab097d9bfa9d819ebb8ed683 Mon Sep 17 00:00:00 2001 From: Jared Dueck Date: Mon, 13 Jan 2025 23:44:55 -0600 Subject: [PATCH 23/42] minor change --- docs/docs/install/environment-variables.md | 9 ++++----- machine-learning/app/config.py | 11 +++++++++-- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/docs/docs/install/environment-variables.md b/docs/docs/install/environment-variables.md index 1f4dff834d93b..a4c9c94cd11e5 100644 --- a/docs/docs/install/environment-variables.md +++ b/docs/docs/install/environment-variables.md @@ -160,11 +160,10 @@ Redis (Sentinel) URL example JSON before encoding: | `MACHINE_LEARNING_HTTP_KEEPALIVE_TIMEOUT_S`\*3 | HTTP Keep-alive time in seconds | `2` | machine learning | | `MACHINE_LEARNING_WORKER_TIMEOUT` | Maximum time (s) of unresponsiveness before a worker is killed | `120` (`300` if using OpenVINO) | machine learning | | `MACHINE_LEARNING_PRELOAD__CLIP__MODEL` | Name of a CLIP model to be preloaded and kept in cache | | machine learning | -| `MACHINE_LEARNING_PRELOAD__CLIP__TEXTUAL` | Preloads the textual model | `True` | machine learning | -| `MACHINE_LEARNING_PRELOAD__CLIP__VISUAL` | Preloads the visual model | `True` | machine learning | -| `MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION__MODEL` | Name of a facial recognition model to be preloaded and kept in cache | | machine learning | -| `MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION__RECOGNITION` | Preloads the recognition model | `True` | machine learning | -| `MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION__DETECTION` | Preloads the detection model | `True` | machine learning | +| `MACHINE_LEARNING_PRELOAD__CLIP__TEXTUAL` | Name of the textual CLIP model to be preloaded and kept in cache | | machine learning | +| `MACHINE_LEARNING_PRELOAD__CLIP__VISUAL` | Name of the visual CLIP model to be preloaded and kept in cache | | machine learning | +| `MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION__RECOGNITION` | Name of the recognition portion of the facial recognition model to be preloaded and kept in cache | | machine learning | +| `MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION__DETECTION` | Name of the detection portion of the facial recognition model to be preloaded and kept in cache | | machine learning | | `MACHINE_LEARNING_ANN` | Enable ARM-NN hardware acceleration if supported | `True` | machine learning | | `MACHINE_LEARNING_ANN_FP16_TURBO` | Execute operations in FP16 precision: increasing speed, reducing precision (applies only to ARM-NN) | `False` | machine learning | | `MACHINE_LEARNING_ANN_TUNING_LEVEL` | ARM-NN GPU tuning level (1: rapid, 2: normal, 3: exhaustive) | `2` | machine learning | diff --git a/machine-learning/app/config.py b/machine-learning/app/config.py index ec625d7781a0a..07657c543b771 100644 --- a/machine-learning/app/config.py +++ b/machine-learning/app/config.py @@ -24,13 +24,20 @@ class FacialRecognitionSettings(BaseModel): detection: str | None = None +# Define str | None as a separate variable +clip_model_fallback: str | None = None +facial_recognition_model_fallback: str | None = None + + class PreloadModelData(BaseModel): clip: ClipSettings = ClipSettings() facial_recognition: FacialRecognitionSettings = FacialRecognitionSettings() # Define fallback environment variables - clip_model_fallback: str | None = Field(None, env="MACHINE_LEARNING_PRELOAD__CLIP") - facial_recognition_model_fallback: str | None = Field(None, env="MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION") + clip_model_fallback: str | None = Field(clip_model_fallback, env="MACHINE_LEARNING_PRELOAD__CLIP") + facial_recognition_model_fallback: str | None = Field( + facial_recognition_model_fallback, env="MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION" + ) class MaxBatchSize(BaseModel): From 624445b3ceb21622405270e3c747183e034d1e70 Mon Sep 17 00:00:00 2001 From: Jared Dueck Date: Mon, 13 Jan 2025 23:49:46 -0600 Subject: [PATCH 24/42] minor change --- machine-learning/app/config.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/machine-learning/app/config.py b/machine-learning/app/config.py index 07657c543b771..4c2d70f612233 100644 --- a/machine-learning/app/config.py +++ b/machine-learning/app/config.py @@ -13,6 +13,9 @@ from uvicorn import Server from uvicorn.workers import UvicornWorker +# for fallback environment variables +from typing import Optional + class ClipSettings(BaseModel): textual: str | None = None @@ -24,20 +27,13 @@ class FacialRecognitionSettings(BaseModel): detection: str | None = None -# Define str | None as a separate variable -clip_model_fallback: str | None = None -facial_recognition_model_fallback: str | None = None - - class PreloadModelData(BaseModel): clip: ClipSettings = ClipSettings() facial_recognition: FacialRecognitionSettings = FacialRecognitionSettings() # Define fallback environment variables - clip_model_fallback: str | None = Field(clip_model_fallback, env="MACHINE_LEARNING_PRELOAD__CLIP") - facial_recognition_model_fallback: str | None = Field( - facial_recognition_model_fallback, env="MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION" - ) + clip_model_fallback: Optional[str] = Field(None, env="MACHINE_LEARNING_PRELOAD__CLIP") + facial_recognition_model_fallback: Optional[str] = Field(None, env="MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION") class MaxBatchSize(BaseModel): From e2b5f7185da3af96aeef6a16aac8a2cff88ac37a Mon Sep 17 00:00:00 2001 From: Jared Dueck Date: Mon, 13 Jan 2025 23:56:37 -0600 Subject: [PATCH 25/42] minor update --- machine-learning/app/config.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/machine-learning/app/config.py b/machine-learning/app/config.py index 4c2d70f612233..0bcdaaf18caf5 100644 --- a/machine-learning/app/config.py +++ b/machine-learning/app/config.py @@ -4,6 +4,7 @@ import sys from pathlib import Path from socket import socket +from typing import Optional from gunicorn.arbiter import Arbiter from pydantic import BaseModel, Field @@ -13,9 +14,6 @@ from uvicorn import Server from uvicorn.workers import UvicornWorker -# for fallback environment variables -from typing import Optional - class ClipSettings(BaseModel): textual: str | None = None @@ -27,6 +25,11 @@ class FacialRecognitionSettings(BaseModel): detection: str | None = None +# Define str | None as a separate variable +clip_model_fallback: str | None = None +facial_recognition_model_fallback: str | None = None + + class PreloadModelData(BaseModel): clip: ClipSettings = ClipSettings() facial_recognition: FacialRecognitionSettings = FacialRecognitionSettings() From 71527ead412aad729171dae2a9cfc46e19a9feb7 Mon Sep 17 00:00:00 2001 From: Jared Dueck Date: Tue, 14 Jan 2025 00:52:48 -0600 Subject: [PATCH 26/42] add logs, resolve errors --- machine-learning/app/config.py | 31 ++++++++++++++++++++++--------- machine-learning/app/test_main.py | 12 ++++++++---- 2 files changed, 30 insertions(+), 13 deletions(-) diff --git a/machine-learning/app/config.py b/machine-learning/app/config.py index 0bcdaaf18caf5..405cad0c6b3de 100644 --- a/machine-learning/app/config.py +++ b/machine-learning/app/config.py @@ -4,10 +4,9 @@ import sys from pathlib import Path from socket import socket -from typing import Optional from gunicorn.arbiter import Arbiter -from pydantic import BaseModel, Field +from pydantic import BaseModel from pydantic_settings import BaseSettings, SettingsConfigDict from rich.console import Console from rich.logging import RichHandler @@ -25,18 +24,32 @@ class FacialRecognitionSettings(BaseModel): detection: str | None = None -# Define str | None as a separate variable -clip_model_fallback: str | None = None -facial_recognition_model_fallback: str | None = None - - class PreloadModelData(BaseModel): clip: ClipSettings = ClipSettings() facial_recognition: FacialRecognitionSettings = FacialRecognitionSettings() # Define fallback environment variables - clip_model_fallback: Optional[str] = Field(None, env="MACHINE_LEARNING_PRELOAD__CLIP") - facial_recognition_model_fallback: Optional[str] = Field(None, env="MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION") + clip_model_fallback: str | None = os.getenv("MACHINE_LEARNING_PRELOAD__CLIP", None) + facial_recognition_model_fallback: str | None = os.getenv("MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION", None) + + def update_from_fallbacks(self) -> None: # Add type annotation here + # Apply fallback to clip model + if not self.clip.textual and self.clip_model_fallback: + self.clip.textual = self.clip_model_fallback + logger.warning("Deprecated environment variable: MACHINE_LEARNING_PRELOAD__CLIP. Use MACHINE_LEARNING_PRELOAD__CLIP__TEXTUAL instead.") + + if not self.clip.visual and self.clip_model_fallback: + self.clip.visual = self.clip_model_fallback + logger.warning("Deprecated environment variable: MACHINE_LEARNING_PRELOAD__CLIP. Use MACHINE_LEARNING_PRELOAD__CLIP__VISUAL instead.") + + # Apply fallback to facial recognition model + if not self.facial_recognition.recognition and self.facial_recognition_model_fallback: + self.facial_recognition.recognition = self.facial_recognition_model_fallback + logger.warning("Deprecated environment variable: MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION. Use MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION__RECOGNITION instead.") + + if not self.facial_recognition.detection and self.facial_recognition_model_fallback: + self.facial_recognition.detection = self.facial_recognition_model_fallback + logger.warning("Deprecated environment variable: MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION. Use MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION__DETECTION instead.") class MaxBatchSize(BaseModel): diff --git a/machine-learning/app/test_main.py b/machine-learning/app/test_main.py index e5cb63997cfb3..6f349381446bd 100644 --- a/machine-learning/app/test_main.py +++ b/machine-learning/app/test_main.py @@ -704,7 +704,8 @@ async def test_preloads_clip_models(self, monkeypatch: MonkeyPatch, mock_get_mod settings = Settings() assert settings.preload is not None - assert settings.preload.clip == "ViT-B-32__openai" + assert settings.preload.clip.textual == "ViT-B-32__openai" + assert settings.preload.clip.visual == "ViT-B-32__openai" model_cache = ModelCache() monkeypatch.setattr("app.main.model_cache", model_cache) @@ -725,7 +726,8 @@ async def test_preloads_facial_recognition_models( settings = Settings() assert settings.preload is not None - assert settings.preload.facial_recognition == "buffalo_s" + assert settings.preload.facial_recognition.detection == "buffalo_s" + assert settings.preload.facial_recognition.recognition == "buffalo_s" model_cache = ModelCache() monkeypatch.setattr("app.main.model_cache", model_cache) @@ -745,8 +747,10 @@ async def test_preloads_all_models(self, monkeypatch: MonkeyPatch, mock_get_mode settings = Settings() assert settings.preload is not None - assert settings.preload.clip == "ViT-B-32__openai" - assert settings.preload.facial_recognition == "buffalo_s" + assert settings.preload.clip.visual == "ViT-B-32__openai" + assert settings.preload.clip.textual == "ViT-B-32__openai" + assert settings.preload.facial_recognition.recognition == "buffalo_s" + assert settings.preload.facial_recognition.detection == "buffalo_s" model_cache = ModelCache() monkeypatch.setattr("app.main.model_cache", model_cache) From 8cfa05585473ad0c43adb402d386bc25e2bf19de Mon Sep 17 00:00:00 2001 From: Jared Dueck Date: Tue, 14 Jan 2025 01:01:15 -0600 Subject: [PATCH 27/42] minor change --- machine-learning/app/config.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/machine-learning/app/config.py b/machine-learning/app/config.py index 405cad0c6b3de..cf2fb101a02ba 100644 --- a/machine-learning/app/config.py +++ b/machine-learning/app/config.py @@ -36,20 +36,32 @@ def update_from_fallbacks(self) -> None: # Add type annotation here # Apply fallback to clip model if not self.clip.textual and self.clip_model_fallback: self.clip.textual = self.clip_model_fallback - logger.warning("Deprecated environment variable: MACHINE_LEARNING_PRELOAD__CLIP. Use MACHINE_LEARNING_PRELOAD__CLIP__TEXTUAL instead.") + log.warning( + "Deprecated env variable: MACHINE_LEARNING_PRELOAD__CLIP. " + "Use MACHINE_LEARNING_PRELOAD__CLIP__TEXTUAL instead." + ) if not self.clip.visual and self.clip_model_fallback: self.clip.visual = self.clip_model_fallback - logger.warning("Deprecated environment variable: MACHINE_LEARNING_PRELOAD__CLIP. Use MACHINE_LEARNING_PRELOAD__CLIP__VISUAL instead.") + log.warning( + "Deprecated environment variable: MACHINE_LEARNING_PRELOAD__CLIP. " + "Use MACHINE_LEARNING_PRELOAD__CLIP__VISUAL instead." + ) # Apply fallback to facial recognition model if not self.facial_recognition.recognition and self.facial_recognition_model_fallback: self.facial_recognition.recognition = self.facial_recognition_model_fallback - logger.warning("Deprecated environment variable: MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION. Use MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION__RECOGNITION instead.") + log.warning( + "Deprecated environment variable: MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION. " + "Use MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION__RECOGNITION instead." + ) if not self.facial_recognition.detection and self.facial_recognition_model_fallback: self.facial_recognition.detection = self.facial_recognition_model_fallback - logger.warning("Deprecated environment variable: MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION. Use MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION__DETECTION instead.") + log.warning( + "Deprecated environment variable: MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION. " + "Use MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION__DETECTION instead." + ) class MaxBatchSize(BaseModel): From 85ca87c0f1552a2dc56e77dd359d76864d5c9d3a Mon Sep 17 00:00:00 2001 From: Jared Dueck Date: Tue, 14 Jan 2025 01:27:56 -0600 Subject: [PATCH 28/42] add new enviornment variables --- machine-learning/app/test_main.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/machine-learning/app/test_main.py b/machine-learning/app/test_main.py index 6f349381446bd..5da3baded7ad8 100644 --- a/machine-learning/app/test_main.py +++ b/machine-learning/app/test_main.py @@ -700,7 +700,8 @@ async def test_raises_exception_if_unknown_model_name(self) -> None: await model_cache.get("test_model_name", ModelType.TEXTUAL, ModelTask.SEARCH) async def test_preloads_clip_models(self, monkeypatch: MonkeyPatch, mock_get_model: mock.Mock) -> None: - os.environ["MACHINE_LEARNING_PRELOAD__CLIP"] = "ViT-B-32__openai" + os.environ["MACHINE_LEARNING_PRELOAD__CLIP__TEXTUAL"] = "ViT-B-32__openai" + os.environ["MACHINE_LEARNING_PRELOAD__CLIP__VISUAL"] = "ViT-B-32__openai" settings = Settings() assert settings.preload is not None @@ -722,7 +723,8 @@ async def test_preloads_clip_models(self, monkeypatch: MonkeyPatch, mock_get_mod async def test_preloads_facial_recognition_models( self, monkeypatch: MonkeyPatch, mock_get_model: mock.Mock ) -> None: - os.environ["MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION"] = "buffalo_s" + os.environ["MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION__DETECTION"] = "buffalo_s" + os.environ["MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION__RECOGNITION"] = "buffalo_s" settings = Settings() assert settings.preload is not None @@ -742,8 +744,10 @@ async def test_preloads_facial_recognition_models( ) async def test_preloads_all_models(self, monkeypatch: MonkeyPatch, mock_get_model: mock.Mock) -> None: - os.environ["MACHINE_LEARNING_PRELOAD__CLIP"] = "ViT-B-32__openai" - os.environ["MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION"] = "buffalo_s" + os.environ["MACHINE_LEARNING_PRELOAD__CLIP__TEXTUAL"] = "ViT-B-32__openai" + os.environ["MACHINE_LEARNING_PRELOAD__CLIP__VISUAL"] = "ViT-B-32__openai" + os.environ["MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION__RECOGNITION"] = "buffalo_s" + os.environ["MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION__DETECTION"] = "buffalo_s" settings = Settings() assert settings.preload is not None From 1a26a3f95f0957b400622815fb2eda807c3207f4 Mon Sep 17 00:00:00 2001 From: Jared Dueck Date: Tue, 14 Jan 2025 15:38:07 -0600 Subject: [PATCH 29/42] minor revisons --- docs/docs/install/environment-variables.md | 1 - machine-learning/app/config.py | 24 +++++++--------------- 2 files changed, 7 insertions(+), 18 deletions(-) diff --git a/docs/docs/install/environment-variables.md b/docs/docs/install/environment-variables.md index a4c9c94cd11e5..b4cb905a0cbc4 100644 --- a/docs/docs/install/environment-variables.md +++ b/docs/docs/install/environment-variables.md @@ -159,7 +159,6 @@ Redis (Sentinel) URL example JSON before encoding: | `MACHINE_LEARNING_WORKERS`\*2 | Number of worker processes to spawn | `1` | machine learning | | `MACHINE_LEARNING_HTTP_KEEPALIVE_TIMEOUT_S`\*3 | HTTP Keep-alive time in seconds | `2` | machine learning | | `MACHINE_LEARNING_WORKER_TIMEOUT` | Maximum time (s) of unresponsiveness before a worker is killed | `120` (`300` if using OpenVINO) | machine learning | -| `MACHINE_LEARNING_PRELOAD__CLIP__MODEL` | Name of a CLIP model to be preloaded and kept in cache | | machine learning | | `MACHINE_LEARNING_PRELOAD__CLIP__TEXTUAL` | Name of the textual CLIP model to be preloaded and kept in cache | | machine learning | | `MACHINE_LEARNING_PRELOAD__CLIP__VISUAL` | Name of the visual CLIP model to be preloaded and kept in cache | | machine learning | | `MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION__RECOGNITION` | Name of the recognition portion of the facial recognition model to be preloaded and kept in cache | | machine learning | diff --git a/machine-learning/app/config.py b/machine-learning/app/config.py index cf2fb101a02ba..2694c401b51e3 100644 --- a/machine-learning/app/config.py +++ b/machine-learning/app/config.py @@ -34,33 +34,23 @@ class PreloadModelData(BaseModel): def update_from_fallbacks(self) -> None: # Add type annotation here # Apply fallback to clip model - if not self.clip.textual and self.clip_model_fallback: + if self.clip_model_fallback: self.clip.textual = self.clip_model_fallback - log.warning( - "Deprecated env variable: MACHINE_LEARNING_PRELOAD__CLIP. " - "Use MACHINE_LEARNING_PRELOAD__CLIP__TEXTUAL instead." - ) - - if not self.clip.visual and self.clip_model_fallback: self.clip.visual = self.clip_model_fallback log.warning( - "Deprecated environment variable: MACHINE_LEARNING_PRELOAD__CLIP. " - "Use MACHINE_LEARNING_PRELOAD__CLIP__VISUAL instead." + "Deprecated env variable: MACHINE_LEARNING_PRELOAD__CLIP. " + "Use MACHINE_LEARNING_PRELOAD__CLIP__TEXTUAL and " + "MACHINE_LEARNING_PRELOAD__CLIP__VISUAL instead." ) # Apply fallback to facial recognition model - if not self.facial_recognition.recognition and self.facial_recognition_model_fallback: + if self.facial_recognition_model_fallback: self.facial_recognition.recognition = self.facial_recognition_model_fallback - log.warning( - "Deprecated environment variable: MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION. " - "Use MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION__RECOGNITION instead." - ) - - if not self.facial_recognition.detection and self.facial_recognition_model_fallback: self.facial_recognition.detection = self.facial_recognition_model_fallback log.warning( "Deprecated environment variable: MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION. " - "Use MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION__DETECTION instead." + "Use MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION__RECOGNITION and " + "MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION__DETECTION instead." ) From 10446e136e8c887dcdc4488c75714a4c9044164b Mon Sep 17 00:00:00 2001 From: Jared Dueck Date: Tue, 14 Jan 2025 15:59:02 -0600 Subject: [PATCH 30/42] remove comments --- machine-learning/app/config.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/machine-learning/app/config.py b/machine-learning/app/config.py index 2694c401b51e3..fa7dc61d47fde 100644 --- a/machine-learning/app/config.py +++ b/machine-learning/app/config.py @@ -28,12 +28,10 @@ class PreloadModelData(BaseModel): clip: ClipSettings = ClipSettings() facial_recognition: FacialRecognitionSettings = FacialRecognitionSettings() - # Define fallback environment variables clip_model_fallback: str | None = os.getenv("MACHINE_LEARNING_PRELOAD__CLIP", None) facial_recognition_model_fallback: str | None = os.getenv("MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION", None) - def update_from_fallbacks(self) -> None: # Add type annotation here - # Apply fallback to clip model + def update_from_fallbacks(self) -> None: if self.clip_model_fallback: self.clip.textual = self.clip_model_fallback self.clip.visual = self.clip_model_fallback @@ -43,7 +41,6 @@ def update_from_fallbacks(self) -> None: # Add type annotation here "MACHINE_LEARNING_PRELOAD__CLIP__VISUAL instead." ) - # Apply fallback to facial recognition model if self.facial_recognition_model_fallback: self.facial_recognition.recognition = self.facial_recognition_model_fallback self.facial_recognition.detection = self.facial_recognition_model_fallback From 78dac700462b29aa25bf8c81fcd331c46291ee9a Mon Sep 17 00:00:00 2001 From: Jared Dueck Date: Thu, 16 Jan 2025 16:12:52 -0600 Subject: [PATCH 31/42] add additional variables to ML (fixed) --- machine-learning/app/config.py | 39 +++++++++++++++------------------- machine-learning/app/main.py | 21 +++++++++++++++++- 2 files changed, 37 insertions(+), 23 deletions(-) diff --git a/machine-learning/app/config.py b/machine-learning/app/config.py index fa7dc61d47fde..af3c8f7cd097b 100644 --- a/machine-learning/app/config.py +++ b/machine-learning/app/config.py @@ -17,45 +17,40 @@ class ClipSettings(BaseModel): textual: str | None = None visual: str | None = None + fallback: str | None = None class FacialRecognitionSettings(BaseModel): recognition: str | None = None detection: str | None = None + fallback: str | None = None class PreloadModelData(BaseModel): clip: ClipSettings = ClipSettings() facial_recognition: FacialRecognitionSettings = FacialRecognitionSettings() - clip_model_fallback: str | None = os.getenv("MACHINE_LEARNING_PRELOAD__CLIP", None) - facial_recognition_model_fallback: str | None = os.getenv("MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION", None) - - def update_from_fallbacks(self) -> None: - if self.clip_model_fallback: - self.clip.textual = self.clip_model_fallback - self.clip.visual = self.clip_model_fallback - log.warning( - "Deprecated env variable: MACHINE_LEARNING_PRELOAD__CLIP. " - "Use MACHINE_LEARNING_PRELOAD__CLIP__TEXTUAL and " - "MACHINE_LEARNING_PRELOAD__CLIP__VISUAL instead." - ) - - if self.facial_recognition_model_fallback: - self.facial_recognition.recognition = self.facial_recognition_model_fallback - self.facial_recognition.detection = self.facial_recognition_model_fallback - log.warning( - "Deprecated environment variable: MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION. " - "Use MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION__RECOGNITION and " - "MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION__DETECTION instead." - ) - class MaxBatchSize(BaseModel): facial_recognition: int | None = None class Settings(BaseSettings): + + if os.environ.get("MACHINE_LEARNING_PRELOAD__CLIP") is not None: + os.environ["MACHINE_LEARNING_PRELOAD__CLIP__TEXTUAL"] = os.environ["MACHINE_LEARNING_PRELOAD__CLIP"] + os.environ["MACHINE_LEARNING_PRELOAD__CLIP__VISUAL"] = os.environ["MACHINE_LEARNING_PRELOAD__CLIP"] + os.environ["MACHINE_LEARNING_PRELOAD__CLIP__FALLBACK"] = "True" + del os.environ["MACHINE_LEARNING_PRELOAD__CLIP"] + if os.environ.get("MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION") is not None: + os.environ["MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION__RECOGNITION"] = os.environ[ + "MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION" + ] + os.environ["MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION__DETECTION"] = os.environ[ + "MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION" + ] + os.environ["MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION__FALLBACK"] = "True" + del os.environ["MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION"] model_config = SettingsConfigDict( env_prefix="MACHINE_LEARNING_", case_sensitive=False, diff --git a/machine-learning/app/main.py b/machine-learning/app/main.py index ee8881b6c604c..cd7575dcbbdc2 100644 --- a/machine-learning/app/main.py +++ b/machine-learning/app/main.py @@ -75,7 +75,12 @@ async def lifespan(_: FastAPI) -> AsyncGenerator[None, None]: async def preload_models(preload: PreloadModelData) -> None: - log.info(f"Preloading models: {preload}") + log.info( + f"Preloading models: " + f"clip_textual={preload.clip.textual} clip_visual={preload.clip.visual} " + f"facial_recognition_recognition={preload.facial_recognition.recognition} " + f"facial_recognition_detection={preload.facial_recognition.detection}" + ) if preload.clip.textual is not None: model = await model_cache.get(preload.clip.textual, ModelType.TEXTUAL, ModelTask.SEARCH) @@ -101,6 +106,20 @@ async def preload_models(preload: PreloadModelData) -> None: ) await load(model) + if preload.clip.fallback is not None: + log.warning( + "Deprecated env variable: MACHINE_LEARNING_PRELOAD__CLIP. " + "Use MACHINE_LEARNING_PRELOAD__CLIP__TEXTUAL and " + "MACHINE_LEARNING_PRELOAD__CLIP__VISUAL instead." + ) + + if preload.facial_recognition.fallback is not None: + log.warning( + "Deprecated env variable: MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION. " + "Use MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION__DETECTION and " + "MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION__RECOGNITION instead." + ) + def update_state() -> Iterator[None]: global active_requests, last_called From 9e0fc1bf6f2169d58051a88f66482550ab6a3a78 Mon Sep 17 00:00:00 2001 From: Jared Dueck Date: Thu, 16 Jan 2025 16:19:48 -0600 Subject: [PATCH 32/42] add additional variables to ML (fixed) --- machine-learning/app/config.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/machine-learning/app/config.py b/machine-learning/app/config.py index af3c8f7cd097b..b53222d946396 100644 --- a/machine-learning/app/config.py +++ b/machine-learning/app/config.py @@ -27,16 +27,6 @@ class FacialRecognitionSettings(BaseModel): class PreloadModelData(BaseModel): - clip: ClipSettings = ClipSettings() - facial_recognition: FacialRecognitionSettings = FacialRecognitionSettings() - - -class MaxBatchSize(BaseModel): - facial_recognition: int | None = None - - -class Settings(BaseSettings): - if os.environ.get("MACHINE_LEARNING_PRELOAD__CLIP") is not None: os.environ["MACHINE_LEARNING_PRELOAD__CLIP__TEXTUAL"] = os.environ["MACHINE_LEARNING_PRELOAD__CLIP"] os.environ["MACHINE_LEARNING_PRELOAD__CLIP__VISUAL"] = os.environ["MACHINE_LEARNING_PRELOAD__CLIP"] @@ -51,6 +41,15 @@ class Settings(BaseSettings): ] os.environ["MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION__FALLBACK"] = "True" del os.environ["MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION"] + clip: ClipSettings = ClipSettings() + facial_recognition: FacialRecognitionSettings = FacialRecognitionSettings() + + +class MaxBatchSize(BaseModel): + facial_recognition: int | None = None + + +class Settings(BaseSettings): model_config = SettingsConfigDict( env_prefix="MACHINE_LEARNING_", case_sensitive=False, From 45841bb6af9235b880128d847985ac39cabeb7ee Mon Sep 17 00:00:00 2001 From: Jared Dueck Date: Thu, 16 Jan 2025 17:07:05 -0600 Subject: [PATCH 33/42] add additional variables to ML --- machine-learning/app/config.py | 34 ++++++++++++++++------------------ machine-learning/app/main.py | 9 ++++----- 2 files changed, 20 insertions(+), 23 deletions(-) diff --git a/machine-learning/app/config.py b/machine-learning/app/config.py index b53222d946396..39632845bc87f 100644 --- a/machine-learning/app/config.py +++ b/machine-learning/app/config.py @@ -17,32 +17,30 @@ class ClipSettings(BaseModel): textual: str | None = None visual: str | None = None - fallback: str | None = None class FacialRecognitionSettings(BaseModel): recognition: str | None = None detection: str | None = None - fallback: str | None = None class PreloadModelData(BaseModel): - if os.environ.get("MACHINE_LEARNING_PRELOAD__CLIP") is not None: - os.environ["MACHINE_LEARNING_PRELOAD__CLIP__TEXTUAL"] = os.environ["MACHINE_LEARNING_PRELOAD__CLIP"] - os.environ["MACHINE_LEARNING_PRELOAD__CLIP__VISUAL"] = os.environ["MACHINE_LEARNING_PRELOAD__CLIP"] - os.environ["MACHINE_LEARNING_PRELOAD__CLIP__FALLBACK"] = "True" - del os.environ["MACHINE_LEARNING_PRELOAD__CLIP"] - if os.environ.get("MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION") is not None: - os.environ["MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION__RECOGNITION"] = os.environ[ - "MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION" - ] - os.environ["MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION__DETECTION"] = os.environ[ - "MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION" - ] - os.environ["MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION__FALLBACK"] = "True" - del os.environ["MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION"] - clip: ClipSettings = ClipSettings() - facial_recognition: FacialRecognitionSettings = FacialRecognitionSettings() + clip: ClipSettings | str = ClipSettings() + facial_recognition: FacialRecognitionSettings | str = FacialRecognitionSettings() + clip_fallback: str | None = None + facial_recognition_fallback: str | None = None + + def __init__(self, **kwargs): + super().__init__(**kwargs) + + # If clip is a string, set both textual and visual to that string + if isinstance(self.clip, str): + self.clip = ClipSettings(textual=self.clip, visual=self.clip) + self.clip_fallback = "True" + if isinstance(self.facial_recognition, str): + self.clip = ClipSettings(textual=self.facial_recognition, visual=self.facial_recognition) + self.facial_recognition_fallback = "True" + class MaxBatchSize(BaseModel): diff --git a/machine-learning/app/main.py b/machine-learning/app/main.py index cd7575dcbbdc2..7f92ba283f549 100644 --- a/machine-learning/app/main.py +++ b/machine-learning/app/main.py @@ -77,9 +77,8 @@ async def lifespan(_: FastAPI) -> AsyncGenerator[None, None]: async def preload_models(preload: PreloadModelData) -> None: log.info( f"Preloading models: " - f"clip_textual={preload.clip.textual} clip_visual={preload.clip.visual} " - f"facial_recognition_recognition={preload.facial_recognition.recognition} " - f"facial_recognition_detection={preload.facial_recognition.detection}" + f"clip:{preload.clip} " + f"facial_recognition:{preload.facial_recognition}" ) if preload.clip.textual is not None: @@ -106,14 +105,14 @@ async def preload_models(preload: PreloadModelData) -> None: ) await load(model) - if preload.clip.fallback is not None: + if preload.clip_fallback is not None: log.warning( "Deprecated env variable: MACHINE_LEARNING_PRELOAD__CLIP. " "Use MACHINE_LEARNING_PRELOAD__CLIP__TEXTUAL and " "MACHINE_LEARNING_PRELOAD__CLIP__VISUAL instead." ) - if preload.facial_recognition.fallback is not None: + if preload.facial_recognition_fallback is not None: log.warning( "Deprecated env variable: MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION. " "Use MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION__DETECTION and " From aae30f391e8a88238bf651210310a8290aa8d3b6 Mon Sep 17 00:00:00 2001 From: Jared Dueck Date: Thu, 16 Jan 2025 17:10:47 -0600 Subject: [PATCH 34/42] formatting --- machine-learning/app/config.py | 1 - machine-learning/app/main.py | 6 +----- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/machine-learning/app/config.py b/machine-learning/app/config.py index 39632845bc87f..eea2e8c52b7c0 100644 --- a/machine-learning/app/config.py +++ b/machine-learning/app/config.py @@ -42,7 +42,6 @@ def __init__(self, **kwargs): self.facial_recognition_fallback = "True" - class MaxBatchSize(BaseModel): facial_recognition: int | None = None diff --git a/machine-learning/app/main.py b/machine-learning/app/main.py index 7f92ba283f549..4bf9f26891a75 100644 --- a/machine-learning/app/main.py +++ b/machine-learning/app/main.py @@ -75,11 +75,7 @@ async def lifespan(_: FastAPI) -> AsyncGenerator[None, None]: async def preload_models(preload: PreloadModelData) -> None: - log.info( - f"Preloading models: " - f"clip:{preload.clip} " - f"facial_recognition:{preload.facial_recognition}" - ) + log.info(f"Preloading models: " f"clip:{preload.clip} " f"facial_recognition:{preload.facial_recognition}") if preload.clip.textual is not None: model = await model_cache.get(preload.clip.textual, ModelType.TEXTUAL, ModelTask.SEARCH) From 35dd7489c2ae486066d2f795763a4ac0eda5ef6c Mon Sep 17 00:00:00 2001 From: Jared Dueck Date: Thu, 16 Jan 2025 17:13:57 -0600 Subject: [PATCH 35/42] remove comment --- machine-learning/app/config.py | 1 - 1 file changed, 1 deletion(-) diff --git a/machine-learning/app/config.py b/machine-learning/app/config.py index eea2e8c52b7c0..7fa60b6f1e257 100644 --- a/machine-learning/app/config.py +++ b/machine-learning/app/config.py @@ -33,7 +33,6 @@ class PreloadModelData(BaseModel): def __init__(self, **kwargs): super().__init__(**kwargs) - # If clip is a string, set both textual and visual to that string if isinstance(self.clip, str): self.clip = ClipSettings(textual=self.clip, visual=self.clip) self.clip_fallback = "True" From d862aec72a6d14965752074ac2afd6f5e0c4cf0f Mon Sep 17 00:00:00 2001 From: Jared Dueck Date: Fri, 17 Jan 2025 01:48:44 -0600 Subject: [PATCH 36/42] remove mypy error --- machine-learning/app/config.py | 27 +++++++++++++-------------- machine-learning/app/main.py | 12 ++++++------ 2 files changed, 19 insertions(+), 20 deletions(-) diff --git a/machine-learning/app/config.py b/machine-learning/app/config.py index 7fa60b6f1e257..384574efd1a1a 100644 --- a/machine-learning/app/config.py +++ b/machine-learning/app/config.py @@ -4,6 +4,7 @@ import sys from pathlib import Path from socket import socket +from typing import Any from gunicorn.arbiter import Arbiter from pydantic import BaseModel @@ -25,20 +26,18 @@ class FacialRecognitionSettings(BaseModel): class PreloadModelData(BaseModel): - clip: ClipSettings | str = ClipSettings() - facial_recognition: FacialRecognitionSettings | str = FacialRecognitionSettings() - clip_fallback: str | None = None - facial_recognition_fallback: str | None = None - - def __init__(self, **kwargs): - super().__init__(**kwargs) - - if isinstance(self.clip, str): - self.clip = ClipSettings(textual=self.clip, visual=self.clip) - self.clip_fallback = "True" - if isinstance(self.facial_recognition, str): - self.clip = ClipSettings(textual=self.facial_recognition, visual=self.facial_recognition) - self.facial_recognition_fallback = "True" + clip_fallback: str | None = os.getenv("MACHINE_LEARNING_PRELOAD__CLIP", None) + facial_recognition_fallback: str | None = os.getenv("MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION", None) + if clip_fallback is not None: + os.environ["MACHINE_LEARNING_PRELOAD__CLIP__TEXTUAL"] = clip_fallback + os.environ["MACHINE_LEARNING_PRELOAD__CLIP__VISUAL"] = clip_fallback + del os.environ["MACHINE_LEARNING_PRELOAD__CLIP"] + if facial_recognition_fallback is not None: + os.environ["MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION__RECOGNITION"] = facial_recognition_fallback + os.environ["MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION__DETECTION"] = facial_recognition_fallback + del os.environ["MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION"] + clip: ClipSettings = ClipSettings() + facial_recognition: FacialRecognitionSettings = FacialRecognitionSettings() class MaxBatchSize(BaseModel): diff --git a/machine-learning/app/main.py b/machine-learning/app/main.py index 4bf9f26891a75..9ef672f9aebc0 100644 --- a/machine-learning/app/main.py +++ b/machine-learning/app/main.py @@ -103,16 +103,16 @@ async def preload_models(preload: PreloadModelData) -> None: if preload.clip_fallback is not None: log.warning( - "Deprecated env variable: MACHINE_LEARNING_PRELOAD__CLIP. " - "Use MACHINE_LEARNING_PRELOAD__CLIP__TEXTUAL and " - "MACHINE_LEARNING_PRELOAD__CLIP__VISUAL instead." + "Deprecated env variable: 'MACHINE_LEARNING_PRELOAD__CLIP'. " + "Use 'MACHINE_LEARNING_PRELOAD__CLIP__TEXTUAL' and " + "'MACHINE_LEARNING_PRELOAD__CLIP__VISUAL' instead." ) if preload.facial_recognition_fallback is not None: log.warning( - "Deprecated env variable: MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION. " - "Use MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION__DETECTION and " - "MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION__RECOGNITION instead." + "Deprecated env variable: 'MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION'. " + "Use 'MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION__DETECTION' and " + "'MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION__RECOGNITION' instead." ) From 813ffc70760e1ce982d714b95bfc578bcc7c723c Mon Sep 17 00:00:00 2001 From: Jared Dueck Date: Fri, 17 Jan 2025 01:52:18 -0600 Subject: [PATCH 37/42] remove unused module --- machine-learning/app/config.py | 1 - 1 file changed, 1 deletion(-) diff --git a/machine-learning/app/config.py b/machine-learning/app/config.py index 384574efd1a1a..e77531cbd64d4 100644 --- a/machine-learning/app/config.py +++ b/machine-learning/app/config.py @@ -4,7 +4,6 @@ import sys from pathlib import Path from socket import socket -from typing import Any from gunicorn.arbiter import Arbiter from pydantic import BaseModel From f599668425941f4ed40c970786b36ca4c0f6a7a0 Mon Sep 17 00:00:00 2001 From: Jared Dueck Date: Fri, 17 Jan 2025 11:48:17 -0600 Subject: [PATCH 38/42] merge f strings --- machine-learning/app/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/machine-learning/app/main.py b/machine-learning/app/main.py index 9ef672f9aebc0..fb6f84499af00 100644 --- a/machine-learning/app/main.py +++ b/machine-learning/app/main.py @@ -75,7 +75,7 @@ async def lifespan(_: FastAPI) -> AsyncGenerator[None, None]: async def preload_models(preload: PreloadModelData) -> None: - log.info(f"Preloading models: " f"clip:{preload.clip} " f"facial_recognition:{preload.facial_recognition}") + log.info(f"Preloading models: clip:{preload.clip} facial_recognition:{preload.facial_recognition}") if preload.clip.textual is not None: model = await model_cache.get(preload.clip.textual, ModelType.TEXTUAL, ModelTask.SEARCH) From ad008bb99a41ca40c40950b6352b00ab46e7bdb8 Mon Sep 17 00:00:00 2001 From: Jared Dueck Date: Fri, 17 Jan 2025 20:53:08 -0600 Subject: [PATCH 39/42] Add multi-model preload functionality --- docs/docs/install/environment-variables.md | 8 ++++---- machine-learning/app/main.py | 18 ++++++++++-------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/docs/docs/install/environment-variables.md b/docs/docs/install/environment-variables.md index b4cb905a0cbc4..a57eef540d8bf 100644 --- a/docs/docs/install/environment-variables.md +++ b/docs/docs/install/environment-variables.md @@ -159,10 +159,10 @@ Redis (Sentinel) URL example JSON before encoding: | `MACHINE_LEARNING_WORKERS`\*2 | Number of worker processes to spawn | `1` | machine learning | | `MACHINE_LEARNING_HTTP_KEEPALIVE_TIMEOUT_S`\*3 | HTTP Keep-alive time in seconds | `2` | machine learning | | `MACHINE_LEARNING_WORKER_TIMEOUT` | Maximum time (s) of unresponsiveness before a worker is killed | `120` (`300` if using OpenVINO) | machine learning | -| `MACHINE_LEARNING_PRELOAD__CLIP__TEXTUAL` | Name of the textual CLIP model to be preloaded and kept in cache | | machine learning | -| `MACHINE_LEARNING_PRELOAD__CLIP__VISUAL` | Name of the visual CLIP model to be preloaded and kept in cache | | machine learning | -| `MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION__RECOGNITION` | Name of the recognition portion of the facial recognition model to be preloaded and kept in cache | | machine learning | -| `MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION__DETECTION` | Name of the detection portion of the facial recognition model to be preloaded and kept in cache | | machine learning | +| `MACHINE_LEARNING_PRELOAD__CLIP__TEXTUAL` | Comma-separated list of (textual) CLIP model(s) to preload and cache | | machine learning | +| `MACHINE_LEARNING_PRELOAD__CLIP__VISUAL` | Comma-separated list of (visual) CLIP model(s) to preload and cache | | machine learning | +| `MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION__RECOGNITION` | Comma-separated list of (recognition) facial recognition model(s) to preload and cache | | machine learning | +| `MACHINE_LEARNING_PRELOAD__FACIAL_RECOGNITION__DETECTION` | Comma-separated list of (detection) facial recognition model(s) to preload and cache | | machine learning | | `MACHINE_LEARNING_ANN` | Enable ARM-NN hardware acceleration if supported | `True` | machine learning | | `MACHINE_LEARNING_ANN_FP16_TURBO` | Execute operations in FP16 precision: increasing speed, reducing precision (applies only to ARM-NN) | `False` | machine learning | | `MACHINE_LEARNING_ANN_TUNING_LEVEL` | ARM-NN GPU tuning level (1: rapid, 2: normal, 3: exhaustive) | `2` | machine learning | diff --git a/machine-learning/app/main.py b/machine-learning/app/main.py index fb6f84499af00..af49934561d39 100644 --- a/machine-learning/app/main.py +++ b/machine-learning/app/main.py @@ -77,29 +77,31 @@ async def lifespan(_: FastAPI) -> AsyncGenerator[None, None]: async def preload_models(preload: PreloadModelData) -> None: log.info(f"Preloading models: clip:{preload.clip} facial_recognition:{preload.facial_recognition}") + async def load_models(model_string, model_type, model_task): + models = [model.strip() for model in model_string.split(",")] + for model_name in models: + model = await model_cache.get(model_name, model_type, model_task) + await load(model) + if preload.clip.textual is not None: - model = await model_cache.get(preload.clip.textual, ModelType.TEXTUAL, ModelTask.SEARCH) - await load(model) + await load_models(preload.clip.textual, ModelType.TEXTUAL, ModelTask.SEARCH) if preload.clip.visual is not None: - model = await model_cache.get(preload.clip.visual, ModelType.VISUAL, ModelTask.SEARCH) - await load(model) + await load_models(preload.clip.visual, ModelType.VISUAL, ModelTask.SEARCH) if preload.facial_recognition.detection is not None: - model = await model_cache.get( + await load_models( preload.facial_recognition.detection, ModelType.DETECTION, ModelTask.FACIAL_RECOGNITION, ) - await load(model) if preload.facial_recognition.recognition is not None: - model = await model_cache.get( + await load_models( preload.facial_recognition.recognition, ModelType.RECOGNITION, ModelTask.FACIAL_RECOGNITION, ) - await load(model) if preload.clip_fallback is not None: log.warning( From e396c9eba0c218ce43d549165caa637740d0f061 Mon Sep 17 00:00:00 2001 From: Jared Dueck Date: Fri, 17 Jan 2025 21:10:09 -0600 Subject: [PATCH 40/42] Add type annotation --- machine-learning/app/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/machine-learning/app/main.py b/machine-learning/app/main.py index af49934561d39..4681e16fc168d 100644 --- a/machine-learning/app/main.py +++ b/machine-learning/app/main.py @@ -77,7 +77,7 @@ async def lifespan(_: FastAPI) -> AsyncGenerator[None, None]: async def preload_models(preload: PreloadModelData) -> None: log.info(f"Preloading models: clip:{preload.clip} facial_recognition:{preload.facial_recognition}") - async def load_models(model_string, model_type, model_task): + async def load_models(model_string: str, model_type: "ModelType", model_task: "ModelTask") -> None: models = [model.strip() for model in model_string.split(",")] for model_name in models: model = await model_cache.get(model_name, model_type, model_task) From 76c18bfd260045f90965339bb56e150b65ac60ff Mon Sep 17 00:00:00 2001 From: Tempest <110401501+1-tempest@users.noreply.github.com> Date: Sun, 19 Jan 2025 07:27:18 -0600 Subject: [PATCH 41/42] Update machine-learning/app/main.py Co-authored-by: Mert <101130780+mertalev@users.noreply.github.com> --- machine-learning/app/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/machine-learning/app/main.py b/machine-learning/app/main.py index 4681e16fc168d..71d9ad6ccd7ae 100644 --- a/machine-learning/app/main.py +++ b/machine-learning/app/main.py @@ -78,8 +78,8 @@ async def preload_models(preload: PreloadModelData) -> None: log.info(f"Preloading models: clip:{preload.clip} facial_recognition:{preload.facial_recognition}") async def load_models(model_string: str, model_type: "ModelType", model_task: "ModelTask") -> None: - models = [model.strip() for model in model_string.split(",")] - for model_name in models: + for model in model_string.split(","): + model_name = model.strip() model = await model_cache.get(model_name, model_type, model_task) await load(model) From 86627a9355d6372cb9dc3301ad090e22f1828780 Mon Sep 17 00:00:00 2001 From: Tempest <110401501+1-tempest@users.noreply.github.com> Date: Sun, 19 Jan 2025 07:48:53 -0600 Subject: [PATCH 42/42] Update main.py --- machine-learning/app/main.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/machine-learning/app/main.py b/machine-learning/app/main.py index 71d9ad6ccd7ae..0f50257a4e694 100644 --- a/machine-learning/app/main.py +++ b/machine-learning/app/main.py @@ -77,9 +77,9 @@ async def lifespan(_: FastAPI) -> AsyncGenerator[None, None]: async def preload_models(preload: PreloadModelData) -> None: log.info(f"Preloading models: clip:{preload.clip} facial_recognition:{preload.facial_recognition}") - async def load_models(model_string: str, model_type: "ModelType", model_task: "ModelTask") -> None: - for model in model_string.split(","): - model_name = model.strip() + async def load_models(model_string: str, model_type: ModelType, model_task: ModelTask) -> None: + for model_name in model_string.split(","): + model_name = model_name.strip() model = await model_cache.get(model_name, model_type, model_task) await load(model)