Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Application Insights warning on llm model deprecation #2524

Open
wants to merge 15 commits into
base: main
Choose a base branch
from
Open
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ namespace System.AI;
#if not CLEAN25
using System.Environment;
#endif
using System.Telemetry;

codeunit 7769 "AOAI Deployments Impl"
{
Expand All @@ -15,11 +16,15 @@ codeunit 7769 "AOAI Deployments Impl"
InherentPermissions = X;

var
Telemetry: Codeunit Telemetry;
UnableToGetDeploymentNameErr: Label 'Unable to get deployment name, if this is a third party capability you must specify your own deployment name. You may need to contact your partner.';
GPT4oLatestLbl: Label 'gpt-4o-latest', Locked = true;
GPT4oPreviewLbl: Label 'gpt-4o-preview', Locked = true;
GPT4oMiniLatestLbl: Label 'gpt-4o-mini-latest', Locked = true;
GPT4oMiniPreviewLbl: Label 'gpt-4o-mini-preview', Locked = true;
DeprecatedDeployments: Dictionary of [Text, Date];
DeprecationDatesInitialized: Boolean;
DeprecationMessageLbl: Label 'We detected usage of the Azure OpenAI deployment "%1". This model is obsoleted starting %2 and the quality of your results might vary after that date. Check out codeunit 7768 AOAI Deployments to find the supported deployments.', Comment = 'Telemetry message where %1 is the name of the deployment and %2 is the date of deprecation';
#if not CLEAN25
GPT4LatestLbl: Label 'gpt-4-latest', Locked = true;
GPT4PreviewLbl: Label 'gpt-4-preview', Locked = true;
Expand All @@ -28,6 +33,7 @@ codeunit 7769 "AOAI Deployments Impl"
Turbo0301SaasLbl: Label 'turbo-0301', Locked = true;
GPT40613SaasLbl: Label 'gpt4-0613', Locked = true;
Turbo0613SaasLbl: Label 'turbo-0613', Locked = true;

Turbo0301Lbl: Label 'chatGPT_GPT35-turbo-0301', Locked = true;
GPT40613Lbl: Label 'gpt-4-32k', Locked = true;
Turbo031316kLbl: Label 'gpt-35-turbo-16k', Locked = true;
Expand Down Expand Up @@ -103,12 +109,61 @@ codeunit 7769 "AOAI Deployments Impl"
exit(GetDeploymentName(GPT4oMiniLatestLbl, CallerModuleInfo));
end;

// Initializes dictionary of deprecated models
local procedure InitializeDeploymentDeprecationDates()
begin
if DeprecationDatesInitialized then
exit;

// Add deprecated deployments with their deprecation dates here
#if not CLEAN25
DeprecatedDeployments.Add(GPT4LatestLbl, DMY2Date(1, 11, 2024));
DeprecatedDeployments.Add(GPT4PreviewLbl, DMY2Date(1, 11, 2024));
DeprecatedDeployments.Add(GPT35TurboLatestLbl, DMY2Date(1, 11, 2024));
DeprecatedDeployments.Add(GPT35TurboPreviewLbl, DMY2Date(1, 11, 2024));
DeprecatedDeployments.Add(Turbo0301SaasLbl, DMY2Date(1, 11, 2024));
christian-andersen-msft marked this conversation as resolved.
Show resolved Hide resolved
DeprecatedDeployments.Add(GPT40613SaasLbl, DMY2Date(1, 11, 2024));
DeprecatedDeployments.Add(Turbo0613SaasLbl, DMY2Date(1, 11, 2024));
#endif
DeprecatedDeployments.Add(GPT4oLatestLbl, DMY2Date(13, 5, 2025));
DeprecatedDeployments.Add(GPT4oPreviewLbl, DMY2Date(13, 5, 2025));
DeprecatedDeployments.Add(GPT4oMiniLatestLbl, DMY2Date(18, 7, 2025));
DeprecatedDeployments.Add(GPT4oMiniPreviewLbl, DMY2Date(18, 7, 2025));

DeprecationDatesInitialized := true;
end;

// Application Insights telemetry on deprecated models
local procedure LogDeprecationTelemetry(DeploymentName: Text)
christian-andersen-msft marked this conversation as resolved.
Show resolved Hide resolved
var
CustomDimensions: Dictionary of [Text, Text];
IsDeprecated: Boolean;
DeprecatedDate: Date;
begin
InitializeDeploymentDeprecationDates();
IsDeprecated := DeprecatedDeployments.ContainsKey(DeploymentName);
if IsDeprecated then begin
DeprecatedDate := DeprecatedDeployments.Get(DeploymentName);
CustomDimensions.Add('DeploymentName', DeploymentName);
CustomDimensions.Add('DeprecationDate', Format(DeprecatedDate));
Telemetry.LogMessage('0000AD1',
StrSubstNo(DeprecationMessageLbl, DeploymentName, DeprecatedDate),
Verbosity::Warning,
DataClassification::SystemMetadata,
Enum::"AL Telemetry Scope"::All,
CustomDimensions);
end;
end;

local procedure GetDeploymentName(DeploymentName: Text; CallerModuleInfo: ModuleInfo): Text
var
AzureOpenAiImpl: Codeunit "Azure OpenAI Impl";
CurrentModuleInfo: ModuleInfo;
begin
LogDeprecationTelemetry(DeploymentName);

NavApp.GetCurrentModuleInfo(CurrentModuleInfo);

if (CallerModuleInfo.Publisher <> CurrentModuleInfo.Publisher) and not AzureOpenAiImpl.IsTenantAllowlistedForFirstPartyCopilotCalls() then
Error(UnableToGetDeploymentNameErr);

Expand Down
Loading