LLM PR reviewer

.azure-pipelines/ultimate-pipeline.yml

@@ -6982,3 +6982,31 @@ stages:
         -d '{"commit": "'$sha'"}' \
           $(slackWebhookUrl)
       displayName: "notify"
+
+- stage: generate_LLM_Report
+  dependsOn: [merge_commit_id]
+  condition: eq(variables['generate_llm_report'], 'true')
+  variables:
+    targetShaId: $[ stageDependencies.merge_commit_id.fetch.outputs['set_sha.sha']]
+    targetBranch: $[ stageDependencies.merge_commit_id.fetch.outputs['set_sha.branch']]
+  jobs:
+  - job: generate_LLM_job
+    timeoutInMinutes: 3
+    dependsOn: []
+    pool:
+      name: azure-windows-scale-set-3
+
+    steps:
+    - template: steps/clone-repo.yml
+      parameters:
+        targetShaId: $(targetShaId)
+        targetBranch: $(targetBranch)
+    - template: steps/install-latest-dotnet-sdk.yml
+
+    - powershell: tracer/build.ps1 LLMPRReview
+      displayName: Generate LLM report
+      name: generate_llm_step
+      env:
+        PullRequestNumber: $(System.PullRequest.PullRequestNumber)
+        GITHUB_TOKEN: $(GITHUB_TOKEN)
+        OPEN_AI_KEY: $(OPEN_AI_KEY)

tracer/build/_build/Build.GitHub.cs

@@ -34,6 +34,8 @@
 using Milestone = Octokit.Milestone;
 using Release = Octokit.Release;
 using Logger = Serilog.Log;
+using Nuke.Common.Utilities;
+using OpenAI;
 
 partial class Build
 {
@@ -43,6 +45,9 @@ partial class Build
     [Parameter("Git repository name", Name = "GITHUB_REPOSITORY_NAME", List = false)]
     readonly string GitHubRepositoryName = "dd-trace-dotnet";
 
+    [Parameter("An OpenAI key", Name = "OPEN_AI_KEY")]
+    readonly string OpenAIKey;
+
     [Parameter("An Azure Devops PAT (for use in GitHub Actions)", Name = "AZURE_DEVOPS_TOKEN")]
     readonly string AzureDevopsToken;
 
@@ -94,6 +99,71 @@ await client.Issue.Update(
             Console.WriteLine($"PR assigned");
         });
 
+    Target LLMPRReview => _ => _
+        .Unlisted()
+        .Requires(() => GitHubRepositoryName)
+        .Requires(() => GitHubToken)
+        .Requires(() => OpenAIKey)
+        .Requires(() => PullRequestNumber)
+        .Executes(async () =>
+        {
+            var executeLocal = IsLocalBuild;
+            var excludePath = new string[] { "Datadog.Trace/Generated", ".g.cs" };
+            var extensionsToReview = new[] { ".csproj", ".cs", ".yml", ".h", ".cpp", ".dockerfile" };
+            var prompt = "Review this pull request with a focus on improving performance and bug detection. Make a list of the most important areas that need enhancement. For each suggestion, name the involved file and include both the original code and your recommended change, adding the corrected code if applicable. The code to be reviewed is the result of running the \"git --diff\" command. Highlight any performance bottlenecks and opportunities for optimization." + Environment.NewLine;
+
+            string result = string.Empty;
+            var client = GetGitHubClient();
+
+            var pullRequest = await client.PullRequest.Get(GitHubRepositoryOwner, GitHubRepositoryName, PullRequestNumber.Value);
+            var pullRequestFiles = await client.PullRequest.Files(GitHubRepositoryOwner, GitHubRepositoryName, PullRequestNumber.Value);
+
+            string changesText = string.Empty;
+            foreach (var file in pullRequestFiles)
+            {
+                if (!extensionsToReview.Any(ext => file.FileName.EndsWith(ext)) || excludePath.Any(x => file.FileName.Contains(x, StringComparison.OrdinalIgnoreCase)))
+                {
+                    continue;
+                }
+
+                changesText += ($"Filename: {file.FileName}" + Environment.NewLine + ($"{file.Patch}") + Environment.NewLine + Environment.NewLine);
+            }
+
+            if (string.IsNullOrEmpty(changesText))
+            {
+                result = "No changes detected.";
+            }
+            else
+            {
+                var fullPrompt = prompt + changesText;
+
+                result = OpenAiApiCall.TryGetReponse(ref fullPrompt, OpenAIKey);
+
+                if (executeLocal)
+                {
+                    Console.Write(fullPrompt);
+                }
+            }
+
+            if (string.IsNullOrEmpty(result))
+            {
+                Console.WriteLine("Error in OpenAI's response");
+                result = "Error in OpenAI's response";
+            }
+
+            var llmReport = new StringBuilder();
+            llmReport.AppendLine("## LLM Report").AppendLine(result).AppendLine();
+
+            if (executeLocal)
+            {
+                Console.WriteLine(llmReport.ToString());
+            }
+            else
+            {
+                await ReplaceCommentInPullRequest(PullRequestNumber.Value, "## LLM Report", llmReport.ToString());
+            }
+        });
+
     Target SummaryOfSnapshotChanges => _ => _
            .Unlisted()
            .Requires(() => GitHubRepositoryName)

tracer/build/_build/OpenAI/OpenAIAPICall.cs

@@ -0,0 +1,94 @@
+using System;
+using System.IO;
+using System.Net;
+using System.Net.Http;
+using System.Text;
+using System.Text.Json;
+using System.Threading.Tasks;
+
+namespace OpenAI;
+
+public class OpenAiApiCall
+{
+    private static readonly HttpClient httpClient = new HttpClient() { Timeout = TimeSpan.FromSeconds(30) };
+    //That is the current maximum number of tokens that can be requested in a single call.
+    private static readonly int maxTokens = 16383;
+
+    private static int GetApproxTokenCount(string prompt)
+    {
+        // This is a rough estimate of the number of tokens in the prompt.
+        // More info in https://help.openai.com/en/articles/4936856-what-are-tokens-and-how-to-count-them
+        return prompt.Length / 4;
+    }
+
+    // We try the full promt first, if it fails we try a smaller prompt
+    public static string TryGetReponse(ref string prompt, string key)
+    {
+        // We can do a first request without counting tokens
+        var result = GetResponse(prompt, key).Result;
+
+        // If result is null, we probably are facing a situation where we have too many tokens in the prompt
+        // In that case, we just truncate the request instead of doing multiple queries (which would increase the cost)
+        if (string.IsNullOrEmpty(result) && GetApproxTokenCount(prompt) > maxTokens)
+        {
+            Console.WriteLine("Warning: Prompt too long, trying a smaller prompt");
+            prompt = prompt.Substring(0, (int)(maxTokens * 0.95));
+            result = GetResponse(prompt, key).Result;
+        }
+
+        return result;
+    }
+
+    public static async Task<string> GetResponse(string prompt, string key)
+    {
+        var url = "https://api.openai.com/v1/chat/completions";
+
+        var requestContent = new
+        {
+            model = "gpt-4o",
+            messages = new[] 
+            {
+                new { role = "system", content = "You are a helpful assistant." },
+                new { role = "user", content = prompt }
+            },
+            max_tokens = maxTokens
+        };
+
+        var jsonContent = JsonSerializer.Serialize(requestContent);
+        var httpContent = new StringContent(jsonContent, Encoding.UTF8, "application/json");
+
+        using (var request = new HttpRequestMessage(HttpMethod.Post, url))
+        {
+            request.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Bearer", key);
+            request.Content = httpContent;
+
+            try
+            {
+                using var response = await httpClient.SendAsync(request);
+                response.EnsureSuccessStatusCode();
+                return ParseResponseText(await response.Content.ReadAsStringAsync());
+            }
+            catch (HttpRequestException e)
+            {
+                Console.WriteLine($"Request error: {e.Message}");
+                return string.Empty;
+            }
+        }
+    }
+
+    private static string ParseResponseText(string response)
+    {
+        try
+        {
+            using var jsonDoc = JsonDocument.Parse(response);
+            var root = jsonDoc.RootElement;
+            var text = root.GetProperty("choices")[0].GetProperty("message").GetProperty("content").GetString();
+            return text;
+        }
+        catch (JsonException e)
+        {
+            Console.WriteLine($"JSON parsing error: {e.Message}");
+            return string.Empty;
+        }
+    }
+}