Include extra tags in error logs

tracer/src/Datadog.Trace/Ci/CITracerManagerFactory.cs

@@ -56,6 +56,7 @@ protected override TracerManager CreateTracerManagerFrom(
             IDynamicConfigurationManager dynamicConfigurationManager,
             ITracerFlareManager tracerFlareManager)
         {
+            telemetry.RecordCiVisibilitySettings(_settings);
             if (_useLockedManager)
             {
                 return new CITracerManager.LockedManager(settings, agentWriter, scopeManager, statsd, runtimeMetrics, logSubmissionManager, telemetry, discoveryService, dataStreamsManager, defaultServiceName, gitMetadataTagsProvider, traceSampler, spanSampler, remoteConfigurationManager, dynamicConfigurationManager, tracerFlareManager);

tracer/src/Datadog.Trace/Telemetry/Collectors/RedactedErrorLogCollector.cs

@@ -32,7 +32,7 @@ internal class RedactedErrorLogCollector
     private ConcurrentDictionary<uint, int> _logCounts = new();
     private ConcurrentDictionary<uint, int> _logCountsReserve = new();
 
-    public List<List<LogMessageData>>? GetLogs()
+    public List<List<LogMessageData>>? GetLogs(string? tags = null)
     {
         // This method should only be called in a single-threaded loop
         List<List<LogMessageData>>? batches = null;
@@ -42,6 +42,7 @@ internal class RedactedErrorLogCollector
 
         while (_queue.TryDequeue(out var log))
         {
+            log.Tags = tags;
             var logSize = log.GetApproximateSerializationSize();
             // modify the message to add the final log count
             var eventId = EventIdHash.Compute(log.Message, log.StackTrace);

tracer/src/Datadog.Trace/Telemetry/ITelemetryController.cs

@@ -6,6 +6,7 @@
 using System;
 using System.Threading.Tasks;
 using Datadog.Trace.AppSec;
+using Datadog.Trace.Ci.Configuration;
 using Datadog.Trace.Configuration;
 using Datadog.Trace.ContinuousProfiler;
 using Datadog.Trace.Iast.Settings;
@@ -40,6 +41,11 @@ internal interface ITelemetryController
         /// </summary>
         public void RecordProfilerSettings(Profiler profiler);
 
+        /// <summary>
+        /// Called to record ci-vis-related telemetry
+        /// </summary>
+        public void RecordCiVisibilitySettings(CIVisibilitySettings settings);
+
         /// <summary>
         /// Dispose resources for sending telemetry
         /// </summary>

tracer/src/Datadog.Trace/Telemetry/NullTelemetryController.cs

@@ -4,6 +4,7 @@
 // </copyright>
 
 using System.Threading.Tasks;
+using Datadog.Trace.Ci.Configuration;
 using Datadog.Trace.Configuration;
 using Datadog.Trace.ContinuousProfiler;
 
@@ -33,6 +34,10 @@ public void RecordProfilerSettings(Profiler profiler)
         {
         }
 
+        public void RecordCiVisibilitySettings(CIVisibilitySettings settings)
+        {
+        }
+
         public void Start()
         {
         }

tracer/src/Datadog.Trace/Telemetry/TelemetryController.cs

@@ -11,6 +11,8 @@
 using System.Reflection;
 using System.Threading;
 using System.Threading.Tasks;
+using Datadog.Trace.Ci;
+using Datadog.Trace.Ci.Configuration;
 using Datadog.Trace.Configuration;
 using Datadog.Trace.Configuration.Telemetry;
 using Datadog.Trace.ContinuousProfiler;
@@ -38,6 +40,7 @@ internal class TelemetryController : ITelemetryController
     private readonly IMetricsTelemetryCollector _metrics;
     private readonly RedactedErrorLogCollector? _redactedErrorLogs;
     private readonly TaskCompletionSource<bool> _processExit = new();
+    private readonly TagBuilder _logTagBuilder = new();
     private readonly Task _flushTask;
     private readonly Scheduler _scheduler;
     private TelemetryTransportManager _transportManager;
@@ -93,6 +96,7 @@ public void RecordTracerSettings(TracerSettings settings, string defaultServiceN
         settings.Telemetry.CopyTo(_configuration);
         _application.RecordTracerSettings(settings, defaultServiceName);
         _namingVersion = ((int)settings.MetadataSchemaVersion).ToString();
+        _logTagBuilder.Update(settings);
         _queue.Enqueue(new WorkItem(WorkItem.ItemType.EnableSending, null));
     }
 
@@ -111,14 +115,23 @@ public void Start()
     }
 
     public void ProductChanged(TelemetryProductType product, bool enabled, ErrorData? error)
-        => _products.ProductChanged(product, enabled, error);
+    {
+        _products.ProductChanged(product, enabled, error);
+        _logTagBuilder.Update(product, enabled);
+    }
 
     public void RecordProfilerSettings(Profiler profiler)
     {
         _configuration.Record(ConfigTelemetryData.ProfilerLoaded, profiler.Status.IsProfilerReady, ConfigurationOrigins.Default);
         _configuration.Record(ConfigTelemetryData.CodeHotspotsEnabled, profiler.ContextTracker.IsEnabled, ConfigurationOrigins.Default);
     }
 
+    public void RecordCiVisibilitySettings(CIVisibilitySettings settings)
+    {
+        // CI Vis records the settings _directly_ in the global config so don't need to record them again here
+        _logTagBuilder.Update(settings, CIVisibility.Enabled);
+    }
+
     public void IntegrationRunning(IntegrationId integrationId)
         => _integrations.IntegrationRunning(integrationId);
 
@@ -298,7 +311,7 @@ private async Task PushTelemetry(bool includeLogs, bool sendAppClosing)
                 return;
             }
 
-            if (includeLogs && _redactedErrorLogs?.GetLogs() is { } batches)
+            if (includeLogs && _redactedErrorLogs?.GetLogs(_logTagBuilder.GetLogTags()) is { } batches)
             {
                 foreach (var batch in batches)
                 {
@@ -350,6 +363,74 @@ public enum ItemType
         public object? State { get; }
     }
 
+    internal class TagBuilder
+    {
+        private bool _isCiVisEnabled;
+        private bool _isAsmEnabled;
+        private bool _isProfilingEnabled;
+        private bool _isDynamicInstrumentationEnabled;
+        private string? _cloudEnv;
+        private bool _isUpdateRequired;
+        private string? _tags;
+
+        public void Update(ImmutableTracerSettings settings)
+        {
+            _cloudEnv = settings switch
+            {
+                { IsRunningInGCPFunctions: true } => ",gcp",
+                { LambdaMetadata.IsRunningInLambda: true } => ",aws",
+                { IsRunningMiniAgentInAzureFunctions: true } => ",azf",
+                { IsRunningInAzureAppService: true } => ",aas",
+                _ => null,
+            };
+            _isUpdateRequired = true;
+        }
+
+        public void Update(CIVisibilitySettings settings, bool enabled)
+        {
+            // We don't actually need to record these, because they're added to the global config
+            // This isn't nice, as it calls the static property,
+            // but we don't have a better way of getting this info right now
+            _isCiVisEnabled = enabled;
+            _isUpdateRequired = true;
+        }
+
+        public void Update(TelemetryProductType product, bool enabled)
+        {
+            if (product == TelemetryProductType.Profiler)
+            {
+                _isProfilingEnabled = enabled;
+            }
+            else if (product == TelemetryProductType.AppSec)
+            {
+                _isAsmEnabled = enabled;
+            }
+            else if (product == TelemetryProductType.DynamicInstrumentation)
+            {
+                _isDynamicInstrumentationEnabled = enabled;
+            }
+
+            // unknown product, ignore
+            _isUpdateRequired = true;
+        }
+
+        public string GetLogTags()
+        {
+            if (_isUpdateRequired || _tags is null)
+            {
+                _isUpdateRequired = false;
+                // using 1/0 to save bytes!
+                _tags = $"ci:{(_isCiVisEnabled ? '1' : '0')}" +
+                        $",asm:{(_isAsmEnabled ? '1' : '0')}" +
+                        $",prof:{(_isProfilingEnabled ? '1' : '0')}" +
+                        $",dyn:{(_isDynamicInstrumentationEnabled ? '1' : '0')}" +
+                        $"{_cloudEnv}";
+            }
+
+            return _tags;
+        }
+    }
+
     /// <summary>
     /// Internal for testing
     /// </summary>

tracer/test/Datadog.Trace.Tests/Telemetry/CallTargetInvokerTelemetryTests.cs

@@ -9,6 +9,7 @@
 using System.Threading.Tasks;
 using Datadog.Trace.Agent;
 using Datadog.Trace.AppSec;
+using Datadog.Trace.Ci.Configuration;
 using Datadog.Trace.ClrProfiler.AutoInstrumentation.Http.HttpClient.HttpClientHandler;
 using Datadog.Trace.ClrProfiler.CallTarget;
 using Datadog.Trace.Configuration;
@@ -129,6 +130,10 @@ public void RecordProfilerSettings(Profiler profiler)
             {
             }
 
+            public void RecordCiVisibilitySettings(CIVisibilitySettings settings)
+            {
+            }
+
             public Task DisposeAsync()
             {
                 return Task.CompletedTask;

tracer/test/Datadog.Trace.Tests/Telemetry/Collectors/RedactedErrorLogCollectorTests.cs

@@ -378,6 +378,23 @@ public void WhenDeDupeEnabled_AndUniqueMessages_DoesNotSetCount()
             .OnlyContain(x => x.Count == null);
     }
 
+    [Fact]
+    public void AddsTagsToLogDuringSerialization()
+    {
+        var tagBuilder = new TelemetryController.TagBuilder();
+        var collector = new RedactedErrorLogCollector();
+        collector.EnqueueLog(new($"Some log", TelemetryLogLevel.WARN, DateTimeOffset.UtcNow));
+
+        var logTags = tagBuilder.GetLogTags();
+        var logs = collector.GetLogs(logTags);
+        logs.Should()
+            .ContainSingle()
+            .Which.Should()
+            .ContainSingle()
+            .Which.Tags.Should()
+            .Be(logTags);
+    }
+
     private static string RandomString(int length)
     {
         const string options = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789 ,!\"£$%^&*()[]{}<>`¬";

tracer/test/Datadog.Trace.Tests/Telemetry/TelemetryControllerLogTagBuilderTests.cs

@@ -0,0 +1,74 @@
+// <copyright file="TelemetryControllerLogTagBuilderTests.cs" company="Datadog">
+// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License.
+// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc.
+// </copyright>
+
+using Datadog.Trace.Ci.Configuration;
+using Datadog.Trace.Configuration;
+using Datadog.Trace.Configuration.Telemetry;
+using Datadog.Trace.Telemetry;
+using FluentAssertions;
+using Xunit;
+
+namespace Datadog.Trace.Tests.Telemetry;
+
+public class TelemetryControllerLogTagBuilderTests
+{
+    [Fact]
+    public void TagBuilder_ReturnsExpectedDefaults()
+    {
+        var builder = new TelemetryController.TagBuilder();
+        builder.GetLogTags().Should().Be("ci:0,asm:0,prof:0,dyn:0");
+    }
+
+    [Fact]
+    public void TagBuilder_UpdateCiVisTag()
+    {
+        var builder = new TelemetryController.TagBuilder();
+        builder.Update(new CIVisibilitySettings(NullConfigurationSource.Instance, NullConfigurationTelemetry.Instance), enabled: true);
+        builder.GetLogTags().Should().Be("ci:1,asm:0,prof:0,dyn:0");
+    }
+
+    [Theory]
+    [InlineData((int)TelemetryProductType.AppSec, "ci:0,asm:1,prof:0,dyn:0")]
+    [InlineData((int)TelemetryProductType.Profiler, "ci:0,asm:0,prof:1,dyn:0")]
+    [InlineData((int)TelemetryProductType.DynamicInstrumentation, "ci:0,asm:0,prof:0,dyn:1")]
+    public void TagBuilder_UpdateProductTag(int product, string expected)
+    {
+        var builder = new TelemetryController.TagBuilder();
+        builder.Update((TelemetryProductType)product, enabled: true);
+        builder.GetLogTags().Should().Be(expected);
+    }
+
+    [Fact]
+    public void TagBuilder_UpdateCloudTag()
+    {
+        var builder = new TelemetryController.TagBuilder();
+        builder.Update(TracerSettings.Create(new() { { "DD_AZURE_APP_SERVICES", "true" } }).Build());
+        builder.GetLogTags().Should().Be("ci:0,asm:0,prof:0,dyn:0,aas");
+    }
+
+    [Fact]
+    public void TagBuilder_AddsEverything()
+    {
+        var builder = new TelemetryController.TagBuilder();
+        builder.Update(TracerSettings.Create(new() { { "FUNCTIONS_EXTENSION_VERSION", "true" } }).Build());
+        builder.Update(TelemetryProductType.Profiler, enabled: true);
+        builder.Update(TelemetryProductType.DynamicInstrumentation, enabled: true);
+        builder.Update(TelemetryProductType.AppSec, enabled: true);
+        builder.Update(new CIVisibilitySettings(NullConfigurationSource.Instance, NullConfigurationTelemetry.Instance), enabled: true);
+        builder.GetLogTags().Should().Be("ci:1,asm:1,prof:1,dyn:1,azf");
+    }
+
+    [Fact]
+    public void TagBuilder_WhenAdded()
+    {
+        var builder = new TelemetryController.TagBuilder();
+        builder.Update(TracerSettings.Create(new() { { "FUNCTIONS_EXTENSION_VERSION", "true" } }).Build());
+        builder.Update(TelemetryProductType.Profiler, enabled: true);
+        builder.Update(TelemetryProductType.DynamicInstrumentation, enabled: true);
+        builder.Update(TelemetryProductType.AppSec, enabled: true);
+        builder.Update(new CIVisibilitySettings(NullConfigurationSource.Instance, NullConfigurationTelemetry.Instance), enabled: true);
+        builder.GetLogTags().Should().Be("ci:1,asm:1,prof:1,dyn:1,azf");
+    }
+}