Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Tracing] Update telemetry metrics for OpenTelemetry integration #6454

Merged
merged 3 commits into from
Jan 10, 2025
Merged

[Tracing] Update telemetry metrics for OpenTelemetry integration #6454

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
e4f92d8
Add test to ensure that we are collecting spans_created metrics and a…
zacharycmontoya Dec 17, 2024
68d194e
For the OpenTelemetry integration, update the tag generated for the s…
zacharycmontoya Dec 18, 2024
716ae47
Add MetricsTelemetryCollector instance to test cases in TelemetryHelp…
zacharycmontoya Jan 9, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions ...enerators/TelemetryMetricGenerator/CiVisibilityMetricsTelemetryCollector_CountShared.g.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -176,9 +176,9 @@ private static AggregatedMetric[] GetCountSharedBuffer()
new(new[] { "integration_name:symmetricalgorithm", "error_type:duck_typing" }),
new(new[] { "integration_name:symmetricalgorithm", "error_type:invoker" }),
new(new[] { "integration_name:symmetricalgorithm", "error_type:execution" }),
new(new[] { "integration_name:opentelemetry", "error_type:duck_typing" }),
new(new[] { "integration_name:opentelemetry", "error_type:invoker" }),
new(new[] { "integration_name:opentelemetry", "error_type:execution" }),
new(new[] { "integration_name:otel", "error_type:duck_typing" }),
new(new[] { "integration_name:otel", "error_type:invoker" }),
new(new[] { "integration_name:otel", "error_type:execution" }),
new(new[] { "integration_name:pathtraversal", "error_type:duck_typing" }),
new(new[] { "integration_name:pathtraversal", "error_type:invoker" }),
new(new[] { "integration_name:pathtraversal", "error_type:execution" }),
Expand Down
4 changes: 2 additions & 2 deletions ...adog.Trace.SourceGenerators/TelemetryMetricGenerator/MetricsTelemetryCollector_Count.g.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ private static AggregatedMetric[] GetCountBuffer()
new(new[] { "integration_name:process" }),
new(new[] { "integration_name:hashalgorithm" }),
new(new[] { "integration_name:symmetricalgorithm" }),
new(new[] { "integration_name:opentelemetry" }),
new(new[] { "integration_name:otel" }),
new(new[] { "integration_name:pathtraversal" }),
new(new[] { "integration_name:ssrf" }),
new(new[] { "integration_name:ldap" }),
Expand Down Expand Up @@ -498,7 +498,7 @@ private static AggregatedMetric[] GetCountBuffer()
new(new[] { "integration_name:process" }),
new(new[] { "integration_name:hashalgorithm" }),
new(new[] { "integration_name:symmetricalgorithm" }),
new(new[] { "integration_name:opentelemetry" }),
new(new[] { "integration_name:otel" }),
new(new[] { "integration_name:pathtraversal" }),
new(new[] { "integration_name:ssrf" }),
new(new[] { "integration_name:ldap" }),
Expand Down
6 changes: 3 additions & 3 deletions ...race.SourceGenerators/TelemetryMetricGenerator/MetricsTelemetryCollector_CountShared.g.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -176,9 +176,9 @@ private static AggregatedMetric[] GetCountSharedBuffer()
new(new[] { "integration_name:symmetricalgorithm", "error_type:duck_typing" }),
new(new[] { "integration_name:symmetricalgorithm", "error_type:invoker" }),
new(new[] { "integration_name:symmetricalgorithm", "error_type:execution" }),
new(new[] { "integration_name:opentelemetry", "error_type:duck_typing" }),
new(new[] { "integration_name:opentelemetry", "error_type:invoker" }),
new(new[] { "integration_name:opentelemetry", "error_type:execution" }),
new(new[] { "integration_name:otel", "error_type:duck_typing" }),
new(new[] { "integration_name:otel", "error_type:invoker" }),
new(new[] { "integration_name:otel", "error_type:execution" }),
new(new[] { "integration_name:pathtraversal", "error_type:duck_typing" }),
new(new[] { "integration_name:pathtraversal", "error_type:invoker" }),
new(new[] { "integration_name:pathtraversal", "error_type:execution" }),
Expand Down
6 changes: 3 additions & 3 deletions ...enerators/TelemetryMetricGenerator/CiVisibilityMetricsTelemetryCollector_CountShared.g.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -176,9 +176,9 @@ private static AggregatedMetric[] GetCountSharedBuffer()
new(new[] { "integration_name:symmetricalgorithm", "error_type:duck_typing" }),
new(new[] { "integration_name:symmetricalgorithm", "error_type:invoker" }),
new(new[] { "integration_name:symmetricalgorithm", "error_type:execution" }),
new(new[] { "integration_name:opentelemetry", "error_type:duck_typing" }),
new(new[] { "integration_name:opentelemetry", "error_type:invoker" }),
new(new[] { "integration_name:opentelemetry", "error_type:execution" }),
new(new[] { "integration_name:otel", "error_type:duck_typing" }),
new(new[] { "integration_name:otel", "error_type:invoker" }),
new(new[] { "integration_name:otel", "error_type:execution" }),
new(new[] { "integration_name:pathtraversal", "error_type:duck_typing" }),
new(new[] { "integration_name:pathtraversal", "error_type:invoker" }),
new(new[] { "integration_name:pathtraversal", "error_type:execution" }),
Expand Down
4 changes: 2 additions & 2 deletions ...adog.Trace.SourceGenerators/TelemetryMetricGenerator/MetricsTelemetryCollector_Count.g.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ private static AggregatedMetric[] GetCountBuffer()
new(new[] { "integration_name:process" }),
new(new[] { "integration_name:hashalgorithm" }),
new(new[] { "integration_name:symmetricalgorithm" }),
new(new[] { "integration_name:opentelemetry" }),
new(new[] { "integration_name:otel" }),
new(new[] { "integration_name:pathtraversal" }),
new(new[] { "integration_name:ssrf" }),
new(new[] { "integration_name:ldap" }),
Expand Down Expand Up @@ -498,7 +498,7 @@ private static AggregatedMetric[] GetCountBuffer()
new(new[] { "integration_name:process" }),
new(new[] { "integration_name:hashalgorithm" }),
new(new[] { "integration_name:symmetricalgorithm" }),
new(new[] { "integration_name:opentelemetry" }),
new(new[] { "integration_name:otel" }),
new(new[] { "integration_name:pathtraversal" }),
new(new[] { "integration_name:ssrf" }),
new(new[] { "integration_name:ldap" }),
Expand Down
6 changes: 3 additions & 3 deletions ...race.SourceGenerators/TelemetryMetricGenerator/MetricsTelemetryCollector_CountShared.g.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -176,9 +176,9 @@ private static AggregatedMetric[] GetCountSharedBuffer()
new(new[] { "integration_name:symmetricalgorithm", "error_type:duck_typing" }),
new(new[] { "integration_name:symmetricalgorithm", "error_type:invoker" }),
new(new[] { "integration_name:symmetricalgorithm", "error_type:execution" }),
new(new[] { "integration_name:opentelemetry", "error_type:duck_typing" }),
new(new[] { "integration_name:opentelemetry", "error_type:invoker" }),
new(new[] { "integration_name:opentelemetry", "error_type:execution" }),
new(new[] { "integration_name:otel", "error_type:duck_typing" }),
new(new[] { "integration_name:otel", "error_type:invoker" }),
new(new[] { "integration_name:otel", "error_type:execution" }),
new(new[] { "integration_name:pathtraversal", "error_type:duck_typing" }),
new(new[] { "integration_name:pathtraversal", "error_type:invoker" }),
new(new[] { "integration_name:pathtraversal", "error_type:execution" }),
Expand Down
6 changes: 3 additions & 3 deletions ...enerators/TelemetryMetricGenerator/CiVisibilityMetricsTelemetryCollector_CountShared.g.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -176,9 +176,9 @@ private static AggregatedMetric[] GetCountSharedBuffer()
new(new[] { "integration_name:symmetricalgorithm", "error_type:duck_typing" }),
new(new[] { "integration_name:symmetricalgorithm", "error_type:invoker" }),
new(new[] { "integration_name:symmetricalgorithm", "error_type:execution" }),
new(new[] { "integration_name:opentelemetry", "error_type:duck_typing" }),
new(new[] { "integration_name:opentelemetry", "error_type:invoker" }),
new(new[] { "integration_name:opentelemetry", "error_type:execution" }),
new(new[] { "integration_name:otel", "error_type:duck_typing" }),
new(new[] { "integration_name:otel", "error_type:invoker" }),
new(new[] { "integration_name:otel", "error_type:execution" }),
new(new[] { "integration_name:pathtraversal", "error_type:duck_typing" }),
new(new[] { "integration_name:pathtraversal", "error_type:invoker" }),
new(new[] { "integration_name:pathtraversal", "error_type:execution" }),
Expand Down
4 changes: 2 additions & 2 deletions ...adog.Trace.SourceGenerators/TelemetryMetricGenerator/MetricsTelemetryCollector_Count.g.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ private static AggregatedMetric[] GetCountBuffer()
new(new[] { "integration_name:process" }),
new(new[] { "integration_name:hashalgorithm" }),
new(new[] { "integration_name:symmetricalgorithm" }),
new(new[] { "integration_name:opentelemetry" }),
new(new[] { "integration_name:otel" }),
new(new[] { "integration_name:pathtraversal" }),
new(new[] { "integration_name:ssrf" }),
new(new[] { "integration_name:ldap" }),
Expand Down Expand Up @@ -498,7 +498,7 @@ private static AggregatedMetric[] GetCountBuffer()
new(new[] { "integration_name:process" }),
new(new[] { "integration_name:hashalgorithm" }),
new(new[] { "integration_name:symmetricalgorithm" }),
new(new[] { "integration_name:opentelemetry" }),
new(new[] { "integration_name:otel" }),
new(new[] { "integration_name:pathtraversal" }),
new(new[] { "integration_name:ssrf" }),
new(new[] { "integration_name:ldap" }),
Expand Down
6 changes: 3 additions & 3 deletions ...race.SourceGenerators/TelemetryMetricGenerator/MetricsTelemetryCollector_CountShared.g.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -176,9 +176,9 @@ private static AggregatedMetric[] GetCountSharedBuffer()
new(new[] { "integration_name:symmetricalgorithm", "error_type:duck_typing" }),
new(new[] { "integration_name:symmetricalgorithm", "error_type:invoker" }),
new(new[] { "integration_name:symmetricalgorithm", "error_type:execution" }),
new(new[] { "integration_name:opentelemetry", "error_type:duck_typing" }),
new(new[] { "integration_name:opentelemetry", "error_type:invoker" }),
new(new[] { "integration_name:opentelemetry", "error_type:execution" }),
new(new[] { "integration_name:otel", "error_type:duck_typing" }),
new(new[] { "integration_name:otel", "error_type:invoker" }),
new(new[] { "integration_name:otel", "error_type:execution" }),
new(new[] { "integration_name:pathtraversal", "error_type:duck_typing" }),
new(new[] { "integration_name:pathtraversal", "error_type:invoker" }),
new(new[] { "integration_name:pathtraversal", "error_type:execution" }),
Expand Down
6 changes: 3 additions & 3 deletions ...enerators/TelemetryMetricGenerator/CiVisibilityMetricsTelemetryCollector_CountShared.g.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -176,9 +176,9 @@ private static AggregatedMetric[] GetCountSharedBuffer()
new(new[] { "integration_name:symmetricalgorithm", "error_type:duck_typing" }),
new(new[] { "integration_name:symmetricalgorithm", "error_type:invoker" }),
new(new[] { "integration_name:symmetricalgorithm", "error_type:execution" }),
new(new[] { "integration_name:opentelemetry", "error_type:duck_typing" }),
new(new[] { "integration_name:opentelemetry", "error_type:invoker" }),
new(new[] { "integration_name:opentelemetry", "error_type:execution" }),
new(new[] { "integration_name:otel", "error_type:duck_typing" }),
new(new[] { "integration_name:otel", "error_type:invoker" }),
new(new[] { "integration_name:otel", "error_type:execution" }),
new(new[] { "integration_name:pathtraversal", "error_type:duck_typing" }),
new(new[] { "integration_name:pathtraversal", "error_type:invoker" }),
new(new[] { "integration_name:pathtraversal", "error_type:execution" }),
Expand Down
4 changes: 2 additions & 2 deletions ...adog.Trace.SourceGenerators/TelemetryMetricGenerator/MetricsTelemetryCollector_Count.g.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ private static AggregatedMetric[] GetCountBuffer()
new(new[] { "integration_name:process" }),
new(new[] { "integration_name:hashalgorithm" }),
new(new[] { "integration_name:symmetricalgorithm" }),
new(new[] { "integration_name:opentelemetry" }),
new(new[] { "integration_name:otel" }),
new(new[] { "integration_name:pathtraversal" }),
new(new[] { "integration_name:ssrf" }),
new(new[] { "integration_name:ldap" }),
Expand Down Expand Up @@ -498,7 +498,7 @@ private static AggregatedMetric[] GetCountBuffer()
new(new[] { "integration_name:process" }),
new(new[] { "integration_name:hashalgorithm" }),
new(new[] { "integration_name:symmetricalgorithm" }),
new(new[] { "integration_name:opentelemetry" }),
new(new[] { "integration_name:otel" }),
new(new[] { "integration_name:pathtraversal" }),
new(new[] { "integration_name:ssrf" }),
new(new[] { "integration_name:ldap" }),
Expand Down
6 changes: 3 additions & 3 deletions ...race.SourceGenerators/TelemetryMetricGenerator/MetricsTelemetryCollector_CountShared.g.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -176,9 +176,9 @@ private static AggregatedMetric[] GetCountSharedBuffer()
new(new[] { "integration_name:symmetricalgorithm", "error_type:duck_typing" }),
new(new[] { "integration_name:symmetricalgorithm", "error_type:invoker" }),
new(new[] { "integration_name:symmetricalgorithm", "error_type:execution" }),
new(new[] { "integration_name:opentelemetry", "error_type:duck_typing" }),
new(new[] { "integration_name:opentelemetry", "error_type:invoker" }),
new(new[] { "integration_name:opentelemetry", "error_type:execution" }),
new(new[] { "integration_name:otel", "error_type:duck_typing" }),
new(new[] { "integration_name:otel", "error_type:invoker" }),
new(new[] { "integration_name:otel", "error_type:execution" }),
new(new[] { "integration_name:pathtraversal", "error_type:duck_typing" }),
new(new[] { "integration_name:pathtraversal", "error_type:invoker" }),
new(new[] { "integration_name:pathtraversal", "error_type:execution" }),
Expand Down
2 changes: 1 addition & 1 deletion tracer/src/Datadog.Trace/Telemetry/Metrics/MetricTags.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -239,7 +239,7 @@ internal enum IntegrationName
[Description("integration_name:process")]Process,
[Description("integration_name:hashalgorithm")]HashAlgorithm,
[Description("integration_name:symmetricalgorithm")]SymmetricAlgorithm,
[Description("integration_name:opentelemetry")]OpenTelemetry,
[Description("integration_name:otel")]OpenTelemetry, // Note: The naming of this tag value breaks the convention of using the integration name to use a standardized value
[Description("integration_name:pathtraversal")]PathTraversal,
[Description("integration_name:ssrf")]Ssrf,
[Description("integration_name:ldap")]Ldap,
Expand Down
40 changes: 38 additions & 2 deletions tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/Helpers/TelemetryHelper.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// <copyright file="TelemetryHelper.cs" company="Datadog">
// <copyright file="TelemetryHelper.cs" company="Datadog">
// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License.
// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc.
// </copyright>
Expand Down Expand Up @@ -165,9 +165,45 @@ _ when data.TryGetPayload<AppIntegrationsChangedPayload>(TelemetryRequestTypes.A
}
}

var spansCreatedByIntegration = new ConcurrentDictionary<string, MetricData>();
var metricsPayloads =
allData
.Select(
data => data switch
{
_ when data.TryGetPayload<GenerateMetricsPayload>(TelemetryRequestTypes.GenerateMetrics) is { } p => p.Series.Where(s => s.Metric == "spans_created"),
_ => null,
})
.Where(x => x is not null);

// Flatten the spans_created metrics
foreach (var metricPayload in metricsPayloads)
{
foreach (var metricEntry in metricPayload)
{
spansCreatedByIntegration.TryAdd(metricEntry.Tags.First(s => s.StartsWith("integration_name:")), metricEntry);
}
}

var integrationName = integrationId.ToString();
var integrationNameTagValue = integrationName switch
{
"OpenTelemetry" => "otel",
_ => integrationName.ToLowerInvariant(),
};

Comment on lines +189 to +193
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not sure if you want to do this or not, because this grabs directly from the metric tag:

Suggested change
var integrationNameTagValue = integrationName switch
{
"OpenTelemetry" => "otel",
_ => integrationName.ToLowerInvariant(),
};
var integrationNameTagValue = integrationId.GetMetricTag()

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I was thinking of not doing this, so that we could write the assertion without invoking the exact product code. So this "otel" or convert from integration name seemed like a better test case

if (enabled)
{
spansCreatedByIntegration.Should().NotBeEmpty();

var spansCreated = spansCreatedByIntegration.Should().ContainKey($"integration_name:{integrationNameTagValue}").WhoseValue;
spansCreated.Points.Should().NotBeEmpty();
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This adds a new check (for all our integration tests!) that when the integration is enabled we also receive a spans_created metric that corresponds to the enabled integration.

The "otel" check above shows how this is the only integration whose tag value differs from its corresponding IntegrationId.ToString() 😅

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I have a mild concern that this will overall increase flakiness in the tests. Telemetry is one of the areas most likely to flake, because it is typically sent at shutdown in our small samples, and that's where there are known bugs in the runtime etc.

If we do see flake around this, I'd suggest that most of this check is (or could be) checked with unit tests already 🙂

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Acknowledged, we can keep a close eye on it post-merge 👍🏼

spansCreated.Points.Sum(p => p.Value).Should().BeGreaterThanOrEqualTo(1);
}

latestIntegrations.Should().NotBeEmpty();

var integration = latestIntegrations.Should().ContainKey(integrationId.ToString()).WhoseValue;
var integration = latestIntegrations.Should().ContainKey(integrationName).WhoseValue;

integration.Enabled.Should().Be(enabled, $"{integration.Name} should only be enabled if we generate a span");
if (autoEnabled.HasValue)
Expand Down
22 changes: 17 additions & 5 deletions tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/Helpers/TelemetryHelperTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
using Datadog.Trace.Configuration.ConfigurationSources.Telemetry;
using Datadog.Trace.Configuration.Telemetry;
using Datadog.Trace.Telemetry;
using Datadog.Trace.Telemetry.Metrics;
using Datadog.Trace.TestHelpers;
using FluentAssertions;
using FluentAssertions.Execution;
Expand Down Expand Up @@ -48,25 +49,35 @@ public TelemetryHelperTests(ITestOutputHelper output)
public void AssertIntegration_HandlesMultipleTelemetryPushes()
{
var collector = new IntegrationTelemetryCollector();
var metricsCollector = new MetricsTelemetryCollector();
var telemetryData = new List<TelemetryData>();

collector.IntegrationRunning(IntegrationId.Aerospike);

telemetryData.Add(BuildTelemetryData(collector.GetData()));
metricsCollector.AggregateMetrics();
telemetryData.Add(BuildTelemetryData(collector.GetData(), metrics: metricsCollector.GetMetrics()));

// The updates to both the IntegrationTelemetryCollector and the MetricsTelemetryCollector
// are typically handled by TelemetryController.IntegrationGeneratedSpan(IntegrationId),
// so we simulate that here with the separate calls
collector.IntegrationGeneratedSpan(IntegrationId.Aerospike);
metricsCollector.RecordCountSpanCreated(IntegrationId.Aerospike.GetMetricTag());

collector.IntegrationRunning(IntegrationId.Couchbase);
telemetryData.Add(BuildTelemetryData(collector.GetData(), sendAppStarted: false));

metricsCollector.AggregateMetrics();
telemetryData.Add(BuildTelemetryData(collector.GetData(), metrics: metricsCollector.GetMetrics(), sendAppStarted: false));

collector.IntegrationRunning(IntegrationId.Kafka);
collector.IntegrationGeneratedSpan(IntegrationId.Msmq);
collector.IntegrationRunning(IntegrationId.Msmq);
var tracerSettings = TracerSettings.Create(new()
{
{ ConfigurationKeys.DisabledIntegrations, $"{nameof(IntegrationId.Kafka)};{nameof(IntegrationId.Msmq)}" }
});

collector.RecordTracerSettings(tracerSettings);
telemetryData.Add(BuildTelemetryData(collector.GetData(), sendAppClosing: true));
metricsCollector.AggregateMetrics();
telemetryData.Add(BuildTelemetryData(collector.GetData(), metrics: metricsCollector.GetMetrics(), sendAppClosing: true));

using var s = new AssertionScope();
TelemetryHelper.AssertIntegration(telemetryData, IntegrationId.Aerospike, enabled: true, autoEnabled: true);
Expand Down Expand Up @@ -150,12 +161,13 @@ public void AssertConfiguration_HandlesMultipleTelemetryPushes()
private TelemetryData BuildTelemetryData(
ICollection<IntegrationTelemetryData> integrations,
ICollection<ConfigurationKeyValue> configuration = null,
MetricResults? metrics = null,
bool sendAppStarted = true,
bool sendAppClosing = false)
=> _dataBuilder.BuildTelemetryData(
_app,
_host,
new TelemetryInput(configuration, null, integrations, null, null, sendAppStarted),
new TelemetryInput(configuration, null, integrations, metrics, null, sendAppStarted),
namingSchemeVersion: "1",
sendAppClosing);
}
Loading