Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Consolidate all integration test targets into a single target #6501

Open
wants to merge 1 commit into
base: andrew/ci/build-samples-separately
Choose a base branch
from
+36 −176
Open

Consolidate all integration test targets into a single target #6501

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .azure-pipelines/ultimate-pipeline.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1781,7 +1781,7 @@ stages:
displayName: 'Initialize LocalDB'
workingDirectory: $(Build.Repository.LocalPath)

- script: tracer\build.cmd CompileTrimmingSamples BuildWindowsIntegrationTests BuildWindowsRegressionTests RunWindowsIntegrationTests RunWindowsRegressionTests -Framework $(framework) --code-coverage-enabled $(CodeCoverageEnabled)
- script: tracer\build.cmd CompileTrimmingSamples BuildWindowsIntegrationTests BuildWindowsRegressionTests RunIntegrationTests RunWindowsRegressionTests -Framework $(framework) --code-coverage-enabled $(CodeCoverageEnabled)
displayName: Run integration tests
env:
DD_LOGGER_DD_API_KEY: $(ddApiKey)
Expand Down
6 changes: 3 additions & 3 deletions docker-compose.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -394,7 +394,7 @@ services:
args:
- DOTNETSDK_VERSION=${dotnetCoreSdkLatestVersion:-9.0.100}
image: dd-trace-dotnet/${baseImage:-debian}-tester:${dotnetCoreSdkLatestVersion:-9.0.100}
command: dotnet /build/bin/Debug/_build.dll RunLinuxIntegrationTests
command: dotnet /build/bin/Debug/_build.dll RunIntegrationTests
volumes:
- ./:/project
cap_add:
Expand Down Expand Up @@ -541,7 +541,7 @@ services:
args:
- DOTNETSDK_VERSION=${dotnetCoreSdkLatestVersion:-9.0.100}
image: dd-trace-dotnet/${baseImage:-debian}-tester:${dotnetCoreSdkLatestVersion:-9.0.100}
command: dotnet /build/bin/Debug/_build.dll RunLinuxIntegrationTests
command: dotnet /build/bin/Debug/_build.dll RunIntegrationTests
volumes:
- ./:/project
cap_add:
Expand Down Expand Up @@ -672,7 +672,7 @@ services:
args:
- DOTNETSDK_VERSION=${dotnetCoreSdkLatestVersion:-9.0.100}
image: dd-trace-dotnet/${baseImage:-debian}-tester:${dotnetCoreSdkLatestVersion:-9.0.100}
command: dotnet /build/bin/Debug/_build.dll RunLinuxIntegrationTests
command: dotnet /build/bin/Debug/_build.dll RunIntegrationTests
volumes:
- ./:/project
cap_add:
Expand Down
198 changes: 29 additions & 169 deletions tracer/build/_build/Build.Steps.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1518,20 +1518,21 @@ void CompileSamplesThatDependOnDatadogTrace()
);
});

Target RunWindowsIntegrationTests => _ => _
Target RunIntegrationTests => _ => _
.Unlisted()
.After(BuildTracerHome)
.After(CompileIntegrationTests)
.After(CompileSamples)
.After(CompileTrimmingSamples)
.After(BuildWindowsIntegrationTests)
.After(CompileLinuxOrOsxIntegrationTests)
.DependsOn(CleanTestLogs)
.Requires(() => IsWin)
.Requires(() => Framework)
.Triggers(PrintSnapshotsDiff)
.Executes(() =>
{
var isDebugRun = IsDebugRun();
var filter = GetFilter();

try
{
Expand All @@ -1548,6 +1549,7 @@ void CompileSamplesThatDependOnDatadogTrace()
.SetIsDebugRun(isDebugRun)
.SetProcessEnvironmentVariable("MonitoringHomeDirectory", MonitoringHomeDirectory)
.SetLogsDirectory(TestLogsDirectory)
// Don't apply a custom filter to these tests, they should all be able to be run
.When(!string.IsNullOrWhiteSpace(Filter), c => c.SetFilter(Filter))
.When(TestAllPackageVersions, o => o.SetProcessEnvironmentVariable("TestAllPackageVersions", "true"))
.When(CodeCoverageEnabled, ConfigureCodeCoverage)
Expand All @@ -1556,22 +1558,20 @@ void CompileSamplesThatDependOnDatadogTrace()
.WithDatadogLogger()
.SetProjectFile(project)), degreeOfParallelism: 4);


// TODO: I think we should change this filter to run on Windows by default
// (RunOnWindows!=False|Category=Smoke)&LoadFromGAC!=True&IIS!=True
DotNetTest(config => config
.SetDotnetPath(TargetPlatform)
.SetConfiguration(BuildConfiguration)
.SetTargetPlatformAnyCPU()
.SetFramework(Framework)
//.WithMemoryDumpAfter(timeoutInMinutes: 30)
.EnableCrashDumps()
.EnableNoRestore()
.EnableNoBuild()
.SetFilter(string.IsNullOrWhiteSpace(Filter) ? "(RunOnWindows=True)&(LoadFromGAC!=True)&(IIS!=True)&(Category!=AzureFunctions)&(SkipInCI!=True)" : Filter)
.SetTestTargetPlatform(TargetPlatform)
.SetIsDebugRun(isDebugRun)
.SetProcessEnvironmentVariable("MonitoringHomeDirectory", MonitoringHomeDirectory)
.SetLogsDirectory(TestLogsDirectory)
.When(!string.IsNullOrWhiteSpace(filter), c => c.SetFilter(filter))
.When(TestAllPackageVersions, o => o.SetProcessEnvironmentVariable("TestAllPackageVersions", "true"))
.When(CodeCoverageEnabled, ConfigureCodeCoverage)
.CombineWith(ClrProfilerIntegrationTests, (s, project) => s
Expand All @@ -1583,6 +1583,29 @@ void CompileSamplesThatDependOnDatadogTrace()
{
CopyDumpsToBuildData();
}

string GetFilter()
{
var dockerFilter = IncludeTestsRequiringDocker switch
{
true => "&(RequiresDockerDependency=true)",
false => "&(RequiresDockerDependency!=true)",
null => string.Empty,
};

var armFilter = IsArm64 ? "&(Category!=ArmUnsupported)" : string.Empty;

var filter = (string.IsNullOrWhiteSpace(Filter), IsWin) switch
{
(false, _) => $"({Filter}){dockerFilter}{armFilter}",
(true, false) => $"(Category!=LinuxUnsupported)&(Category!=Lambda)&(Category!=AzureFunctions)&(SkipInCI!=True){dockerFilter}{armFilter}",
// TODO: I think we should change this filter to run on Windows by default, e.g.
// (RunOnWindows!=False|Category=Smoke)&LoadFromGAC!=True&IIS!=True
(true, true) => "(RunOnWindows=True)&(LoadFromGAC!=True)&(IIS!=True)&(Category!=AzureFunctions)&(SkipInCI!=True)",
};

return filter;
}
});

Target CompileAzureFunctionsSamplesWindows => _ => _
Expand Down Expand Up @@ -1851,169 +1874,6 @@ void RunWindowsIisIntegrationTests(Project project)
}
});

Target RunLinuxIntegrationTests => _ => _
.After(CompileLinuxOrOsxIntegrationTests)
.After(CompileTrimmingSamples)
.DependsOn(CleanTestLogs)
.Description("Runs the linux integration tests")
.Requires(() => Framework)
.Requires(() => !IsWin)
.Triggers(PrintSnapshotsDiff)
.Executes(() =>
{
var isDebugRun = IsDebugRun();

var dockerFilter = IncludeTestsRequiringDocker switch
{
true => "&(RequiresDockerDependency=true)",
false => "&(RequiresDockerDependency!=true)",
null => string.Empty,
};

var armFilter = IsArm64 ? "&(Category!=ArmUnsupported)" : string.Empty;

var filter = string.IsNullOrWhiteSpace(Filter) switch
{
false => $"({Filter}){dockerFilter}{armFilter}",
true => $"(Category!=LinuxUnsupported)&(Category!=Lambda)&(Category!=AzureFunctions)&(SkipInCI!=True){dockerFilter}{armFilter}",
};

try
{
// Run these ones in parallel
DotNetTest(config => config
.SetConfiguration(BuildConfiguration)
.EnableNoRestore()
.EnableNoBuild()
.SetFramework(Framework)
//.WithMemoryDumpAfter(timeoutInMinutes: 30)
.EnableCrashDumps()
.SetFilter(filter)
.SetIsDebugRun(isDebugRun)
.SetProcessEnvironmentVariable("MonitoringHomeDirectory", MonitoringHomeDirectory)
.SetTestTargetPlatform(TargetPlatform)
.SetLogsDirectory(TestLogsDirectory)
.When(TestAllPackageVersions, o => o.SetProcessEnvironmentVariable("TestAllPackageVersions", "true"))
.When(IncludeMinorPackageVersions, o => o.SetProperty("IncludeMinorPackageVersions", "true"))
.When(IncludeTestsRequiringDocker is not null, o => o.SetProperty("IncludeTestsRequiringDocker", IncludeTestsRequiringDocker.Value ? "true" : "false"))
.When(CodeCoverageEnabled, ConfigureCodeCoverage)
.CombineWith(ParallelIntegrationTests, (s, project) => s
.EnableTrxLogOutput(GetResultsDirectory(project))
.WithDatadogLogger()
.SetProjectFile(project)),
degreeOfParallelism: 2);

// Run this one separately so we can tail output
DotNetTest(config => config
.SetConfiguration(BuildConfiguration)
.EnableNoRestore()
.EnableNoBuild()
.SetFramework(Framework)
//.WithMemoryDumpAfter(timeoutInMinutes: 30)
.EnableCrashDumps()
.SetFilter(filter)
.SetProcessEnvironmentVariable("MonitoringHomeDirectory", MonitoringHomeDirectory)
.SetTestTargetPlatform(TargetPlatform)
.SetLogsDirectory(TestLogsDirectory)
.When(TestAllPackageVersions, o => o.SetProcessEnvironmentVariable("TestAllPackageVersions", "true"))
.When(IncludeMinorPackageVersions, o => o.SetProperty("IncludeMinorPackageVersions", "true"))
.When(IncludeTestsRequiringDocker is not null, o => o.SetProperty("IncludeTestsRequiringDocker", IncludeTestsRequiringDocker.Value ? "true" : "false"))
.When(CodeCoverageEnabled, ConfigureCodeCoverage)
.CombineWith(ClrProfilerIntegrationTests, (s, project) => s
.EnableTrxLogOutput(GetResultsDirectory(project))
.WithDatadogLogger()
.SetProjectFile(project))
);
}
finally
{
CopyDumpsToBuildData();
}
});

Target RunOsxIntegrationTests => _ => _
.After(CompileLinuxOrOsxIntegrationTests)
.DependsOn(CleanTestLogs)
.Description("Runs the osx integration tests")
.Requires(() => Framework)
.Requires(() => IsOsx)
.Triggers(PrintSnapshotsDiff)
.Executes(() =>
{
var isDebugRun = IsDebugRun();

var dockerFilter = IncludeTestsRequiringDocker switch
{
true => "&(RequiresDockerDependency=true)",
false => "&(RequiresDockerDependency!=true)",
null => string.Empty,
};

var armFilter = IsArm64 ? "&(Category!=ArmUnsupported)" : string.Empty;

var filter = string.IsNullOrWhiteSpace(Filter) switch
{
false => Filter,
true => $"(Category!=LinuxUnsupported)&(Category!=Lambda)&(Category!=AzureFunctions)&(SkipInCI!=True){dockerFilter}{armFilter}",
};

var targetPlatform = IsArm64 ? (MSBuildTargetPlatform)"arm64" : TargetPlatform;

try
{
// Run these ones in parallel
DotNetTest(config => config
.SetConfiguration(BuildConfiguration)
.EnableNoRestore()
.EnableNoBuild()
.SetFramework(Framework)
//.WithMemoryDumpAfter(timeoutInMinutes: 30)
.EnableCrashDumps()
.SetFilter(filter)
.SetIsDebugRun(isDebugRun)
.SetProcessEnvironmentVariable("MonitoringHomeDirectory", MonitoringHomeDirectory)
.SetLocalOsxEnvironmentVariables()
.SetTestTargetPlatform(targetPlatform)
.SetLogsDirectory(TestLogsDirectory)
.When(TestAllPackageVersions, o => o.SetProcessEnvironmentVariable("TestAllPackageVersions", "true"))
.When(IncludeMinorPackageVersions, o => o.SetProperty("IncludeMinorPackageVersions", "true"))
.When(IncludeTestsRequiringDocker is not null, o => o.SetProperty("IncludeTestsRequiringDocker", IncludeTestsRequiringDocker.Value ? "true" : "false"))
.When(CodeCoverageEnabled, ConfigureCodeCoverage)
.CombineWith(ParallelIntegrationTests, (s, project) => s
.EnableTrxLogOutput(GetResultsDirectory(project))
.WithDatadogLogger()
.SetProjectFile(project)),
degreeOfParallelism: 2);

// Run this one separately so we can tail output
DotNetTest(config => config
.SetConfiguration(BuildConfiguration)
.EnableNoRestore()
.EnableNoBuild()
.SetFramework(Framework)
//.WithMemoryDumpAfter(timeoutInMinutes: 30)
.EnableCrashDumps()
.SetFilter(filter)
.SetProcessEnvironmentVariable("MonitoringHomeDirectory", MonitoringHomeDirectory)
.SetLocalOsxEnvironmentVariables()
.SetTestTargetPlatform(targetPlatform)
.SetLogsDirectory(TestLogsDirectory)
.When(TestAllPackageVersions, o => o.SetProcessEnvironmentVariable("TestAllPackageVersions", "true"))
.When(IncludeMinorPackageVersions, o => o.SetProperty("IncludeMinorPackageVersions", "true"))
.When(IncludeTestsRequiringDocker is not null, o => o.SetProperty("IncludeTestsRequiringDocker", IncludeTestsRequiringDocker.Value ? "true" : "false"))
.When(CodeCoverageEnabled, ConfigureCodeCoverage)
.CombineWith(ClrProfilerIntegrationTests, (s, project) => s
.EnableTrxLogOutput(GetResultsDirectory(project))
.WithDatadogLogger()
.SetProjectFile(project))
);
}
finally
{
CopyDumpsToBuildData();
}
});

Target InstallDdTraceTool => _ => _
.Description("Installs the dd-trace tool")
.OnlyWhenDynamic(() => (ToolSource != null))
Expand Down
6 changes: 3 additions & 3 deletions tracer/build/_build/Build.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -296,7 +296,7 @@ void DeleteReparsePoints(string path)
.DependsOn(BuildWindowsIntegrationTests)
.DependsOn(CompileSamples)
.DependsOn(CompileTrimmingSamples)
.DependsOn(RunWindowsIntegrationTests);
.DependsOn(RunIntegrationTests);

Target BuildAndRunWindowsRegressionTests => _ => _
.Requires(() => IsWin)
Expand Down Expand Up @@ -327,7 +327,7 @@ void DeleteReparsePoints(string path)
.Requires(() => !IsWin)
.Description("Builds and runs the linux integration tests. Requires docker-compose dependencies")
.DependsOn(BuildLinuxIntegrationTests)
.DependsOn(RunLinuxIntegrationTests)
.DependsOn(RunIntegrationTests)
.DependsOn(RunLinuxDdDotnetIntegrationTests);

Target BuildOsxIntegrationTests => _ => _
Expand All @@ -344,7 +344,7 @@ void DeleteReparsePoints(string path)
.DependsOn(BuildOsxIntegrationTests)
.DependsOn(CompileSamples)
.DependsOn(CompileTrimmingSamples)
.DependsOn(RunOsxIntegrationTests);
.DependsOn(RunIntegrationTests);

Target BuildAndRunToolArtifactTests => _ => _
.Description("Builds and runs the tool artifacts tests")
Expand Down
Loading