diff --git a/Source/Cake.Recipe/Content/addins.cake b/Source/Cake.Recipe/Content/addins.cake index 9ffc33a9..5408783e 100644 --- a/Source/Cake.Recipe/Content/addins.cake +++ b/Source/Cake.Recipe/Content/addins.cake @@ -4,7 +4,6 @@ #addin nuget:?package=Cake.Codecov&version=1.0.1 #addin nuget:?package=Cake.Coveralls&version=1.1.0 -#addin nuget:?package=Cake.Coverlet&version=2.5.4 #addin nuget:?package=Portable.BouncyCastle&version=1.8.5 #addin nuget:?package=Cake.Email&version=2.0.0&loaddependencies=true #addin nuget:?package=Cake.Incubator&version=7.0.0 diff --git a/Source/Cake.Recipe/Content/coverage.cake b/Source/Cake.Recipe/Content/coverage.cake new file mode 100644 index 00000000..c6a87f57 --- /dev/null +++ b/Source/Cake.Recipe/Content/coverage.cake @@ -0,0 +1,10 @@ +public enum CoverageToolType +{ + None = 0, + Auto, + CoverletAuto, + CoverletCollector, + CoverletConsole, + CoverletMSBuild, + OpenCover +} diff --git a/Source/Cake.Recipe/Content/coverlet.cake b/Source/Cake.Recipe/Content/coverlet.cake new file mode 100644 index 00000000..2825b795 --- /dev/null +++ b/Source/Cake.Recipe/Content/coverlet.cake @@ -0,0 +1,441 @@ +[Flags] +public enum CoverletOutputFormat +{ + Cobertura = 1, + JSON = 2, + LCOV = 4, + OpenCover = 8, + TeamCity = 16, + + Deterministic = Cobertura | JSON, + All = Cobertura | JSON | LCOV | OpenCover | TeamCity +} + +public class CoverletSettings +{ + public CoverletOutputFormat Format { get; set; } + + public bool? UseSourceLink { get; set; } + + public bool UseDeterministicReport { get; set; } + + public List ExcludeByFile { get; } = new List(); + public List ExcludeByAttribute { get; } = new List(); + public List Excludes { get; } = new List(); + public List Includes { get; } = new List(); +} + +private class CoverletConsoleContext : CakeContextAdapter +{ + private readonly CoverletConsoleProcessRunner _runner; + + public override ICakeLog Log { get; } + + public override IProcessRunner ProcessRunner => _runner; + + public FilePath FilePath => _runner.FilePath; + + public ProcessSettings Settings => _runner.ProcessSettings; + + public CoverletConsoleContext(ICakeContext context) + : base(context) + { + Log = new NullLog(); + _runner = new CoverletConsoleProcessRunner(); + } +} + +private class CoverletConsoleProcessRunner : IProcessRunner +{ + public FilePath FilePath { get; set; } + + public ProcessSettings ProcessSettings { get; set; } + + private sealed class InterceptedProcess : IProcess + { + public void Dispose() + { + } + + public void WaitForExit() + { + } + + public bool WaitForExit(int milliseconds) + { + return true; + } + + public int GetExitCode() + { + return 0; + } + + public IEnumerable GetStandardError() + { + return Enumerable.Empty(); + } + + public IEnumerable GetStandardOutput() + { + return Enumerable.Empty(); + } + + public void Kill() + { + } + } + + public IProcess Start(FilePath filePath, ProcessSettings settings) + { + FilePath = filePath; + ProcessSettings = settings; + return new InterceptedProcess(); + } +} + +public void RunCoverletConsole(ICakeContext context, Action action, FilePathCollection files) + => RequireTool(ToolSettings.CoverletGlobalTool, () => +{ + var tool = BuildParameters.BuildAgentOperatingSystem == PlatformFamily.Windows + ? context.Tools.Resolve("coverlet.exe") + : context.Tools.Resolve("coverlet"); + + if (tool is null) + { + throw new CakeException("Coverlet tool was not found."); + } + + var settings = ToolSettings.Coverlet ?? new CoverletSettings(); + + if (settings.UseDeterministicReport) + { + context.Warning("Deterministic Report is not supported when running Coverlet Console. Disabling deterministic report!"); + settings.UseDeterministicReport = false; + } + + UpdateCoverletFilters(settings); + + var outputDir = BuildParameters.Paths.Directories.TestCoverage.Combine("coverlet"); + + foreach (var file in files) + { + var absoluteFilePath = file.MakeAbsolute(context.Environment); + var interceptor = new CoverletConsoleContext(context); + action(interceptor, file); + + var arguments = new ProcessSettings + { + Arguments = absoluteFilePath.FullPath.Quote() + }; + + arguments.Arguments.AppendSwitchQuoted("--target", interceptor.FilePath.MakeAbsolute(context.Environment).FullPath); + var renderedArguments = interceptor.Settings.Arguments?.Render(); + + if (!string.IsNullOrEmpty(renderedArguments)) + { + renderedArguments = renderedArguments.Replace("\"", "\\\""); + arguments.Arguments.AppendSwitchQuoted("--targetargs", renderedArguments); + } + + arguments.Arguments.AppendSwitchQuoted("--output", outputDir.Combine(Guid.NewGuid().ToString()) + "/"); + + foreach (var format in GetFormatList(context, settings)) + { + arguments.Arguments.AppendSwitchQuoted("--format", format); + } + + if (settings.UseSourceLink.GetValueOrDefault()) + { + arguments.Arguments.Append("--use-source-link"); + } + + var exitCode = context.StartProcess(tool, arguments); + + if (exitCode != 0) + { + throw new CakeException(exitCode); + } + } +}); + +// An enumerable of project paths that was not handled by coverlet +public static IEnumerable RunCoverlet(ICakeContext context, DotNetCoreMSBuildSettings mSBuildSettings) +{ + var settings = ToolSettings.Coverlet ?? new CoverletSettings(); + + UpdateCoverletFilters(settings); + + var isAuto = ToolSettings.CoverageTool == CoverageToolType.Auto || ToolSettings.CoverageTool == CoverageToolType.CoverletAuto; + + if (isAuto || ToolSettings.CoverageTool == CoverageToolType.CoverletMSBuild) + { + var determineSourceLink = isAuto && settings.UseSourceLink is null; + + var projects = context.GetFiles(BuildParameters.TestDirectoryPath + (BuildParameters.TestFilePattern ?? "/**/*Tests.csproj")); + context.Information("Found {0} test projects", projects.Count); + + foreach (var project in projects) + { + var parsedProject = context.ParseProject(project, BuildParameters.Configuration); + + if (determineSourceLink) + { + settings.UseSourceLink = HasSourceLinkPackage(parsedProject); + } + + if (isAuto && parsedProject.HasPackage("coverlet.collector")) + { + RunCoverletCollector(context, project.FullPath, mSBuildSettings, settings); + } + else if (!isAuto || parsedProject.HasPackage("coverlet.msbuild")) + { + context.Warning("Using legacy Coverlet MSBuild for {0}. Due to problems on Linux it is recommended to switch to coverlet.collector instead!", project); + var name = parsedProject.RootNameSpace.Replace('.', '-'); + var outputDirectory = context.MakeAbsolute(BuildParameters.Paths.Directories.TestCoverage.Combine("coverlet").Combine(name)).FullPath; + RunCoverletDotNetMsBuild(context, project.FullPath, mSBuildSettings, settings, outputDirectory); + } + else if (ToolSettings.CoverageTool == CoverageToolType.CoverletAuto) + { + context.Warning("Unable to run Coverlet for project. Please add a reference to coverlet.collector or coverlet.msbuild."); + + var dotNetSettings = new DotNetCoreTestSettings + { + Configuration = BuildParameters.Configuration, + NoBuild = true, + }; + context.DotNetCoreTest(project.FullPath, dotNetSettings); + } + else + { + // No supported package was found, as such we + // return the path of the project so it can run + // through other processes. + yield return project.FullPath; + } + } + } + else if (ToolSettings.CoverageTool == CoverageToolType.CoverletCollector) + { + RunCoverletCollector(context, BuildParameters.SolutionFilePath.FullPath, mSBuildSettings, settings); + } + else + { + // We return string.Empty to indicate that no projects + // was attempted. + yield return string.Empty; + } +} + +public static void RunCoverletDotNetMsBuild(ICakeContext context, string solutionOrProject, DotNetCoreMSBuildSettings msBuildSettings, CoverletSettings settings, string outputDirectory) +{ + context.EnsureDirectoryExists(outputDirectory); + + if (!outputDirectory.EndsWith("/") && !outputDirectory.EndsWith("\\")) + { + outputDirectory = outputDirectory + "/"; + } + + var dotNetSettings = new DotNetCoreTestSettings + { + Configuration = BuildParameters.Configuration, + NoBuild = true, + ArgumentCustomization = args => { + args.AppendMSBuildSettings(msBuildSettings, context.Environment); + + args.AppendSwitch("/p:CollectCoverage", "=", "true"); + + var formats = GetFormat(context, settings); + args.AppendSwitch("/p:CoverletOutputFormat", "=", "\\\"" + formats + "\\\""); + args.AppendSwitchQuoted("/p:CoverletOutput", "=", outputDirectory); + + if (settings.UseSourceLink.GetValueOrDefault()) + { + args.AppendSwitch("/p:UseSourceLink", "=", "true"); + } + + if (settings.UseDeterministicReport) + { + args.AppendSwitch("/p:DeterministicReport", "=", "true"); + } + + if (settings.ExcludeByFile.Count > 0) + { + args.AppendSwitch("/p:ExcludeByFile", "=", "\\\"" + string.Join(",", settings.ExcludeByFile) + "\\\""); + } + + if (settings.ExcludeByAttribute.Count > 0) + { + args.AppendSwitch("/p:ExcludeByAttribute", "=", "\\\"" + string.Join(",", settings.ExcludeByAttribute) + "\\\""); + } + + if (settings.Includes.Count > 0) + { + args.AppendSwitch("/p:Include", "=", "\\\"" + string.Join(",", settings.Includes) + "\\\""); + } + + if (settings.Excludes.Count > 0) + { + args.AppendSwitchQuoted("/p:Exclude", "=", "\\\"" + string.Join(",", settings.Excludes) + "\\\""); + } + + return args; + } + }; + + context.DotNetCoreTest(solutionOrProject, dotNetSettings); +} + +public static void RunCoverletCollector(ICakeContext context, string solutionOrProject, DotNetCoreMSBuildSettings msBuildSettings, CoverletSettings settings) +{ + var collectorSb = new StringBuilder("XPlat Code Coverage"); + + var formats = GetFormat(context, settings); + collectorSb.Append(";Format=").Append(formats); + + if (settings.UseSourceLink.GetValueOrDefault()) + { + collectorSb.Append(";UseSourceLink=True"); + } + + if (settings.UseDeterministicReport) + { + collectorSb.Append(";DeterministicReport=True"); + } + + if (settings.ExcludeByFile.Count > 0) + { + collectorSb.Append(";ExcludeByFile=").Append(string.Join(",", settings.ExcludeByFile)); + } + + if (settings.ExcludeByAttribute.Count > 0) + { + collectorSb.Append(";ExcludeByAttribute=").Append(string.Join(",", settings.ExcludeByAttribute)); + } + + if (settings.Includes.Count > 0) + { + collectorSb.Append(";Include=").Append(string.Join(",", settings.Includes)); + } + + if (settings.Excludes.Count > 0) + { + collectorSb.Append(";Exclude=").Append(string.Join(",", settings.Excludes)); + } + + var dotNetSettings = new DotNetCoreTestSettings + { + Configuration = BuildParameters.Configuration, + NoBuild = true, + Collectors = new[]{ collectorSb.ToString() }, + ArgumentCustomization = args => { + args.AppendMSBuildSettings(msBuildSettings, context.Environment); + return args; + }, + ResultsDirectory = BuildParameters.Paths.Directories.TestCoverage.Combine("coverlet") + }; + + context.DotNetCoreTest(BuildParameters.SolutionFilePath.FullPath, dotNetSettings); +} + +private static bool HasSourceLinkPackage(CustomProjectParserResult project) +{ + return + project.HasPackage("Microsoft.SourceLink.GitHub") || + project.HasPackage("Microsoft.SourceLink.AzureRepos.Git") || + project.HasPackage("Microsoft.SourceLink.GitLab") || + project.HasPackage("Microsoft.SourceLink.BitBucket.Git") || + project.HasPackage("Microsoft.SourceLink.AzureDevOpsServer.Git") || + project.HasPackage("Microsoft.SourceLink.Gitea") || + project.HasPackage("Microsoft.SourceLink.GitWeb"); +} + +private static void UpdateCoverletFilters(CoverletSettings settings) +{ + settings.ExcludeByFile.AddRange(ToolSettings.TestCoverageExcludeByFile.Split(new[] { ';', ',' }, StringSplitOptions.RemoveEmptyEntries)); + settings.ExcludeByAttribute.AddRange(ToolSettings.TestCoverageExcludeByAttribute.Split(new[] { ';', ',' }, StringSplitOptions.RemoveEmptyEntries)); + + foreach (var filter in ToolSettings.TestCoverageFilter.Split(new[] { ' ', ';', ',' }, StringSplitOptions.RemoveEmptyEntries).Select(f => f.Trim())) + { + if (filter.Length == 0) + { + continue; + } + + if (filter[0] == '+') + { + settings.Includes.Add(filter.Substring(1)); + } + else if (filter[0] == '-') + { + settings.Excludes.Add(filter.Substring(1)); + } + else + { + settings.Includes.Add(filter); + } + } +} + +private static List GetFormatList(ICakeContext context, CoverletSettings settings) +{ + var results = new List(); + + ReadOnlySpan unsupportedDeterministicFormats = stackalloc CoverletOutputFormat[] + { + CoverletOutputFormat.OpenCover, + CoverletOutputFormat.LCOV, + CoverletOutputFormat.TeamCity + }; + + foreach (var availableFormat in Enum.GetValues(typeof(CoverletOutputFormat)).Cast()) + { + if (availableFormat == CoverletOutputFormat.All || availableFormat == CoverletOutputFormat.Deterministic) + { + continue; + } + + if (settings.Format.HasFlag(availableFormat)) + { + if (settings.UseDeterministicReport && HasFormat(unsupportedDeterministicFormats, availableFormat)) + { + context.Warning("Deterministic report is not supported when using {0} format. We will not be adding the {0} format!", availableFormat); + continue; + } + + results.Add(availableFormat.ToString().ToLowerInvariant()); + } + } + + if (results.Count == 0) + { + if (settings.UseDeterministicReport) + { + results.Add("cobertura"); + } + else + { + results.Add("opencover"); + } + } + + return results; +} + +private static string GetFormat(ICakeContext context, CoverletSettings settings) +{ + return string.Join(",", GetFormatList(context, settings)); +} + +private static bool HasFormat(ReadOnlySpan formats, CoverletOutputFormat testFormat) +{ + for (int i = 0; i < formats.Length; i++) + { + if (formats[i] == testFormat) + { + return true; + } + } + + return false; +} diff --git a/Source/Cake.Recipe/Content/opencover.cake b/Source/Cake.Recipe/Content/opencover.cake new file mode 100644 index 00000000..eabe1444 --- /dev/null +++ b/Source/Cake.Recipe/Content/opencover.cake @@ -0,0 +1,18 @@ +public void RunOpenCover(ICakeContext context, Action toolAction, bool registerUser = false) +{ + RequireTool(ToolSettings.OpenCoverTool, () => + { + context.OpenCover(toolAction, + BuildParameters.Paths.Files.TestCoverageOutputFilePath, + new OpenCoverSettings + { + OldStyle = true, + MergeOutput = context.FileExists(BuildParameters.Paths.Files.TestCoverageOutputFilePath), + ReturnTargetCodeOffset = 0, + Register = registerUser ? "user" : null + } + .WithFilter(ToolSettings.TestCoverageFilter) + .ExcludeByAttribute(ToolSettings.TestCoverageExcludeByAttribute) + .ExcludeByFile(ToolSettings.TestCoverageExcludeByFile)); + }); +} diff --git a/Source/Cake.Recipe/Content/testing.cake b/Source/Cake.Recipe/Content/testing.cake index fedda849..d53eed9d 100644 --- a/Source/Cake.Recipe/Content/testing.cake +++ b/Source/Cake.Recipe/Content/testing.cake @@ -2,203 +2,169 @@ // TASK DEFINITIONS /////////////////////////////////////////////////////////////////////////////// -BuildParameters.Tasks.InstallOpenCoverTask = Task("Install-OpenCover") - .WithCriteria(() => BuildParameters.BuildAgentOperatingSystem == PlatformFamily.Windows, "Not running on windows") - .Does(() => RequireTool(ToolSettings.OpenCoverTool, () => { - })); - BuildParameters.Tasks.TestNUnitTask = Task("Test-NUnit") - .IsDependentOn("Install-OpenCover") .WithCriteria(() => DirectoryExists(BuildParameters.Paths.Directories.PublishedNUnitTests), "No published NUnit tests") - .Does(() => RequireTool(ToolSettings.NUnitTool, () => { - EnsureDirectoryExists(BuildParameters.Paths.Directories.NUnitTestResults); + .Does(() => RequireTool(ToolSettings.NUnitTool, () => + { + var files = GetFiles(BuildParameters.Paths.Directories.PublishedNUnitTests + (BuildParameters.TestFilePattern ?? "/**/*Tests.dll")); + var settings = new NUnit3Settings + { + NoResults = true + }; - if (BuildParameters.BuildAgentOperatingSystem == PlatformFamily.Windows) + if (BuildParameters.BuildAgentOperatingSystem == PlatformFamily.Windows && (ToolSettings.CoverageTool == CoverageToolType.Auto || ToolSettings.CoverageTool == CoverageToolType.OpenCover)) { - OpenCover(tool => { - tool.NUnit3(GetFiles(BuildParameters.Paths.Directories.PublishedNUnitTests + (BuildParameters.TestFilePattern ?? "/**/*Tests.dll")), new NUnit3Settings { - NoResults = true - }); - }, - BuildParameters.Paths.Files.TestCoverageOutputFilePath, - new OpenCoverSettings - { - OldStyle = true, - ReturnTargetCodeOffset = 0 - } - .WithFilter(ToolSettings.TestCoverageFilter) - .ExcludeByAttribute(ToolSettings.TestCoverageExcludeByAttribute) - .ExcludeByFile(ToolSettings.TestCoverageExcludeByFile)); + RunOpenCover(Context, tool => tool.NUnit3(files, settings)); + } + else if (ToolSettings.CoverageTool == CoverageToolType.CoverletConsole || ToolSettings.CoverageTool == CoverageToolType.Auto || ToolSettings.CoverageTool == CoverageToolType.CoverletAuto) + { + RunCoverletConsole(Context, (tool, file) => tool.NUnit3(file.FullPath, settings), files); + } + else + { + NUnit3(files, settings); } }) ); BuildParameters.Tasks.TestxUnitTask = Task("Test-xUnit") - .IsDependentOn("Install-OpenCover") .WithCriteria(() => DirectoryExists(BuildParameters.Paths.Directories.PublishedxUnitTests), "No published xUnit tests") .Does(() => RequireTool(ToolSettings.XUnitTool, () => { EnsureDirectoryExists(BuildParameters.Paths.Directories.xUnitTestResults); - if (BuildParameters.BuildAgentOperatingSystem == PlatformFamily.Windows) - { - OpenCover(tool => { - tool.XUnit2(GetFiles(BuildParameters.Paths.Directories.PublishedxUnitTests + (BuildParameters.TestFilePattern ?? "/**/*Tests.dll")), new XUnit2Settings { - OutputDirectory = BuildParameters.Paths.Directories.xUnitTestResults, - XmlReport = true, - NoAppDomain = true - }); - }, - BuildParameters.Paths.Files.TestCoverageOutputFilePath, - new OpenCoverSettings - { - OldStyle = true, - ReturnTargetCodeOffset = 0 - } - .WithFilter(ToolSettings.TestCoverageFilter) - .ExcludeByAttribute(ToolSettings.TestCoverageExcludeByAttribute) - .ExcludeByFile(ToolSettings.TestCoverageExcludeByFile)); - } - }) -); + var files = GetFiles(BuildParameters.Paths.Directories.PublishedxUnitTests + (BuildParameters.TestFilePattern ?? "/**/*Tests.dll")); + var settings = new XUnit2Settings + { + OutputDirectory = BuildParameters.Paths.Directories.xUnitTestResults, + XmlReport = true, + NoAppDomain = true + }; + + if (BuildParameters.BuildAgentOperatingSystem == PlatformFamily.Windows && (ToolSettings.CoverageTool == CoverageToolType.Auto || ToolSettings.CoverageTool == CoverageToolType.OpenCover)) + { + RunOpenCover(Context, tool => tool.XUnit2(files, settings)); + } + else if (ToolSettings.CoverageTool == CoverageToolType.CoverletConsole || ToolSettings.CoverageTool == CoverageToolType.Auto || ToolSettings.CoverageTool == CoverageToolType.CoverletAuto) + { + RunCoverletConsole(Context, (tool, file) => tool.XUnit2(file.FullPath, settings), files); + } + else + { + XUnit2(files, settings); + } +})); BuildParameters.Tasks.TestMSTestTask = Task("Test-MSTest") - .IsDependentOn("Install-OpenCover") .WithCriteria(() => DirectoryExists(BuildParameters.Paths.Directories.PublishedMSTestTests), "No published MSTest tests") .Does(() => { EnsureDirectoryExists(BuildParameters.Paths.Directories.MSTestTestResults); - - // TODO: Need to add OpenCover here - MSTest(GetFiles(BuildParameters.Paths.Directories.PublishedMSTestTests + (BuildParameters.TestFilePattern ?? "/**/*Tests.dll")), new MSTestSettings() { + var files = GetFiles(BuildParameters.Paths.Directories.PublishedMSTestTests + (BuildParameters.TestFilePattern ?? "/**/*Tests.dll")); + var settings = new MSTestSettings + { NoIsolation = false - }); + }; + + if (BuildParameters.BuildAgentOperatingSystem == PlatformFamily.Windows && (ToolSettings.CoverageTool == CoverageToolType.Auto || ToolSettings.CoverageTool == CoverageToolType.OpenCover)) + { + RunOpenCover(Context, tool => tool.MSTest(files, settings)); + } + else if (ToolSettings.CoverageTool == CoverageToolType.CoverletConsole || ToolSettings.CoverageTool == CoverageToolType.Auto || ToolSettings.CoverageTool == CoverageToolType.CoverletAuto) + { + RunCoverletConsole(Context, (tool, file) => tool.MSTest(file.FullPath, settings), files); + } + else + { + MSTest(files, settings); + } }); BuildParameters.Tasks.TestVSTestTask = Task("Test-VSTest") - .IsDependentOn("Install-OpenCover") .WithCriteria(() => DirectoryExists(BuildParameters.Paths.Directories.PublishedVSTestTests), "No published VSTest tests") .Does(() => { EnsureDirectoryExists(BuildParameters.Paths.Directories.VSTestTestResults); - var vsTestSettings = new VSTestSettings() + var files = GetFiles(BuildParameters.Paths.Directories.PublishedMSTestTests + (BuildParameters.TestFilePattern ?? "/**/*Tests.dll")); + var settings = new VSTestSettings() { InIsolation = true }; if (AppVeyor.IsRunningOnAppVeyor) { - vsTestSettings.WithAppVeyorLogger(); + settings.WithAppVeyorLogger(); } - if (BuildParameters.BuildAgentOperatingSystem == PlatformFamily.Windows) + if (BuildParameters.BuildAgentOperatingSystem == PlatformFamily.Windows && (ToolSettings.CoverageTool == CoverageToolType.Auto || ToolSettings.CoverageTool == CoverageToolType.OpenCover)) { - OpenCover( - tool => { tool.VSTest(GetFiles(BuildParameters.Paths.Directories.PublishedVSTestTests + (BuildParameters.TestFilePattern ?? "/**/*Tests.dll")), vsTestSettings); }, - BuildParameters.Paths.Files.TestCoverageOutputFilePath, - new OpenCoverSettings - { - OldStyle = true, - ReturnTargetCodeOffset = 0 - } - .WithFilter(ToolSettings.TestCoverageFilter) - .ExcludeByAttribute(ToolSettings.TestCoverageExcludeByAttribute) - .ExcludeByFile(ToolSettings.TestCoverageExcludeByFile)); + RunOpenCover(Context, tool => tool.VSTest(files, settings)); + } + else if (ToolSettings.CoverageTool == CoverageToolType.CoverletConsole || ToolSettings.CoverageTool == CoverageToolType.Auto || ToolSettings.CoverageTool == CoverageToolType.CoverletAuto) + { + RunCoverletConsole(Context, (tool, file) => tool.VSTest(file.FullPath, settings), files); + } + else + { + VSTest(files, settings); } }); BuildParameters.Tasks.DotNetCoreTestTask = Task("DotNetCore-Test") - .IsDependentOn("Install-OpenCover") .Does((context, msBuildSettings) => { - var projects = GetFiles(BuildParameters.TestDirectoryPath + (BuildParameters.TestFilePattern ?? "/**/*Tests.csproj")); - // We create the coverlet settings here so we don't have to create the filters several times - var coverletSettings = new CoverletSettings - { - CollectCoverage = true, - // It is problematic to merge the reports into one, as such we use a custom directory for coverage results - CoverletOutputDirectory = BuildParameters.Paths.Directories.TestCoverage.Combine("coverlet"), - CoverletOutputFormat = CoverletOutputFormat.opencover, - ExcludeByFile = ToolSettings.TestCoverageExcludeByFile.Split(new [] {';' }, StringSplitOptions.None).ToList(), - ExcludeByAttribute = ToolSettings.TestCoverageExcludeByAttribute.Split(new [] {';' }, StringSplitOptions.None).ToList() - }; + CleanDirectory(BuildParameters.Paths.Directories.TestCoverage.Combine("coverlet")); - foreach (var filter in ToolSettings.TestCoverageFilter.Split(new [] {' ' }, StringSplitOptions.None)) + if (ToolSettings.CoverageTool == CoverageToolType.None) { - if (filter[0] == '+') + var dotNetSettings = new DotNetCoreTestSettings { - coverletSettings.WithInclusion(filter.TrimStart('+')); - } - else if (filter[0] == '-') - { - coverletSettings.WithFilter(filter.TrimStart('-')); - } + Configuration = BuildParameters.Configuration, + NoBuild = true, + }; + + DotNetCoreTest(BuildParameters.SolutionFilePath.FullPath, dotNetSettings); + return; + } + + var notRunProjects = new List(); + + if (ToolSettings.CoverageTool != CoverageToolType.OpenCover) + { + notRunProjects = RunCoverlet(context, msBuildSettings).ToList(); } + + if ((notRunProjects.Count == 1 && string.IsNullOrEmpty(notRunProjects[0])) || ToolSettings.CoverageTool == CoverageToolType.OpenCover) + { + notRunProjects = context.GetFiles(BuildParameters.TestDirectoryPath + (BuildParameters.TestFilePattern ?? "/**/*Tests.csproj")).Select(f => f.FullPath).ToList(); + } + var settings = new DotNetCoreTestSettings { Configuration = BuildParameters.Configuration, - NoBuild = true + NoBuild = true, }; - foreach (var project in projects) + if (ToolSettings.CoverageTool == CoverageToolType.CoverletAuto || ToolSettings.CoverageTool == CoverageToolType.CoverletConsole) { - Action testAction = tool => - { - tool.DotNetCoreTest(project.FullPath, settings); - }; - - var parsedProject = ParseProject(project, BuildParameters.Configuration); - - var coverletPackage = parsedProject.GetPackage("coverlet.msbuild"); - bool shouldAddSourceLinkArgument = false; // Set it to false by default due to OpenCover - if (coverletPackage != null) + RunCoverletConsole(Context, (tool, file) => tool.DotNetCoreTest(file.FullPath, settings), notRunProjects); + } + else + { + foreach (var project in notRunProjects) { - // If the version is a pre-release, we will assume that it is a later - // version than what we need, and thus TryParse will return false. - // If TryParse is successful we need to compare the coverlet version - // to ensure it is higher or equal to the version that includes the fix - // for using the SourceLink argument. - // https://github.com/coverlet-coverage/coverlet/issues/882 - Version coverletVersion; - shouldAddSourceLinkArgument = !Version.TryParse(coverletPackage.Version, out coverletVersion) - || coverletVersion >= Version.Parse("2.9.1"); - } - - settings.ArgumentCustomization = args => { - args.AppendMSBuildSettings(msBuildSettings, context.Environment); - if (shouldAddSourceLinkArgument && parsedProject.HasPackage("Microsoft.SourceLink.GitHub")) + Action testAction = tool => { - args.Append("/p:UseSourceLink=true"); - } - return args; - }; + tool.DotNetCoreTest(project, settings); + }; - if (parsedProject.IsNetCore && coverletPackage != null) - { - coverletSettings.CoverletOutputName = parsedProject.RootNameSpace.Replace('.', '-'); - DotNetCoreTest(project.FullPath, settings, coverletSettings); - } - else if (BuildParameters.BuildAgentOperatingSystem != PlatformFamily.Windows) - { - testAction(Context); - } - else - { if (BuildParameters.BuildAgentOperatingSystem == PlatformFamily.Windows) { - // We can not use msbuild properties together with opencover - settings.ArgumentCustomization = null; - OpenCover(testAction, - BuildParameters.Paths.Files.TestCoverageOutputFilePath, - new OpenCoverSettings { - ReturnTargetCodeOffset = 0, - OldStyle = true, - Register = "user", - MergeOutput = FileExists(BuildParameters.Paths.Files.TestCoverageOutputFilePath) - } - .WithFilter(ToolSettings.TestCoverageFilter) - .ExcludeByAttribute(ToolSettings.TestCoverageExcludeByAttribute) - .ExcludeByFile(ToolSettings.TestCoverageExcludeByFile)); + RunOpenCover(context, testAction, registerUser: true); + } + else + { + testAction(context); } } } @@ -240,7 +206,7 @@ BuildParameters.Tasks.GenerateFriendlyTestReportTask = Task("Generate-FriendlyTe BuildParameters.Tasks.GenerateLocalCoverageReportTask = Task("Generate-LocalCoverageReport") .WithCriteria(() => BuildParameters.IsLocalBuild, "Skipping due to not running a local build") .Does(() => RequireTool(BuildParameters.IsDotNetCoreBuild ? ToolSettings.ReportGeneratorGlobalTool : ToolSettings.ReportGeneratorTool, () => { - var coverageFiles = GetFiles(BuildParameters.Paths.Directories.TestCoverage + "/coverlet/*.xml"); + var coverageFiles = GetFiles(BuildParameters.Paths.Directories.TestCoverage + "/coverlet/**/*.*"); if (FileExists(BuildParameters.Paths.Files.TestCoverageOutputFilePath)) { coverageFiles += BuildParameters.Paths.Files.TestCoverageOutputFilePath; diff --git a/Source/Cake.Recipe/Content/tools.cake b/Source/Cake.Recipe/Content/tools.cake index d11893b5..b6e337d6 100644 --- a/Source/Cake.Recipe/Content/tools.cake +++ b/Source/Cake.Recipe/Content/tools.cake @@ -29,8 +29,10 @@ Action RequireToolNotRegistered = (tool, toolNames, ac } }; -Action RequireTool = (tool, action) => { +Action RequireTools = (tools, action) => +{ var script = MakeAbsolute(File(string.Format("./{0}.cake", Guid.NewGuid()))); + try { var arguments = new Dictionary(); @@ -43,7 +45,7 @@ Action RequireTool = (tool, action) => { arguments.Add("settings_skipverification", BuildParameters.CakeConfiguration.GetValue("Settings_SkipVerification")); } - System.IO.File.WriteAllText(script.FullPath, tool); + System.IO.File.WriteAllText(script.FullPath, string.Join(System.Environment.NewLine, tools)); CakeExecuteScript(script, new CakeSettings { @@ -60,3 +62,7 @@ Action RequireTool = (tool, action) => { action(); }; + +Action RequireTool = (tool, action) => { + RequireTools(new[] { tool }, action); +}; diff --git a/Source/Cake.Recipe/Content/toolsettings.cake b/Source/Cake.Recipe/Content/toolsettings.cake index 4856e726..e7c5eb5e 100644 --- a/Source/Cake.Recipe/Content/toolsettings.cake +++ b/Source/Cake.Recipe/Content/toolsettings.cake @@ -3,6 +3,7 @@ public static class ToolSettings static ToolSettings() { SetToolPreprocessorDirectives(); + CoverageTool = CoverageToolType.Auto; } public static string TestCoverageFilter { get; private set; } @@ -16,6 +17,7 @@ public static class ToolSettings public static string CodecovTool { get; private set; } public static string CoverallsTool { get; private set; } + public static string CoverletGlobalTool { get; private set; } public static string GitReleaseManagerTool { get; private set; } public static string GitVersionTool { get; private set; } public static string ReSharperTools { get; private set; } @@ -36,6 +38,9 @@ public static class ToolSettings public static string WyamGlobalTool { get; private set; } public static string KuduSyncGlobalTool { get; private set; } + public static CoverageToolType CoverageTool { get; private set; } + public static CoverletSettings Coverlet { get; private set; } + public static void SetToolPreprocessorDirectives( string codecovTool = "#tool nuget:?package=codecov&version=1.13.0", // This is specifically pinned to 0.7.0 as later versions of same package publish .Net Global Tool, rather than full framework version @@ -59,7 +64,8 @@ public static class ToolSettings string reportGeneratorGlobalTool = "#tool dotnet:?package=dotnet-reportgenerator-globaltool&version=4.8.5", string wyamGlobalTool = "#tool dotnet:?package=Wyam.Tool&version=2.2.9", // This is using an unofficial build of kudusync so that we can have a .Net Global tool version. This was generated from this PR: https://github.com/projectkudu/KuduSync.NET/pull/27 - string kuduSyncGlobalTool = "#tool dotnet:https://www.myget.org/F/cake-contrib/api/v3/index.json?package=KuduSync.Tool&version=1.5.4-g3916ad7218" + string kuduSyncGlobalTool = "#tool dotnet:https://www.myget.org/F/cake-contrib/api/v3/index.json?package=KuduSync.Tool&version=1.5.4-g3916ad7218", + string coverletGlobalTool = "#tool dotnet:?package=coverlet.console&version=3.2.0" ) { CodecovTool = codecovTool; @@ -82,6 +88,35 @@ public static class ToolSettings CoverallsGlobalTool = coverallsGlobalTool; WyamGlobalTool = wyamGlobalTool; KuduSyncGlobalTool = kuduSyncGlobalTool; + CoverletGlobalTool = coverletGlobalTool; + } + + public static void SetCoverletSettings( + ICakeContext context, + CoverletOutputFormat? outputFormat = null, + bool? useSourceLink = null, + bool? useDeterministicReport = null + ) + { + if (Coverlet == null) + { + Coverlet = new CoverletSettings(); + } + + if (outputFormat.HasValue) + { + Coverlet.Format = outputFormat.Value; + } + + if (useSourceLink.HasValue) + { + Coverlet.UseSourceLink = useSourceLink.Value; + } + + if (useDeterministicReport.HasValue) + { + Coverlet.UseDeterministicReport = useDeterministicReport.Value; + } } public static void SetToolSettings( @@ -90,10 +125,11 @@ public static class ToolSettings string testCoverageExcludeByAttribute = null, string testCoverageExcludeByFile = null, PlatformTarget? buildPlatformTarget = null, - MSBuildToolVersion buildMSBuildToolVersion = MSBuildToolVersion.Default, + MSBuildToolVersion buildMSBuildToolVersion = MSBuildToolVersion.VS2022, int? maxCpuCount = null, DirectoryPath targetFrameworkPathOverride = null, - bool skipDuplicatePackages = false + bool skipDuplicatePackages = false, + CoverageToolType coverageTool = CoverageToolType.Auto ) { context.Information("Setting up tools..."); @@ -105,6 +141,14 @@ public static class ToolSettings BuildPlatformTarget = buildPlatformTarget ?? PlatformTarget.MSIL; BuildMSBuildToolVersion = buildMSBuildToolVersion; MaxCpuCount = maxCpuCount ?? 0; + if (coverageTool == CoverageToolType.Auto) + { + context.Warning("Coverage tool will be automatically detected, automatic detection will be removed in the future."); + context.Warning("It is recommended to explicitly set the type to use."); + context.Warning(" - For .NET project coverlet collector is recommended, and will require a reference in the test project on the package coverlet.collector"); + context.Warning(" - For .NET Framework it is recommended to continue using OpenCover"); + } + CoverageTool = coverageTool; if (BuildParameters.ShouldUseTargetFrameworkPath && targetFrameworkPathOverride == null) { if (context.Environment.Runtime.IsCoreClr) @@ -122,5 +166,21 @@ public static class ToolSettings { TargetFrameworkPathOverride = targetFrameworkPathOverride?.FullPath; } + + if (CoverageTool != CoverageToolType.OpenCover) + { + var coverageFormat = CoverletOutputFormat.Cobertura; + + if (BuildParameters.BuildProvider.Type == BuildProviderType.TeamCity) + { + coverageFormat |= CoverletOutputFormat.TeamCity; + } + + SetCoverletSettings( + context: context, + outputFormat: coverageFormat, + useDeterministicReport: BuildParameters.IsLocalBuild && BuildParameters.BuildProvider.Type != BuildProviderType.TeamCity + ); + } } }