Skip to content

Commit

Permalink
Merge remote-tracking branch 'origin/master' into memberMutantSupport
Browse files Browse the repository at this point in the history
  • Loading branch information
@dupdob
dupdob committed Dec 3, 2024
2 parents 0cd64fa + 7461da8 commit b78fc59
Show file tree
Hide file tree
Showing 37 changed files with 324 additions and 7,810 deletions.
4 changes: 3 additions & 1 deletion .editorconfig
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,9 @@ csharp_style_pattern_matching_over_is_with_cast_check = true:suggestion

csharp_style_throw_expression = true:suggestion

csharp_style_unused_value_expression_statement_preference = discard_variable:suggestion
# csharp_style_unused_value_expression_statement_preference = discard_variable:suggestion
# Disable IDE0058: Not relevant to fluent api
dotnet_diagnostic.IDE0058.severity = none
csharp_style_unused_value_assignment_preference = discard_variable:suggestion

csharp_style_var_for_built_in_types = true:suggestion
Expand Down
13 changes: 1 addition & 12 deletions azure-pipelines.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -199,18 +199,7 @@ stages:
command: test
projects: '**/Validation.csproj'
arguments: --filter Category=SingleTestProject

- template: pipeline-templates/run-integration-test-steps.yml
parameters:
testName: 'windows-fsharp'
workingDirectory: 'integrationtest/TargetProjects/NetCore/Library.FSharp.XUnit'
- task: DotNetCoreCLI@2
displayName: 'Assert integration test results'
inputs:
command: test
projects: '**/Validation.csproj'
arguments: --filter Category=FSharp


- job: LinuxTests
displayName: Run tests on Linux
pool:
Expand Down
265 changes: 122 additions & 143 deletions integrationtest/Validation/ValidationProject/ValidateStrykerResults.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,68 +11,49 @@
using Stryker.Core.Reporters.Json;
using Xunit;

namespace IntegrationTests
{
public class ValidateStrykerResults
{
private readonly ReadOnlyCollection<SyntaxKind> _blacklistedSyntaxKindsForMutating =
new(new[]
{
// Usings
SyntaxKind.UsingDirective,
SyntaxKind.UsingKeyword,
SyntaxKind.UsingStatement,
// Comments
SyntaxKind.DocumentationCommentExteriorTrivia,
SyntaxKind.EndOfDocumentationCommentToken,
SyntaxKind.MultiLineCommentTrivia,
SyntaxKind.MultiLineDocumentationCommentTrivia,
SyntaxKind.SingleLineCommentTrivia,
SyntaxKind.SingleLineDocumentationCommentTrivia,
SyntaxKind.XmlComment,
SyntaxKind.XmlCommentEndToken,
SyntaxKind.XmlCommentStartToken,
}
);
private readonly ReadOnlyCollection<SyntaxKind> _parentSyntaxKindsForMutating =
new(new[]
{
SyntaxKind.MethodDeclaration,
SyntaxKind.PropertyDeclaration,
SyntaxKind.ConstructorDeclaration,
SyntaxKind.FieldDeclaration,
SyntaxKind.OperatorDeclaration,
SyntaxKind.IndexerDeclaration,
}
);
private const string MutationReportJson = "mutation-report.json";
namespace Validation;

[Fact]
[Trait("Category", "SingleTestProject")]
public async Task CSharp_NetFramework_SingleTestProject()
public class ValidateStrykerResults
{
private readonly ReadOnlyCollection<SyntaxKind> _blacklistedSyntaxKindsForMutating =
new(new[]
{
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
var directory = new DirectoryInfo("../../../../../TargetProjects/NetFramework/FullFrameworkApp.Test/StrykerOutput");
directory.GetFiles("*.json", SearchOption.AllDirectories).ShouldNotBeEmpty("No reports available to assert");

var latestReport = directory.GetFiles(MutationReportJson, SearchOption.AllDirectories)
.OrderByDescending(f => f.LastWriteTime)
.First();

using var strykerRunOutput = File.OpenRead(latestReport.FullName);

var report = await JsonReportSerialization.DeserializeJsonReportAsync(strykerRunOutput);

CheckReportMutants(report, total: 29, ignored: 7, survived: 3, killed: 7, timeout: 0, nocoverage: 11);
}
// Usings
SyntaxKind.UsingDirective,
SyntaxKind.UsingKeyword,
SyntaxKind.UsingStatement,
// Comments
SyntaxKind.DocumentationCommentExteriorTrivia,
SyntaxKind.EndOfDocumentationCommentToken,
SyntaxKind.MultiLineCommentTrivia,
SyntaxKind.MultiLineDocumentationCommentTrivia,
SyntaxKind.SingleLineCommentTrivia,
SyntaxKind.SingleLineDocumentationCommentTrivia,
SyntaxKind.XmlComment,
SyntaxKind.XmlCommentEndToken,
SyntaxKind.XmlCommentStartToken,
}
);
private readonly ReadOnlyCollection<SyntaxKind> _parentSyntaxKindsForMutating =
new(new[]
{
SyntaxKind.MethodDeclaration,
SyntaxKind.PropertyDeclaration,
SyntaxKind.ConstructorDeclaration,
SyntaxKind.FieldDeclaration,
SyntaxKind.OperatorDeclaration,
SyntaxKind.IndexerDeclaration,
}
);
private const string MutationReportJson = "mutation-report.json";

[Fact]
[Trait("Category", "SingleTestProject")]
public async Task CSharp_NetCore_SingleTestProject()
[Fact]
[Trait("Category", "SingleTestProject")]
public async Task CSharp_NetFramework_SingleTestProject()
{
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
var directory = new DirectoryInfo("../../../../../TargetProjects/NetCore/NetCoreTestProject.XUnit/StrykerOutput");
var directory = new DirectoryInfo("../../../../../TargetProjects/NetFramework/FullFrameworkApp.Test/StrykerOutput");
directory.GetFiles("*.json", SearchOption.AllDirectories).ShouldNotBeEmpty("No reports available to assert");

var latestReport = directory.GetFiles(MutationReportJson, SearchOption.AllDirectories)
Expand All @@ -81,116 +62,114 @@ public async Task CSharp_NetCore_SingleTestProject()

using var strykerRunOutput = File.OpenRead(latestReport.FullName);

var report = await JsonReportSerialization.DeserializeJsonReportAsync(strykerRunOutput);
var report = await strykerRunOutput.DeserializeJsonReportAsync();

CheckReportMutants(report, total: 601, ignored: 247, survived: 4, killed: 9, timeout: 2, nocoverage: 308);
CheckReportTestCounts(report, total: 11);
CheckReportMutants(report, total: 29, ignored: 7, survived: 3, killed: 7, timeout: 0, nocoverage: 11);
}
}

[Fact]
[Trait("Category", "FSharp")]
public async Task FSharp_SingleTestProject()
{
var directory = new DirectoryInfo("../../../../../TargetProjects/NetCore/Library.FSharp.XUnit/StrykerOutput");
directory.GetFiles("*.json", SearchOption.AllDirectories).ShouldNotBeEmpty("No reports available to assert");
[Fact]
[Trait("Category", "SingleTestProject")]
public async Task CSharp_NetCore_SingleTestProject()
{
var directory = new DirectoryInfo("../../../../../TargetProjects/NetCore/NetCoreTestProject.XUnit/StrykerOutput");
directory.GetFiles("*.json", SearchOption.AllDirectories).ShouldNotBeEmpty("No reports available to assert");

var latestReport = directory
.GetFiles(MutationReportJson, SearchOption.AllDirectories)
.OrderByDescending(f => f.LastWriteTime)
.First();
var latestReport = directory.GetFiles(MutationReportJson, SearchOption.AllDirectories)
.OrderByDescending(f => f.LastWriteTime)
.First();

using var strykerRunOutput = File.OpenRead(latestReport.FullName);
using var strykerRunOutput = File.OpenRead(latestReport.FullName);

var report = await JsonReportSerialization.DeserializeJsonReportAsync(strykerRunOutput);
var report = await strykerRunOutput.DeserializeJsonReportAsync();

CheckReportMutants(report, total: 0, ignored: 0, survived: 0, killed: 0, timeout: 0, nocoverage: 0);
CheckReportTestCounts(report, total: 0);
}
CheckReportMutants(report, total: 601, ignored: 247, survived: 4, killed: 9, timeout: 2, nocoverage: 308);
CheckReportTestCounts(report, total: 11);
}

[Fact]
[Trait("Category", "MultipleTestProjects")]
public async Task CSharp_NetCore_WithTwoTestProjects()
{
var directory = new DirectoryInfo("../../../../../TargetProjects/NetCore/Targetproject/StrykerOutput");
directory.GetFiles("*.json", SearchOption.AllDirectories).ShouldNotBeEmpty("No reports available to assert");
[Fact]
[Trait("Category", "MultipleTestProjects")]
public async Task CSharp_NetCore_WithTwoTestProjects()
{
var directory = new DirectoryInfo("../../../../../TargetProjects/NetCore/Targetproject/StrykerOutput");
directory.GetFiles("*.json", SearchOption.AllDirectories).ShouldNotBeEmpty("No reports available to assert");

var latestReport = directory.GetFiles(MutationReportJson, SearchOption.AllDirectories)
.OrderByDescending(f => f.LastWriteTime)
.First();
var latestReport = directory.GetFiles(MutationReportJson, SearchOption.AllDirectories)
.OrderByDescending(f => f.LastWriteTime)
.First();

using var strykerRunOutput = File.OpenRead(latestReport.FullName);
using var strykerRunOutput = File.OpenRead(latestReport.FullName);

var report = await JsonReportSerialization.DeserializeJsonReportAsync(strykerRunOutput);
var report = await strykerRunOutput.DeserializeJsonReportAsync();

CheckReportMutants(report, total: 601, ignored: 105, survived: 5, killed: 11, timeout: 2, nocoverage: 447);
CheckReportTestCounts(report, total: 21);
}
CheckReportMutants(report, total: 601, ignored: 105, survived: 5, killed: 11, timeout: 2, nocoverage: 447);
CheckReportTestCounts(report, total: 21);
}

[Fact]
[Trait("Category", "Solution")]
public async Task CSharp_NetCore_SolutionRun()
{
var directory = new DirectoryInfo("../../../../../TargetProjects/NetCore/StrykerOutput");
directory.GetFiles("*.json", SearchOption.AllDirectories).ShouldNotBeEmpty("No reports available to assert");
[Fact]
[Trait("Category", "Solution")]
public async Task CSharp_NetCore_SolutionRun()
{
var directory = new DirectoryInfo("../../../../../TargetProjects/NetCore/StrykerOutput");
directory.GetFiles("*.json", SearchOption.AllDirectories).ShouldNotBeEmpty("No reports available to assert");

var latestReport = directory.GetFiles(MutationReportJson, SearchOption.AllDirectories)
.OrderByDescending(f => f.LastWriteTime)
.First();
var latestReport = directory.GetFiles(MutationReportJson, SearchOption.AllDirectories)
.OrderByDescending(f => f.LastWriteTime)
.First();

using var strykerRunOutput = File.OpenRead(latestReport.FullName);
using var strykerRunOutput = File.OpenRead(latestReport.FullName);

var report = await JsonReportSerialization.DeserializeJsonReportAsync(strykerRunOutput);
var report = await strykerRunOutput.DeserializeJsonReportAsync();

CheckReportMutants(report, total: 601, ignored: 247, survived: 4, killed: 9, timeout: 2, nocoverage: 308);
CheckReportTestCounts(report, total: 23);
}
CheckReportMutants(report, total: 601, ignored: 247, survived: 4, killed: 9, timeout: 2, nocoverage: 308);
CheckReportTestCounts(report, total: 23);
}

private void CheckMutationKindsValidity(IJsonReport report)
private void CheckMutationKindsValidity(IJsonReport report)
{
foreach (var file in report.Files)
{
foreach (var file in report.Files)
var syntaxTreeRootNode = CSharpSyntaxTree.ParseText(file.Value.Source).GetRoot();
var textLines = SourceText.From(file.Value.Source).Lines;

foreach (var mutation in file.Value.Mutants)
{
var syntaxTreeRootNode = CSharpSyntaxTree.ParseText(file.Value.Source).GetRoot();
var textLines = SourceText.From(file.Value.Source).Lines;

foreach (var mutation in file.Value.Mutants)
{
var linePositionSpan = new LinePositionSpan(new LinePosition(mutation.Location.Start.Line - 1, mutation.Location.Start.Column), new LinePosition(mutation.Location.End.Line - 1, mutation.Location.End.Column));
var textSpan = textLines.GetTextSpan(linePositionSpan);
var node = syntaxTreeRootNode.FindNode(textSpan);
var nodeKind = node.Kind();
_blacklistedSyntaxKindsForMutating.ShouldNotContain(nodeKind);

node.AncestorsAndSelf().ShouldContain(pn => _parentSyntaxKindsForMutating.Contains(pn.Kind()));
}
var linePositionSpan = new LinePositionSpan(new LinePosition(mutation.Location.Start.Line - 1, mutation.Location.Start.Column), new LinePosition(mutation.Location.End.Line - 1, mutation.Location.End.Column));
var textSpan = textLines.GetTextSpan(linePositionSpan);
var node = syntaxTreeRootNode.FindNode(textSpan);
var nodeKind = node.Kind();
_blacklistedSyntaxKindsForMutating.ShouldNotContain(nodeKind);

node.AncestorsAndSelf().ShouldContain(pn => _parentSyntaxKindsForMutating.Contains(pn.Kind()));
}
}
}

private void CheckReportMutants(IJsonReport report, int total, int ignored, int survived, int killed, int timeout, int nocoverage)
{
var actualTotal = report.Files.Select(f => f.Value.Mutants.Count()).Sum();
var actualIgnored = report.Files.Select(f => f.Value.Mutants.Count(m => m.Status == MutantStatus.Ignored.ToString())).Sum();
var actualSurvived = report.Files.Select(f => f.Value.Mutants.Count(m => m.Status == MutantStatus.Survived.ToString())).Sum();
var actualKilled = report.Files.Select(f => f.Value.Mutants.Count(m => m.Status == MutantStatus.Killed.ToString())).Sum();
var actualTimeout = report.Files.Select(f => f.Value.Mutants.Count(m => m.Status == MutantStatus.Timeout.ToString())).Sum();
var actualNoCoverage = report.Files.Select(f => f.Value.Mutants.Count(m => m.Status == MutantStatus.NoCoverage.ToString())).Sum();

report.Files.ShouldSatisfyAllConditions(
() => actualTotal.ShouldBe(total),
() => actualIgnored.ShouldBe(ignored),
() => actualSurvived.ShouldBe(survived),
() => actualKilled.ShouldBe(killed),
() => actualTimeout.ShouldBe(timeout),
() => actualNoCoverage.ShouldBe(nocoverage)
);

CheckMutationKindsValidity(report);
}
private void CheckReportMutants(IJsonReport report, int total, int ignored, int survived, int killed, int timeout, int nocoverage)
{
var actualTotal = report.Files.Select(f => f.Value.Mutants.Count()).Sum();
var actualIgnored = report.Files.Select(f => f.Value.Mutants.Count(m => m.Status == MutantStatus.Ignored.ToString())).Sum();
var actualSurvived = report.Files.Select(f => f.Value.Mutants.Count(m => m.Status == MutantStatus.Survived.ToString())).Sum();
var actualKilled = report.Files.Select(f => f.Value.Mutants.Count(m => m.Status == MutantStatus.Killed.ToString())).Sum();
var actualTimeout = report.Files.Select(f => f.Value.Mutants.Count(m => m.Status == MutantStatus.Timeout.ToString())).Sum();
var actualNoCoverage = report.Files.Select(f => f.Value.Mutants.Count(m => m.Status == MutantStatus.NoCoverage.ToString())).Sum();

report.Files.ShouldSatisfyAllConditions(
() => actualTotal.ShouldBe(total),
() => actualIgnored.ShouldBe(ignored),
() => actualSurvived.ShouldBe(survived),
() => actualKilled.ShouldBe(killed),
() => actualTimeout.ShouldBe(timeout),
() => actualNoCoverage.ShouldBe(nocoverage)
);

private void CheckReportTestCounts(IJsonReport report, int total)
{
var actualTotal = report.TestFiles.Sum(tf => tf.Value.Tests.Count);
CheckMutationKindsValidity(report);
}

actualTotal.ShouldBe(total);
}
private void CheckReportTestCounts(IJsonReport report, int total)
{
var actualTotal = report.TestFiles.Sum(tf => tf.Value.Tests.Count);

actualTotal.ShouldBe(total);
}
}
5 changes: 2 additions & 3 deletions src/Directory.Packages.props
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,9 @@
<PackageVersion Include="Microsoft.CodeAnalysis.CSharp" Version="4.11.0" />
<PackageVersion Include="NuGet.Protocol" Version="6.12.1" />
<PackageVersion Include="ResXResourceReader.NetStandard" Version="1.3.0" />
<PackageVersion Include="YamlDotNet" Version="16.2.0" />
<PackageVersion Include="YamlDotNet" Version="16.2.1" />
<PackageVersion Include="Grynwald.MarkdownGenerator" Version="3.0.106" />
<PackageVersion Include="LibGit2Sharp" Version="0.30.0" />
<PackageVersion Include="FSharp.Compiler.Service" Version="42.7.100-preview.22427.1" />
<PackageVersion Include="LibGit2Sharp" Version="0.31.0" />
<PackageVersion Include="DotNet.Glob" Version="3.1.3" />
<PackageVersion Include="Buildalyzer" Version="7.1.0" />
<PackageVersion Include="Microsoft.Web.LibraryManager.Build" Version="2.1.175" />
Expand Down
Loading

0 comments on commit b78fc59

Please sign in to comment.