From c4414a8caa023900b62ef2ab4b8628ec35c8510f Mon Sep 17 00:00:00 2001 From: Andrey Kuleshov Date: Fri, 4 Feb 2022 17:14:52 +0300 Subject: [PATCH] Adding the support for the actual input of analyzers in SARIF format (#357) ### What's done: - supported reading of actual warnings in SARIF format --- buildSrc/src/main/kotlin/Versions.kt | 9 ++++++-- .../save/buildutils/DiktatConfiguration.kt | 7 ++++++ examples/kotlin-diktat/fix_and_warn/save.toml | 1 - examples/kotlin-diktat/sarif-actual/save.toml | 21 +++++++++++++++++ .../src/kotlin/EnumValueSnakeCaseTest.kt | 0 .../save-warnings.sarif | 0 .../{sarif => sarif-expected}/save.toml | 0 .../src/kotlin/EnumValueSnakeCaseTest.kt | 15 ++++++++++++ examples/kotlin-diktat/save.toml | 2 +- examples/kotlin-diktat/warn-dir/save.toml | 2 +- gradle.properties | 2 +- save-cli/build.gradle.kts | 1 + .../kotlin/org/cqfn/save/cli/GeneralTest.kt | 7 ++++-- save-core/build.gradle.kts | 21 +++++++++++++++++ .../save/core/integration/ClassicWarnTest.kt | 10 +++++++- .../org/cqfn/save/plugin/warn/WarnPlugin.kt | 23 +++++++++++++++---- 16 files changed, 108 insertions(+), 13 deletions(-) create mode 100644 examples/kotlin-diktat/sarif-actual/save.toml rename examples/kotlin-diktat/{sarif => sarif-actual}/src/kotlin/EnumValueSnakeCaseTest.kt (100%) rename examples/kotlin-diktat/{sarif => sarif-expected}/save-warnings.sarif (100%) rename examples/kotlin-diktat/{sarif => sarif-expected}/save.toml (100%) create mode 100644 examples/kotlin-diktat/sarif-expected/src/kotlin/EnumValueSnakeCaseTest.kt diff --git a/buildSrc/src/main/kotlin/Versions.kt b/buildSrc/src/main/kotlin/Versions.kt index 1095b73c5..946212df6 100644 --- a/buildSrc/src/main/kotlin/Versions.kt +++ b/buildSrc/src/main/kotlin/Versions.kt @@ -2,9 +2,14 @@ object Versions { object IntegrationTest { - const val ktlint = "0.39.0" + const val ktlint = "0.43.2" const val ktlintLink = "https://github.com/pinterest/ktlint/releases/download/$ktlint/ktlint" - const val diktat = "1.0.0-rc.2" + const val diktat = "1.0.2" const val diktatLink = "https://github.com/cqfn/diKTat/releases/download/v$diktat/diktat-$diktat.jar" + + const val oldKtlint = "0.39.0" + const val oldKtlintLink = "https://github.com/pinterest/ktlint/releases/download/$oldKtlint/ktlint" + const val oldDiktat = "1.0.0-rc.2" + const val oldDiktatLink = "https://github.com/cqfn/diKTat/releases/download/v$oldDiktat/diktat-$oldDiktat.jar" } } diff --git a/buildSrc/src/main/kotlin/org/cqfn/save/buildutils/DiktatConfiguration.kt b/buildSrc/src/main/kotlin/org/cqfn/save/buildutils/DiktatConfiguration.kt index e81941631..c012c081b 100644 --- a/buildSrc/src/main/kotlin/org/cqfn/save/buildutils/DiktatConfiguration.kt +++ b/buildSrc/src/main/kotlin/org/cqfn/save/buildutils/DiktatConfiguration.kt @@ -33,6 +33,8 @@ fun Project.createDiktatTask() { apply() configure { diktatConfigFile = rootProject.file("diktat-analysis.yml") + // FixMe: temporary before the release 1.0.3 of diktat + // reporterType = "sarif" inputs { include( "$rootDir/buildSrc/src/**/*.kt", @@ -54,4 +56,9 @@ fun Project.createDiktatTask() { tasks.findByName("diktatFix")?.let { this@register.dependsOn(it) } } } + + // FixMe: temporary before the release 1.0.3 of diktat + /* this.configurations.getByName("diktat").dependencies.add( + this.dependencies.create("com.pinterest.ktlint:ktlint-reporter-sarif:0.43.2") + ) */ } diff --git a/examples/kotlin-diktat/fix_and_warn/save.toml b/examples/kotlin-diktat/fix_and_warn/save.toml index 22649c7c8..5bb51cfca 100644 --- a/examples/kotlin-diktat/fix_and_warn/save.toml +++ b/examples/kotlin-diktat/fix_and_warn/save.toml @@ -2,7 +2,6 @@ tags = ["fix and warn"] description = "Autofixing of issues discovered by diKTat on smoke tests and warn about issues, which couldn't be auto-corrected" suiteName = "Autofix and Warn" -execCmd = "java -jar ktlint -R diktat.jar" ["fix and warn"] ["fix and warn".fix] diff --git a/examples/kotlin-diktat/sarif-actual/save.toml b/examples/kotlin-diktat/sarif-actual/save.toml new file mode 100644 index 000000000..4f4902515 --- /dev/null +++ b/examples/kotlin-diktat/sarif-actual/save.toml @@ -0,0 +1,21 @@ +[general] + tags = ["warn", "sarif"] + description = "Test warnings discovered by diKTat, reading expected warnings from SARIF file" + suiteName = "Only Warnings: with SARIF format" + expectedWarningsPattern = "// ;warn:?(.*):(\\d+):? ?(.+)?" + execCmd = "java -jar ktlint --disabled_rules=standard -R diktat.jar" + + + +[warn] + execFlags = "--reporter=sarif" + # regular expression to detect tests + testNameRegex = ".*Test.kt" + actualWarningsFormat = "SARIF" + fileNameCaptureGroupOut = 1 + lineCaptureGroupOut = 2 + columnCaptureGroupOut = 3 + messageCaptureGroupOut = 4 + exactWarningsMatch = false + warningTextHasColumn = true + warningTextHasLine = true \ No newline at end of file diff --git a/examples/kotlin-diktat/sarif/src/kotlin/EnumValueSnakeCaseTest.kt b/examples/kotlin-diktat/sarif-actual/src/kotlin/EnumValueSnakeCaseTest.kt similarity index 100% rename from examples/kotlin-diktat/sarif/src/kotlin/EnumValueSnakeCaseTest.kt rename to examples/kotlin-diktat/sarif-actual/src/kotlin/EnumValueSnakeCaseTest.kt diff --git a/examples/kotlin-diktat/sarif/save-warnings.sarif b/examples/kotlin-diktat/sarif-expected/save-warnings.sarif similarity index 100% rename from examples/kotlin-diktat/sarif/save-warnings.sarif rename to examples/kotlin-diktat/sarif-expected/save-warnings.sarif diff --git a/examples/kotlin-diktat/sarif/save.toml b/examples/kotlin-diktat/sarif-expected/save.toml similarity index 100% rename from examples/kotlin-diktat/sarif/save.toml rename to examples/kotlin-diktat/sarif-expected/save.toml diff --git a/examples/kotlin-diktat/sarif-expected/src/kotlin/EnumValueSnakeCaseTest.kt b/examples/kotlin-diktat/sarif-expected/src/kotlin/EnumValueSnakeCaseTest.kt new file mode 100644 index 000000000..a7b1319b0 --- /dev/null +++ b/examples/kotlin-diktat/sarif-expected/src/kotlin/EnumValueSnakeCaseTest.kt @@ -0,0 +1,15 @@ +package org.cqfn.diktat.test.resources.test.paragraph1.naming.enum_ + +// ;warn:3:1: [MISSING_KDOC_TOP_LEVEL] all public and internal top-level classes and functions should have Kdoc: EnumValueSnakeCaseTest (cannot be auto-corrected) +// ;warn:35: [WRONG_DECLARATIONS_ORDER] declarations of constants and enum members should be sorted alphabetically: enum entries order is incorrect +// ;warn:10:5: [ENUMS_SEPARATED] enum is incorrectly formatted: enums must end with semicolon +enum class EnumValueSnakeCaseTest { + // ;warn:$line+1:5: [ENUM_VALUE] enum values should be{{ in }}selected UPPER_CASE snake/PascalCase format: paSC_SAl_l + paSC_SAl_l, + + // ;warn:5: [ENUM_VALUE] enum values{{ should }}be in selected{{ UPPER_CASE }}snake/PascalCase format: PascAsl_f + PascAsl_f + // ;warn:$line-2:5: [ENUMS_SEPARATED] enum is incorrectly formatted: last enum entry must end with a comma + + // ;warn:1:9: {{.*}}[PACKAGE_NAME_INCORRECT_PREFIX] package name should start from company's domain: org.cqfn.save{{.*}} +} diff --git a/examples/kotlin-diktat/save.toml b/examples/kotlin-diktat/save.toml index 6c43a9313..1be4e070d 100644 --- a/examples/kotlin-diktat/save.toml +++ b/examples/kotlin-diktat/save.toml @@ -1,5 +1,5 @@ [general] - execCmd = "java -jar ktlint -R diktat.jar" + execCmd = "java -jar ktlint-old --disabled_rules=standard -R diktat-old.jar" description = "Test for diktat - linter and formatter for Kotlin" language = "Kotlin" # this is the default value, you don't need to add it explicitly, but can be useful, if you have different pattern: diff --git a/examples/kotlin-diktat/warn-dir/save.toml b/examples/kotlin-diktat/warn-dir/save.toml index 56c7352f8..4c92e7593 100644 --- a/examples/kotlin-diktat/warn-dir/save.toml +++ b/examples/kotlin-diktat/warn-dir/save.toml @@ -1,5 +1,5 @@ [general] - execCmd = "java -jar ktlint --disabled_rules=standard -R diktat.jar" + execCmd = "java -jar ktlint-old --disabled_rules=standard -R diktat-old.jar" tags = ["warn"] description = "Test for directory mode" diff --git a/gradle.properties b/gradle.properties index f0625a78f..8ee278b9f 100644 --- a/gradle.properties +++ b/gradle.properties @@ -3,7 +3,7 @@ group=org.cqfn.save # gradle performance org.gradle.parallel=true org.gradle.vfs.watch=true -org.gradle.jvmargs=-Xmx1g -XX:MaxMetaspaceSize=512m +org.gradle.jvmargs=-Xmx4096M -XX:MaxMetaspaceSize=512m org.gradle.caching=true # Kotlin diff --git a/save-cli/build.gradle.kts b/save-cli/build.gradle.kts index b0c31b6c0..680691d23 100644 --- a/save-cli/build.gradle.kts +++ b/save-cli/build.gradle.kts @@ -84,6 +84,7 @@ kotlin { tasks.withType().configureEach { dependsOn(":save-core:downloadTestResources") + dependsOn(":save-core:downloadOldTestResources") } // disable building of some binaries to speed up build diff --git a/save-cli/src/jvmTest/kotlin/org/cqfn/save/cli/GeneralTest.kt b/save-cli/src/jvmTest/kotlin/org/cqfn/save/cli/GeneralTest.kt index fddbe94fa..4e9397c02 100644 --- a/save-cli/src/jvmTest/kotlin/org/cqfn/save/cli/GeneralTest.kt +++ b/save-cli/src/jvmTest/kotlin/org/cqfn/save/cli/GeneralTest.kt @@ -62,8 +62,8 @@ class GeneralTest { assertTrue(fs.exists(destination)) // Check for existence of diktat and ktlint - assertTrue(fs.exists((examplesDir.toPath() / "diktat.jar"))) - assertTrue(fs.exists((examplesDir.toPath() / "ktlint"))) + assertTrue(fs.exists((examplesDir.toPath() / "diktat-old.jar"))) + assertTrue(fs.exists((examplesDir.toPath() / "ktlint-old"))) // Make sure, that we will check report, which will be obtained after current execution; remove old report if exist val reportFile = examplesDir.toPath() / "save.out.json".toPath() @@ -88,6 +88,9 @@ class GeneralTest { assertTrue(fs.exists(reportFile)) val reports: List = json.decodeFromString(fs.readFile(reportFile)) + + println("Following tests failed: ${reports.map { it.testSuite }}") + // Almost all result statuses should be Pass, except the few cases reports.forEach { report -> report.pluginExecutions.forEach { pluginExecution -> diff --git a/save-core/build.gradle.kts b/save-core/build.gradle.kts index f3a4a4506..0ee7cbaf0 100644 --- a/save-core/build.gradle.kts +++ b/save-core/build.gradle.kts @@ -71,6 +71,23 @@ tasks.withType>().forEach { it.dependsOn(generateVersionFileTaskProvider) } +tasks.register("downloadOldTestResources") { + src(listOf( + Versions.IntegrationTest.oldKtlintLink, + Versions.IntegrationTest.oldDiktatLink, + )) + dest("../examples/kotlin-diktat") + doLast { + file("../examples/kotlin-diktat/diktat-${Versions.IntegrationTest.oldDiktat}.jar").renameTo( + file("../examples/kotlin-diktat/diktat-old.jar") + ) + + file("../examples/kotlin-diktat/ktlint").renameTo( + file("../examples/kotlin-diktat/ktlint-old") + ) + } +} + tasks.register("downloadTestResources") { src(listOf( Versions.IntegrationTest.ktlintLink, @@ -83,15 +100,19 @@ tasks.register("downloadTestResources") { ) } } + val cleanupTask = tasks.register("cleanupTestResources") { this.dependsOn(":save-cli:jvmTest") mustRunAfter(tasks.withType()) doFirst { + file("../examples/kotlin-diktat/ktlint-old").delete() file("../examples/kotlin-diktat/ktlint").delete() + file("../examples/kotlin-diktat/diktat-old.jar").delete() file("../examples/kotlin-diktat/diktat.jar").delete() } } tasks.withType().configureEach { dependsOn("downloadTestResources") + dependsOn("downloadOldTestResources") finalizedBy("cleanupTestResources") } diff --git a/save-core/src/commonNonJsTest/kotlin/org/cqfn/save/core/integration/ClassicWarnTest.kt b/save-core/src/commonNonJsTest/kotlin/org/cqfn/save/core/integration/ClassicWarnTest.kt index 0cdad53e5..0d3ca4ddd 100644 --- a/save-core/src/commonNonJsTest/kotlin/org/cqfn/save/core/integration/ClassicWarnTest.kt +++ b/save-core/src/commonNonJsTest/kotlin/org/cqfn/save/core/integration/ClassicWarnTest.kt @@ -142,7 +142,15 @@ class ClassicWarnTest { @Test fun `execute warn-plugin with expected warnings from SARIF`() { runTestsWithDiktat( - listOf("sarif"), + listOf("sarif-expected"), + 1 + ) + } + + @Test + fun `execute warn-plugin with actual warnings from SARIF`() { + runTestsWithDiktat( + listOf("sarif-actual"), 1 ) } diff --git a/save-plugins/warn-plugin/src/commonMain/kotlin/org/cqfn/save/plugin/warn/WarnPlugin.kt b/save-plugins/warn-plugin/src/commonMain/kotlin/org/cqfn/save/plugin/warn/WarnPlugin.kt index 0d1a2abfe..15fd2dfb8 100644 --- a/save-plugins/warn-plugin/src/commonMain/kotlin/org/cqfn/save/plugin/warn/WarnPlugin.kt +++ b/save-plugins/warn-plugin/src/commonMain/kotlin/org/cqfn/save/plugin/warn/WarnPlugin.kt @@ -17,6 +17,7 @@ import org.cqfn.save.core.result.TestResult import org.cqfn.save.core.utils.ExecutionResult import org.cqfn.save.core.utils.ProcessExecutionException import org.cqfn.save.core.utils.ProcessTimeoutException +import org.cqfn.save.plugin.warn.sarif.toWarnings import org.cqfn.save.plugin.warn.utils.CmdExecutorWarn import org.cqfn.save.plugin.warn.utils.ResultsChecker import org.cqfn.save.plugin.warn.utils.Warning @@ -26,10 +27,13 @@ import org.cqfn.save.plugin.warn.utils.collectionSingleWarnings import org.cqfn.save.plugin.warn.utils.extractWarning import org.cqfn.save.plugin.warn.utils.getLineNumber +import io.github.detekt.sarif4k.SarifSchema210 import okio.FileSystem import okio.Path import kotlin.random.Random +import kotlinx.serialization.decodeFromString +import kotlinx.serialization.json.Json private typealias WarningMap = Map> @@ -108,7 +112,12 @@ class WarnPlugin( val expectedWarningsMap: WarningMap = copyPaths.zip(originalPaths).associate { (copyPath, originalPath) -> val warningsForCurrentPath = - copyPath.collectExpectedWarningsWithLineNumbers(warnPluginConfig, generalConfig, originalPaths, originalPath) + copyPath.collectExpectedWarningsWithLineNumbers( + warnPluginConfig, + generalConfig, + originalPaths, + originalPath + ) copyPath.name to warningsForCurrentPath } @@ -236,9 +245,16 @@ class WarnPlugin( @Suppress("AVOID_NULL_CHECKS") private fun collectActualWarningsWithLineNumbers( result: ExecutionResult, - warnPluginConfig: WarnPluginConfig + warnPluginConfig: WarnPluginConfig, ): WarningMap = when (warnPluginConfig.actualWarningsFormat) { - ActualWarningsFormat.SARIF -> throw NotImplementedError() + ActualWarningsFormat.SARIF -> Json.decodeFromString( + result.stdout.joinToString("\n") + ) + // setting emptyList() here instead of originalPaths to avoid invalid mapping + .toWarnings(testConfig.getRootConfig().directory, emptyList()) + .groupBy { it.fileName } + .mapValues { (_, warning) -> warning.sortedBy { it.message } } + else -> result.stdout.mapNotNull { with(warnPluginConfig) { it.extractWarning( @@ -253,7 +269,6 @@ class WarnPlugin( } .groupBy { it.fileName } .mapValues { (_, warning) -> warning.sortedBy { it.message } } - } private fun warnMissingExpectedWarnings(