diff --git a/help/container.txt b/help/container.txt index e6c57b4c26..b9c74053b3 100644 --- a/help/container.txt +++ b/help/container.txt @@ -15,10 +15,19 @@ Options: --exclude-base-image-vulns .............. Exclude from display base image vulnerabilities. --file= ......................... Include the path to the image's Dockerfile for more detailed advice. -h, --help - --json --platform= ..................... For multi-architecture images, specify the platform to test. Options are: [linux/amd64, linux/arm64, linux/riscv64, linux/ppc64le, linux/s390x, linux/386, linux/arm/v7 orlinux/arm/v6] + --json .................................. Return results in JSON format. + --json-file-output= + (test command only) + Save test output in JSON format directly to the specified file, regardless of whether or not you use the `--json` option. + This is especially useful if you want to display the human-readable test output via stdout and at the same time save the JSON format output to a file. + --sarif ................................. Return results in SARIF format. + --sarif-file-output= + (test command only) + Save test output in SARIF format directly to the specified file, regardless of whether or not you use the `--sarif` option. + This is especially useful if you want to display the human-readable test output via stdout and at the same time save the SARIF format output to a file. --print-deps ............................ Print the dependency tree before sending it for analysis. --project-name= ................. Specify a custom Snyk project name. --policy-path= .................... Manually pass a path to a snyk policy file. diff --git a/help/iac.txt b/help/iac.txt index 6fd1568ae8..5d78b2d6bc 100644 --- a/help/iac.txt +++ b/help/iac.txt @@ -12,6 +12,15 @@ Options: -h, --help --json .................................. Return results in JSON format. + --json-file-output= + (test command only) + Save test output in JSON format directly to the specified file, regardless of whether or not you use the `--json` option. + This is especially useful if you want to display the human-readable test output via stdout and at the same time save the JSON format output to a file. + --sarif ................................. Return results in SARIF format. + --sarif-file-output= + (test command only) + Save test output in SARIF format directly to the specified file, regardless of whether or not you use the `--sarif` option. + This is especially useful if you want to display the human-readable test output via stdout and at the same time save the SARIF format output to a file. --severity-threshold=... Only report issues of provided level or higher. Examples: diff --git a/package.json b/package.json index c1350820ed..0d6fbbbf8a 100644 --- a/package.json +++ b/package.json @@ -107,6 +107,7 @@ "@types/needle": "^2.0.4", "@types/node": "8.10.59", "@types/restify": "^8.4.2", + "@types/sarif": "^2.1.2", "@types/sinon": "^7.5.0", "@types/update-notifier": "^4.1.0", "@typescript-eslint/eslint-plugin": "2.18.0", diff --git a/src/cli/commands/test/iac-output.ts b/src/cli/commands/test/iac-output.ts index 0d164d3a7f..c4dfbbd342 100644 --- a/src/cli/commands/test/iac-output.ts +++ b/src/cli/commands/test/iac-output.ts @@ -7,6 +7,9 @@ import { import { getSeverityValue } from './formatters'; import { printPath } from './formatters/remediation-based-format-issues'; import { titleCaseText } from './formatters/legacy-format-issue'; +import * as sarif from 'sarif'; +import { SEVERITY } from '../../../lib/snyk-test/legacy'; +import upperFirst = require('lodash/upperFirst'); const debug = Debug('iac-output'); function formatIacIssue( @@ -122,3 +125,93 @@ export function capitalizePackageManager(type) { } } } + +export function createSarifOutputForIac( + iacTestResponses: IacTestResponse[], +): sarif.Log { + const sarifRes: sarif.Log = { + version: '2.1.0', + runs: [], + }; + + iacTestResponses + .filter((iacTestResponse) => iacTestResponse.result?.cloudConfigResults) + .forEach((iacTestResponse) => { + sarifRes.runs.push({ + tool: mapIacTestResponseToSarifTool(iacTestResponse), + results: mapIacTestResponseToSarifResults(iacTestResponse), + }); + }); + + return sarifRes; +} + +function getIssueLevel(severity: SEVERITY): sarif.ReportingConfiguration.level { + return severity === SEVERITY.HIGH ? 'error' : 'warning'; +} + +export function mapIacTestResponseToSarifTool( + iacTestResponse: IacTestResponse, +): sarif.Tool { + const tool: sarif.Tool = { + driver: { + name: 'Snyk Infrastructure as Code', + rules: [], + }, + }; + + const pushedIds = {}; + iacTestResponse.result.cloudConfigResults.forEach( + (iacIssue: AnnotatedIacIssue) => { + if (pushedIds[iacIssue.id]) { + return; + } + tool.driver.rules?.push({ + id: iacIssue.id, + shortDescription: { + text: `${upperFirst(iacIssue.severity)} - ${iacIssue.title}`, + }, + fullDescription: { + text: `Kubernetes ${iacIssue.subType}`, + }, + help: { + text: '', + markdown: iacIssue.description, + }, + defaultConfiguration: { + level: getIssueLevel(iacIssue.severity), + }, + properties: { + tags: ['security', `kubernetes/${iacIssue.subType}`], + }, + }); + pushedIds[iacIssue.id] = true; + }, + ); + return tool; +} + +export function mapIacTestResponseToSarifResults( + iacTestResponse: IacTestResponse, +): sarif.Result[] { + return iacTestResponse.result.cloudConfigResults.map( + (iacIssue: AnnotatedIacIssue) => ({ + ruleId: iacIssue.id, + message: { + text: `This line contains a potential ${iacIssue.severity} severity misconfiguration affacting the Kubernetes ${iacIssue.subType}`, + }, + locations: [ + { + physicalLocation: { + artifactLocation: { + uri: iacTestResponse.targetFile, + }, + region: { + startLine: iacIssue.lineNumber, + }, + }, + }, + ], + }), + ); +} diff --git a/src/cli/commands/test/index.ts b/src/cli/commands/test/index.ts index f584d175e9..859049cb05 100644 --- a/src/cli/commands/test/index.ts +++ b/src/cli/commands/test/index.ts @@ -13,6 +13,7 @@ import { ShowVulnPaths, SupportedProjectTypes, TestOptions, + OutputDataTypes, } from '../../../lib/types'; import { isLocalFolder } from '../../../lib/detect'; import { MethodArgs } from '../../args'; @@ -47,10 +48,11 @@ import { summariseVulnerableResults, } from './formatters'; import * as utils from './utils'; -import { getIacDisplayedOutput } from './iac-output'; +import { getIacDisplayedOutput, createSarifOutputForIac } from './iac-output'; import { getEcosystem, testEcosystem } from '../../../lib/ecosystems'; import { TestLimitReachedError } from '../../../lib/errors'; import { isMultiProjectScan } from '../../../lib/is-multi-project-scan'; +import { createSarifOutputForContainers } from './sarif-output'; const debug = Debug('snyk-test'); const SEPARATOR = '\n-------------------------------------------------------\n'; @@ -206,13 +208,19 @@ async function test(...args: MethodArgs): Promise { : results.map(mapIacTestResult); // backwards compat - strip array IFF only one result - const dataToSend = + const jsonData = errorMappedResults.length === 1 ? errorMappedResults[0] : errorMappedResults; - const stringifiedData = JSON.stringify(dataToSend, null, 2); - if (options.json) { + const { + stdout: dataToSend, + stringifiedData, + stringifiedJsonData, + stringifiedSarifData, + } = extractDataToSendFromResults(results, jsonData, options); + + if (options.json || options.sarif) { // if all results are ok (.ok == true) then return the json if (errorMappedResults.every((res) => res.ok)) { return TestCommandResult.createJsonTestCommandResult(stringifiedData); @@ -235,7 +243,8 @@ async function test(...args: MethodArgs): Promise { } err.json = stringifiedData; - err.jsonStringifiedResults = stringifiedData; + err.jsonStringifiedResults = stringifiedJsonData; + err.sarifStringifiedResults = stringifiedSarifData; throw err; } @@ -300,7 +309,8 @@ async function test(...args: MethodArgs): Promise { response += chalk.bold.green(summaryMessage); return TestCommandResult.createHumanReadableTestCommandResult( response, - stringifiedData, + stringifiedJsonData, + stringifiedSarifData, ); } } @@ -313,14 +323,16 @@ async function test(...args: MethodArgs): Promise { // first one error.code = vulnerableResults[0].code || 'VULNS'; error.userMessage = vulnerableResults[0].userMessage; - error.jsonStringifiedResults = stringifiedData; + error.jsonStringifiedResults = stringifiedJsonData; + error.sarifStringifiedResults = stringifiedSarifData; throw error; } response += chalk.bold.green(summaryMessage); return TestCommandResult.createHumanReadableTestCommandResult( response, - stringifiedData, + stringifiedJsonData, + stringifiedSarifData, ); } @@ -744,3 +756,31 @@ function dockerUserCTA(options) { } return ''; } + +function extractDataToSendFromResults( + results, + jsonData, + options: Options, +): OutputDataTypes { + let sarifData = {}; + if (options.sarif || options['sarif-file-output']) { + sarifData = !options.iac + ? createSarifOutputForContainers(results) + : createSarifOutputForIac(results); + } + + const stringifiedJsonData = JSON.stringify(jsonData, null, 2); + const stringifiedSarifData = JSON.stringify(sarifData, null, 2); + + const dataToSend = options.sarif ? sarifData : jsonData; + const stringifiedData = options.sarif + ? stringifiedSarifData + : stringifiedJsonData; + + return { + stdout: dataToSend, + stringifiedData, + stringifiedJsonData, + stringifiedSarifData, + }; +} diff --git a/src/cli/commands/test/sarif-output.ts b/src/cli/commands/test/sarif-output.ts new file mode 100644 index 0000000000..4aa7db1e8a --- /dev/null +++ b/src/cli/commands/test/sarif-output.ts @@ -0,0 +1,93 @@ +import * as sarif from 'sarif'; + +export function createSarifOutputForContainers(testResult): sarif.Log { + const sarifRes: sarif.Log = { + version: '2.1.0', + runs: [], + }; + + testResult.forEach((testResult) => { + sarifRes.runs.push({ + tool: getTool(testResult), + results: getResults(testResult), + }); + }); + + return sarifRes; +} + +export function getTool(testResult): sarif.Tool { + const tool: sarif.Tool = { + driver: { + name: 'Snyk Container', + rules: [], + }, + }; + + if (!testResult.vulnerabilities) { + return tool; + } + + const pushedIds = {}; + tool.driver.rules = testResult.vulnerabilities + .map((vuln) => { + if (pushedIds[vuln.id]) { + return; + } + const level = vuln.severity === 'high' ? 'error' : 'warning'; + const cve = vuln['identifiers']['CVE'][0]; + pushedIds[vuln.id] = true; + return { + id: vuln.id, + shortDescription: { + text: `${vuln.severity} severity ${vuln.title} vulnerability in ${vuln.packageName}`, + }, + fullDescription: { + text: cve + ? `(${cve}) ${vuln.name}@${vuln.version}` + : `${vuln.name}@${vuln.version}`, + }, + help: { + text: '', + markdown: vuln.description, + }, + defaultConfiguration: { + level: level, + }, + properties: { + tags: ['security', ...vuln.identifiers.CWE], + }, + }; + }) + .filter(Boolean); + return tool; +} + +export function getResults(testResult): sarif.Result[] { + const results: sarif.Result[] = []; + + if (!testResult.vulnerabilities) { + return results; + } + testResult.vulnerabilities.forEach((vuln) => { + results.push({ + ruleId: vuln.id, + message: { + text: `This file introduces a vulnerable ${vuln.packageName} package with a ${vuln.severity} severity vulnerability.`, + }, + locations: [ + { + physicalLocation: { + artifactLocation: { + uri: testResult.displayTargetFile, + }, + region: { + startLine: vuln.lineNumber || 1, + }, + }, + }, + ], + }); + }); + return results; +} diff --git a/src/cli/commands/types.ts b/src/cli/commands/types.ts index ada4a8006d..3a76d39139 100644 --- a/src/cli/commands/types.ts +++ b/src/cli/commands/types.ts @@ -17,15 +17,26 @@ export class CommandResult { export abstract class TestCommandResult extends CommandResult { protected jsonResult = ''; + protected sarifResult = ''; + public getJsonResult(): string { return this.jsonResult; } + public getSarifResult(): string { + return this.sarifResult; + } + public static createHumanReadableTestCommandResult( humanReadableResult: string, jsonResult: string, + sarifResult?: string, ): HumanReadableTestCommandResult { - return new HumanReadableTestCommandResult(humanReadableResult, jsonResult); + return new HumanReadableTestCommandResult( + humanReadableResult, + jsonResult, + sarifResult, + ); } public static createJsonTestCommandResult( @@ -37,15 +48,27 @@ export abstract class TestCommandResult extends CommandResult { class HumanReadableTestCommandResult extends TestCommandResult { protected jsonResult = ''; + protected sarifResult = ''; - constructor(humanReadableResult: string, jsonResult: string) { + constructor( + humanReadableResult: string, + jsonResult: string, + sarifResult?: string, + ) { super(humanReadableResult); this.jsonResult = jsonResult; + if (sarifResult) { + this.sarifResult = sarifResult; + } } public getJsonResult(): string { return this.jsonResult; } + + public getSarifResult(): string { + return this.sarifResult; + } } class JsonTestCommandResult extends TestCommandResult { diff --git a/src/cli/index.ts b/src/cli/index.ts index fac3922fe3..ecf819f532 100755 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -9,7 +9,7 @@ import * as runtime from './runtime'; import * as analytics from '../lib/analytics'; import * as alerts from '../lib/alerts'; import * as sln from '../lib/sln'; -import { args as argsLib, Args } from './args'; +import { args as argsLib, Args, ArgsOptions } from './args'; import { TestCommandResult } from './commands/types'; import { copy } from './copy'; import spinner = require('../lib/spinner'); @@ -23,6 +23,7 @@ import { OptionMissingErrorError, UnsupportedOptionCombinationError, ExcludeFlagBadInputError, + CustomError, } from '../lib/errors'; import stripAnsi from 'strip-ansi'; import { ExcludeFlagInvalidInputError } from '../lib/errors/exclude-flag-invalid-input'; @@ -38,6 +39,7 @@ import { MonitorOptions, SupportedUserReachableFacingCliArgs, } from '../lib/types'; +import { SarifFileOutputEmptyError } from '../lib/errors/empty-sarif-output-error'; const debug = Debug('snyk'); const EXIT_CODES = { @@ -72,15 +74,10 @@ async function runCommand(args: Args) { // also save the json (in error.json) to file if option is set if (args.command === 'test') { - const jsonOutputFile = args.options['json-file-output']; - if (jsonOutputFile) { - const jsonOutputFileStr = jsonOutputFile as string; - const fullOutputFilePath = getFullPath(jsonOutputFileStr); - saveJsonResultsToFile( - stripAnsi((commandResult as TestCommandResult).getJsonResult()), - fullOutputFilePath, - ); - } + const jsonResults = (commandResult as TestCommandResult).getJsonResult(); + saveResultsToFile(args.options, 'json', jsonResults); + const sarifResults = (commandResult as TestCommandResult).getSarifResult(); + saveResultsToFile(args.options, 'sarif', sarifResults); } return res; @@ -127,15 +124,8 @@ async function handleError(args, error) { } } - // also save the json (in error.json) to file if `--json-file-output` option is set - const jsonOutputFile = args.options['json-file-output']; - if (jsonOutputFile && error.jsonStringifiedResults) { - const fullOutputFilePath = getFullPath(jsonOutputFile); - saveJsonResultsToFile( - stripAnsi(error.jsonStringifiedResults), - fullOutputFilePath, - ); - } + saveResultsToFile(args.options, 'json', error.jsonStringifiedResults); + saveResultsToFile(args.options, 'sarif', error.jsonStringifiedResults); const analyticsError = vulnsFound ? { @@ -272,24 +262,10 @@ async function main() { throw new FileFlagBadInputError(); } - if (args.options['json-file-output'] && args.command !== 'test') { - throw new UnsupportedOptionCombinationError([ - args.command, - 'json-file-output', - ]); - } + validateUnsupportedSarifCombinations(args); - const jsonFileOptionSet: boolean = 'json-file-output' in args.options; - if (jsonFileOptionSet) { - const jsonFileOutputValue = args.options['json-file-output']; - if (!jsonFileOutputValue || typeof jsonFileOutputValue !== 'string') { - throw new JsonFileOutputBadInputError(); - } - // On Windows, seems like quotes get passed in - if (jsonFileOutputValue === "''" || jsonFileOutputValue === '""') { - throw new JsonFileOutputBadInputError(); - } - } + validateOutputFile(args.options, 'json', new JsonFileOutputBadInputError()); + validateOutputFile(args.options, 'sarif', new SarifFileOutputEmptyError()); checkPaths(args); @@ -390,3 +366,80 @@ function validateUnsupportedOptionCombinations( } } } + +function validateUnsupportedSarifCombinations(args) { + if (args.options['json-file-output'] && args.command !== 'test') { + throw new UnsupportedOptionCombinationError([ + args.command, + 'json-file-output', + ]); + } + + if (args.options['sarif'] && args.command !== 'test') { + throw new UnsupportedOptionCombinationError([args.command, 'sarif']); + } + + if (args.options['sarif'] && args.options['json']) { + throw new UnsupportedOptionCombinationError([ + args.command, + 'sarif', + 'json', + ]); + } + + if (args.options['sarif-file-output'] && args.command !== 'test') { + throw new UnsupportedOptionCombinationError([ + args.command, + 'sarif-file-output', + ]); + } + + if ( + args.options['sarif'] && + args.options['docker'] && + !args.options['file'] + ) { + throw new OptionMissingErrorError('sarif', ['--file']); + } + + if ( + args.options['sarif-file-output'] && + args.options['docker'] && + !args.options['file'] + ) { + throw new OptionMissingErrorError('sarif-file-output', ['--file']); + } +} + +function saveResultsToFile( + options: ArgsOptions, + outputType: string, + jsonResults: string, +) { + const outputFile = options[`${outputType}-file-output`]; + if (outputFile && jsonResults) { + const outputFileStr = outputFile as string; + const fullOutputFilePath = getFullPath(outputFileStr); + saveJsonResultsToFile(stripAnsi(jsonResults), fullOutputFilePath); + } +} + +function validateOutputFile( + options: ArgsOptions, + outputType: string, + error: CustomError, +) { + const fileOutputValue = options[`${outputType}-file-output`]; + + if (fileOutputValue === undefined) { + return; + } + + if (!fileOutputValue || typeof fileOutputValue !== 'string') { + throw error; + } + // On Windows, seems like quotes get passed in + if (fileOutputValue === "''" || fileOutputValue === '""') { + throw error; + } +} diff --git a/src/lib/errors/empty-sarif-output-error.ts b/src/lib/errors/empty-sarif-output-error.ts new file mode 100644 index 0000000000..a7e28ce4a1 --- /dev/null +++ b/src/lib/errors/empty-sarif-output-error.ts @@ -0,0 +1,13 @@ +import { CustomError } from './custom-error'; + +export class SarifFileOutputEmptyError extends CustomError { + private static ERROR_CODE = 422; + private static ERROR_MESSAGE = + 'Empty --sarif-file-output argument. Did you mean --file=path/to/output-file.json ?'; + + constructor() { + super(SarifFileOutputEmptyError.ERROR_MESSAGE); + this.code = SarifFileOutputEmptyError.ERROR_CODE; + this.userMessage = SarifFileOutputEmptyError.ERROR_MESSAGE; + } +} diff --git a/src/lib/snyk-test/iac-test-result.ts b/src/lib/snyk-test/iac-test-result.ts index f8e6060ab4..0ec4510543 100644 --- a/src/lib/snyk-test/iac-test-result.ts +++ b/src/lib/snyk-test/iac-test-result.ts @@ -13,6 +13,7 @@ export interface AnnotatedIacIssue { // Legacy fields from Registry, unused. name?: string; from?: string[]; + lineNumber?: number; } type FILTERED_OUT_FIELDS = 'cloudConfigPath' | 'name' | 'from'; diff --git a/src/lib/types.ts b/src/lib/types.ts index 7d19a4cfd8..c26b427b50 100644 --- a/src/lib/types.ts +++ b/src/lib/types.ts @@ -74,6 +74,7 @@ export interface Options { // Used with the Docker plugin only. Allows application scanning. 'app-vulns'?: boolean; debug?: boolean; + sarif?: boolean; } // TODO(kyegupov): catch accessing ['undefined-properties'] via noImplicitAny @@ -138,6 +139,13 @@ export interface SpinnerOptions { cleanup?: any; } +export interface OutputDataTypes { + stdout: any; + stringifiedData: string; + stringifiedJsonData: string; + stringifiedSarifData: string; +} + export type SupportedProjectTypes = IacProjectTypes | SupportedPackageManagers; // TODO: finish typing this there are many more! diff --git a/test/acceptance/cli-args.test.ts b/test/acceptance/cli-args.test.ts index 743595782f..7be9d4b56e 100644 --- a/test/acceptance/cli-args.test.ts +++ b/test/acceptance/cli-args.test.ts @@ -1,6 +1,9 @@ import { test } from 'tap'; import { exec } from 'child_process'; -import { sep } from 'path'; +import { sep, join } from 'path'; +import { readFileSync, unlinkSync, rmdirSync, mkdirSync, existsSync } from 'fs'; +import { v4 as uuidv4 } from 'uuid'; +import { UnsupportedOptionCombinationError } from '../../src/lib/errors/unsupported-option-combination-error'; const osName = require('os-name'); @@ -344,3 +347,173 @@ test('`test --json-file-output no value produces error message`', (t) => { optionsToTest.forEach(validate); }); + +test('`test --json-file-output can save JSON output to file while sending human readable output to stdout`', (t) => { + t.plan(2); + + exec( + `node ${main} test --json-file-output=snyk-direct-json-test-output.json`, + (err, stdout) => { + if (err) { + throw err; + } + t.match(stdout, 'Organization:', 'contains human readable output'); + const outputFileContents = readFileSync( + 'snyk-direct-json-test-output.json', + 'utf-8', + ); + unlinkSync('./snyk-direct-json-test-output.json'); + const jsonObj = JSON.parse(outputFileContents); + const okValue = jsonObj.ok as boolean; + t.ok(okValue, 'JSON output ok'); + }, + ); +}); + +test('`test --json-file-output produces same JSON output as normal JSON output to stdout`', (t) => { + t.plan(1); + + exec( + `node ${main} test --json --json-file-output=snyk-direct-json-test-output.json`, + (err, stdout) => { + if (err) { + throw err; + } + const stdoutJson = stdout; + const outputFileContents = readFileSync( + 'snyk-direct-json-test-output.json', + 'utf-8', + ); + unlinkSync('./snyk-direct-json-test-output.json'); + t.equals(stdoutJson, outputFileContents); + }, + ); +}); + +test('`test --json-file-output can handle a relative path`', (t) => { + t.plan(1); + + // if 'test-output' doesn't exist, created it + if (!existsSync('test-output')) { + mkdirSync('test-output'); + } + + const tempFolder = uuidv4(); + const outputPath = `test-output/${tempFolder}/snyk-direct-json-test-output.json`; + + exec( + `node ${main} test --json --json-file-output=${outputPath}`, + (err, stdout) => { + if (err) { + throw err; + } + const stdoutJson = stdout; + const outputFileContents = readFileSync(outputPath, 'utf-8'); + unlinkSync(outputPath); + rmdirSync(`test-output/${tempFolder}`); + t.equals(stdoutJson, outputFileContents); + }, + ); +}); + +test( + '`test --json-file-output can handle an absolute path`', + { skip: iswindows }, + (t) => { + t.plan(1); + + // if 'test-output' doesn't exist, created it + if (!existsSync('test-output')) { + mkdirSync('test-output'); + } + + const tempFolder = uuidv4(); + const outputPath = join( + process.cwd(), + `test-output/${tempFolder}/snyk-direct-json-test-output.json`, + ); + + exec( + `node ${main} test --json --json-file-output=${outputPath}`, + (err, stdout) => { + if (err) { + throw err; + } + const stdoutJson = stdout; + const outputFileContents = readFileSync(outputPath, 'utf-8'); + unlinkSync(outputPath); + rmdirSync(`test-output/${tempFolder}`); + t.equals(stdoutJson, outputFileContents); + }, + ); + }, +); + +test('flags not allowed with --sarif', (t) => { + t.plan(1); + exec(`node ${main} test --sarif --json`, (err, stdout) => { + if (err) { + throw err; + } + t.match( + stdout.trim(), + new UnsupportedOptionCombinationError(['test', 'sarif', 'json']) + .userMessage, + ); + }); +}); + +test('test --sarif-file-output no value produces error message', (t) => { + const optionsToTest = [ + '--sarif-file-output', + '--sarif-file-output=', + '--sarif-file-output=""', + "--sarif-file-output=''", + ]; + + t.plan(optionsToTest.length); + + const validate = (sarifFileOutputOption: string) => { + const fullCommand = `node ${main} test ${sarifFileOutputOption}`; + exec(fullCommand, (err, stdout) => { + if (err) { + throw err; + } + t.equals( + stdout.trim(), + 'Empty --sarif-file-output argument. Did you mean --file=path/to/output-file.json ?', + ); + }); + }; + + optionsToTest.forEach(validate); +}); + +test('`test --json-file-output can be used at the same time as --sarif-file-output`', (t) => { + t.plan(3); + + exec( + `node ${main} test --json-file-output=snyk-direct-json-test-output.json --sarif-file-output=snyk-direct-sarif-test-output.json`, + (err, stdout) => { + if (err) { + throw err; + } + + const sarifOutput = JSON.parse( + readFileSync('snyk-direct-sarif-test-output.json', 'utf-8'), + ); + const jsonOutput = JSON.parse( + readFileSync('snyk-direct-json-test-output.json', 'utf-8'), + ); + + unlinkSync('./snyk-direct-json-test-output.json'); + unlinkSync('./snyk-direct-sarif-test-output.json'); + + t.match(stdout, 'Organization:', 'contains human readable output'); + + t.ok(jsonOutput.ok, 'JSON output OK'); + t.match(sarifOutput.version, '2.1.0', 'SARIF output OK'); + t.end(); + }, + ); +}); diff --git a/test/acceptance/cli-test/cli-test.docker.spec.ts b/test/acceptance/cli-test/cli-test.docker.spec.ts index 7c4ca628fe..f4f284253c 100644 --- a/test/acceptance/cli-test/cli-test.docker.spec.ts +++ b/test/acceptance/cli-test/cli-test.docker.spec.ts @@ -466,6 +466,36 @@ export const DockerTests: AcceptanceTests = { t.match(msg, 'Fixed in: 5.15.1'); } }, + + '`test --docker --file=Dockerfile --sarif `': (params, utils) => async ( + t, + ) => { + const testableObject = await testSarif(t, utils, params, { sarif: true }); + const results = JSON.parse(testableObject.message); + const sarifResults = require('../fixtures/docker/sarif-container-result.json'); + t.deepEqual(results, sarifResults, 'stdout containing sarif results'); + t.end(); + }, + + '`test --docker --file=Dockerfile --sarif --sarif-output-file`': ( + params, + utils, + ) => async (t) => { + const testableObject = await testSarif(t, utils, params, { + sarif: true, + 'sarif-output-file': 'sarif-test-file.json', + }); + const results = JSON.parse(testableObject.message); + const sarifStringifiedResults = JSON.parse( + testableObject.sarifStringifiedResults, + ); + t.deepEqual( + results, + sarifStringifiedResults, + 'stdout and stringified sarif results are the same', + ); + t.end(); + }, }, }; @@ -485,3 +515,58 @@ function stubDockerPluginResponse(plugins, fixture: string | object, t) { return spyPlugin; } + +async function testSarif(t, utils, params, flags) { + stubDockerPluginResponse( + params.plugins, + { + plugin: { + packageManager: 'deb', + }, + package: { + name: 'docker-image', + dependencies: { + 'apt/libapt-pkg5.0': { + version: '1.6.3ubuntu0.1', + dependencies: { + 'bzip2/libbz2-1.0': { + version: '1.0.6-8.1', + }, + }, + }, + 'bzip2/libbz2-1.0': { + version: '1.0.6-8.1', + }, + 'bzr/libbz2-1.0': { + version: '1.0.6-8.1', + }, + }, + docker: { + binaries: { + Analysis: [{ name: 'node', version: '5.10.1' }], + }, + }, + }, + }, + t, + ); + + const testableObject = await testPrep(t, utils, params, flags); + return testableObject; +} + +async function testPrep(t, utils, params, additionaLpropsForCli) { + utils.chdirWorkspaces(); + const vulns = require('../fixtures/docker/find-result.json'); + params.server.setNextResponse(vulns); + + try { + await params.cli.test('test alpine', { + docker: true, + ...additionaLpropsForCli, + }); + t.fail('should have thrown'); + } catch (testableObject) { + return testableObject; + } +} diff --git a/test/acceptance/cli-test/cli-test.iac-k8s.spec.ts b/test/acceptance/cli-test/cli-test.iac-k8s.spec.ts index 1924f71eaa..70b3ad31d5 100644 --- a/test/acceptance/cli-test/cli-test.iac-k8s.spec.ts +++ b/test/acceptance/cli-test/cli-test.iac-k8s.spec.ts @@ -2,10 +2,13 @@ import * as _ from 'lodash'; import { iacTest, iacTestJson, + iacTestSarif, iacErrorTest, iacTestMetaAssertions, iacTestJsonAssertions, + iacTestSarifAssertions, iacTestResponseFixturesByThreshold, + iacTestSarifFileOutput, } from './cli-test.iac-k8s.utils'; import { CommandResult } from '../../../src/cli/commands/types'; @@ -20,19 +23,6 @@ import { AcceptanceTests } from './cli-test.acceptance.test'; export const IacK8sTests: AcceptanceTests = { language: 'Iac (Kubernetes)', tests: { - '`iac test multi-file.yaml --json - no issues`': (params, utils) => async ( - t, - ) => { - utils.chdirWorkspaces(); - const commandResult: CommandResult = await params.cli.test( - 'iac-kubernetes/multi-file.yaml', - { - iac: true, - }, - ); - const res: any = JSON.parse((commandResult as any).jsonResult); - iacTestJsonAssertions(t, res, null, false); - }, '`iac test multi.yaml - no issues`': (params, utils) => async (t) => { utils.chdirWorkspaces(); @@ -140,6 +130,23 @@ export const IacK8sTests: AcceptanceTests = { utils, ) => async (t) => await iacTest(t, utils, params, 'high', 1), + '`iac test multi-file.yaml --json - no issues`': (params, utils) => async ( + t, + ) => { + utils.chdirWorkspaces(); + let testableObject; + try { + await params.cli.test('iac-kubernetes/multi-file.yaml', { + iac: true, + json: true, + }); + t.fail('should have thrown'); + } catch (error) { + testableObject = error; + } + const res: any = JSON.parse(testableObject.message); + iacTestJsonAssertions(t, res, null, false); + }, '`iac test multi-file.yaml --severity-threshold=low --json`': ( params, utils, @@ -154,5 +161,42 @@ export const IacK8sTests: AcceptanceTests = { params, utils, ) => async (t) => await iacTestJson(t, utils, params, 'high'), + + '`iac test multi-file.yaml --sarif - no issues`': (params, utils) => async ( + t, + ) => { + utils.chdirWorkspaces(); + let testableObject; + try { + await params.cli.test('iac-kubernetes/multi-file.yaml', { + iac: true, + sarif: true, + }); + t.fail('should have thrown'); + } catch (error) { + testableObject = error; + } + const res: any = JSON.parse(testableObject.message); + iacTestSarifAssertions(t, res, null, false); + }, + '`iac test multi-file.yaml --severity-threshold=low --sarif`': ( + params, + utils, + ) => async (t) => await iacTestSarif(t, utils, params, 'low'), + + '`iac test multi-file.yaml --severity-threshold=medium --sarif`': ( + params, + utils, + ) => async (t) => await iacTestSarif(t, utils, params, 'medium'), + + '`iac test multi-file.yaml --severity-threshold=high --sarif`': ( + params, + utils, + ) => async (t) => await iacTestSarif(t, utils, params, 'high'), + + '`iac test multi-file.yaml --severity-threshold=high --sarif --sarif-file-output=test.json`': ( + params, + utils, + ) => async (t) => await iacTestSarifFileOutput(t, utils, params, 'high'), }, }; diff --git a/test/acceptance/cli-test/cli-test.iac-k8s.utils.ts b/test/acceptance/cli-test/cli-test.iac-k8s.utils.ts index ea7b91fe68..ccb737c535 100644 --- a/test/acceptance/cli-test/cli-test.iac-k8s.utils.ts +++ b/test/acceptance/cli-test/cli-test.iac-k8s.utils.ts @@ -4,6 +4,7 @@ import { AnnotatedIacIssue, IacTestResponse, } from '../../../src/lib/snyk-test/iac-test-result'; +import { Log, Run, Result } from 'sarif'; export async function iacTestPrep( t, @@ -60,6 +61,52 @@ export async function iacTestJson(t, utils, params, severityThreshold) { iacTestJsonAssertions(t, results, expectedResults); } +export async function iacTestSarif(t, utils, params, severityThreshold) { + const testableObject = await iacTestPrep( + t, + utils, + params, + severityThreshold, + { severityThreshold, sarif: true }, + ); + const req = params.server.popRequest(); + t.is(req.query.severityThreshold, severityThreshold); + + const results = JSON.parse(testableObject.message); + const expectedResults = mapIacTestResult( + iacTestResponseFixturesByThreshold[severityThreshold], + ); + + iacTestSarifAssertions(t, results, expectedResults); +} + +export async function iacTestSarifFileOutput( + t, + utils, + params, + severityThreshold, +) { + const testableObject = await iacTestPrep( + t, + utils, + params, + severityThreshold, + { severityThreshold, sarif: true }, + ); + const req = params.server.popRequest(); + t.is(req.query.severityThreshold, severityThreshold); + + const results = JSON.parse(testableObject.message); + const sarifStringifiedResults = JSON.parse( + testableObject.sarifStringifiedResults, + ); + t.deepEqual( + results, + sarifStringifiedResults, + 'stdout and stringified sarif results are the same', + ); +} + export async function iacTest( t, utils, @@ -131,6 +178,63 @@ export function iacTestJsonAssertions( } } +function getDistinctIssueIds(infrastructureAsCodeIssues): string[] { + const issueIdsSet = new Set(); + infrastructureAsCodeIssues.forEach((issue) => { + issueIdsSet.add(issue.id); + }); + return [...new Set(issueIdsSet)]; +} + +export function iacTestSarifAssertions( + t, + results: Log, + expectedResults, + foundIssues = true, +) { + t.deepEqual(results.version, '2.1.0', 'version is ok'); + t.deepEqual(results.runs.length, 1, 'number of runs is ok'); + const run: Run = results.runs[0]; + t.deepEqual( + run.tool.driver.name, + 'Snyk Infrastructure as Code', + 'tool name is ok', + ); + if (!foundIssues) { + t.deepEqual(run.tool.driver.rules!.length, 0, 'number of rules is ok'); + t.deepEqual(run.results!.length, 0, 'number of issues is ok'); + + return; + } + + const distictIssueIds = getDistinctIssueIds( + expectedResults.infrastructureAsCodeIssues, + ); + t.deepEqual( + run.tool.driver.rules!.length, + distictIssueIds.length, + 'number of rules is ok', + ); + t.deepEqual( + run.results!.length, + expectedResults.infrastructureAsCodeIssues.length, + 'number of issues is ok', + ); + for (let i = 0; i < run.results!.length; i++) { + const sarifIssue: Result = run.results![i]; + const expectedIssue = expectedResults.infrastructureAsCodeIssues[i]; + t.deepEqual(sarifIssue.ruleId, expectedIssue.id, 'issue id is ok'); + + const messageText = `This line contains a potential ${expectedIssue.severity} severity misconfiguration affacting the Kubernetes ${expectedIssue.subType}`; + t.deepEqual(sarifIssue.message.text, messageText, 'issue message is ok'); + t.deepEqual( + sarifIssue.locations![0].physicalLocation!.region!.startLine, + expectedIssue.lineNumber, + 'issue message is ok', + ); + } +} + function generateDummyIssue(severity): AnnotatedIacIssue { return { id: 'SNYK-CC-K8S-1', @@ -150,6 +254,7 @@ function generateDummyIssue(severity): AnnotatedIacIssue { type: 'k8s', subType: 'Deployment', path: [], + lineNumber: 1, }; } diff --git a/test/acceptance/fixtures/docker/sarif-container-result.json b/test/acceptance/fixtures/docker/sarif-container-result.json new file mode 100644 index 0000000000..60910425a8 --- /dev/null +++ b/test/acceptance/fixtures/docker/sarif-container-result.json @@ -0,0 +1,59 @@ +{ + "version": "2.1.0", + "runs": [ + { + "tool": { + "driver": { + "name": "Snyk Container", + "rules": [ + { + "id": "SNYK-LINUX-BZIP2-106947", + "shortDescription": { + "text": "low severity Denial of Service (DoS) vulnerability in bzip2" + }, + "fullDescription": { + "text": "(CVE-2016-3189) bzip2/libbz2-1.0@1.0.6-8.1" + }, + "help": { + "text": "", + "markdown": "## Overview\nUse-after-free vulnerability in bzip2recover in bzip2 1.0.6 allows remote attackers to cause a denial of service (crash) via a crafted bzip2 file, related to block ends set to before the start of the block.\n\n## References\n- [GENTOO](https://security.gentoo.org/glsa/201708-08)\n- [CONFIRM](https://bugzilla.redhat.com/show_bug.cgi?id=1319648)\n- [SECTRACK](http://www.securitytracker.com/id/1036132)\n- [BID](http://www.securityfocus.com/bid/91297)\n- [CONFIRM](http://www.oracle.com/technetwork/topics/security/bulletinjul2016-3090568.html)\n- [MLIST](http://www.openwall.com/lists/oss-security/2016/06/20/1)\n" + }, + "defaultConfiguration": { "level": "warning" }, + "properties": { "tags": ["security"] } + } + ] + } + }, + "results": [ + { + "ruleId": "SNYK-LINUX-BZIP2-106947", + "message": { + "text": "This file introduces a vulnerable bzip2 package with a low severity vulnerability." + }, + "locations": [ + { + "physicalLocation": { + "artifactLocation": {}, + "region": { "startLine": 1 } + } + } + ] + }, + { + "ruleId": "SNYK-LINUX-BZIP2-106947", + "message": { + "text": "This file introduces a vulnerable bzip2 package with a low severity vulnerability." + }, + "locations": [ + { + "physicalLocation": { + "artifactLocation": {}, + "region": { "startLine": 1 } + } + } + ] + } + ] + } + ] +}