chore: add fuzz tests and daily fuzz workflow (#832)

* chore: add fuzz tests and daily fuzz workflow

* i promise we'll write ok tests linter, please leave me alone

* try more-right syntax

* the cake (linter_settings) was a lie

* there's more to say about testing, but i don't know what it is yet

* fix chron syntax

.github/workflows/test-fuzz.yaml

@@ -0,0 +1,25 @@
+name: Fuzz Testing
+
+permissions:
+  contents: read
+
+on:
+  schedule:
+   - cron: '0 0 * * *' 
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+
+    - name: Setup golang
+      uses: ./.github/actions/golang
+
+    - name: Display Go version
+      run: go version
+
+    - name: Fuzz Tests
+      run: | 
+        make test-fuzz

.golangci.yaml

@@ -4,3 +4,11 @@ linters:
     - gofmt
     - goimports
     - prealloc
+
+issues:
+  exclude-rules:
+    # Exclude some linters from running on tests files.
+    - path: _test\.go
+      linters:
+        - errcheck
+

CONTRIBUTING.md

@@ -50,14 +50,18 @@ Please follow our [Code of Conduct](CODE_OF_CONDUCT.md) to maintain a respectful
 - PRs must pass CI checks.
 - All commits must be signed.
 - PRs should have a related issue, except for trivial fixes.
+- Include tests that fail without your code, and pass with it.
+- Update the code comments, documentation, and examples: whatever is affected by your contribution.
 
-## Running Tests
+## Testing
 
 ### Automated Tests
 
-Automated tests will begin based on the paths you have edited in your Pull Request.
+Automated tests will begin based on the paths you have edited in your Pull Request. 
 ⚠️ NOTE: If you are an external third-party contributor, the pipelines won't run until a [CODEOWNER](CODEOWNERS) approves the pipeline run.
 
+The exception to the automated tests are fuzzing tests, which run nightly.
+
 ### Run Tests Locally
 
 - Run E2E tests: `make test`

Makefile

@@ -25,6 +25,7 @@ TESTFLAGS   := -race -v
 LDFLAGS     := -w -s -X 'github.com/defenseunicorns/lula/src/config.CLIVersion=$(CLI_VERSION)'
 GOFLAGS     :=
 CGO_ENABLED ?= 0
+FUZZTIME := 10s
 
 # Allows us to set VERSION from the command line.
 # Otherwise, if BINARY_VERSION is not set, use the current git tag.
@@ -76,6 +77,10 @@ test-e2e:
 test-cmd: 
 	cd src/cmd && go clean -testcache && go test $(GOFLAGS) -run $(TESTS) $(PKG) $(TESTFLAGS)
 
+.PHONY: test-fuzz
+test-fuzz:
+	cd src && $(SHELL) ../build/scripts/fuzz.sh $(FUZZTIME)
+
 .PHONY: install
 install: ## Install binary to $INSTALL_PATH.
 	@install "$(BINDIR)/$(BINNAME)" "$(INSTALL_PATH)/$(BINNAME)"

build/scripts/fuzz.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+set -e
+
+fuzzTime=${1:-10s}
+
+files=$(grep -r --include='**_test.go' --files-with-matches 'func Fuzz' .)
+
+for file in ${files}
+do
+	funcs=$(grep -o 'func Fuzz\w*' $file | sed 's/func //')
+	for func in ${funcs}
+	do
+		echo "Fuzzing $func in $file"
+		parentDir=$(dirname $file)
+        # Add $ to the end of the func name, since go fuzzing gets stressed out by ambiguiity
+        # ie "testing: will not fuzz, -fuzz matches more than one fuzz test: [FuzzCompareControls FuzzCompareControlsInt]" 
+		go test $parentDir -run=$func -fuzz=$func\$ -fuzztime=${fuzzTime}
+	done
+done

src/cmd/dev/validate_test.go

@@ -0,0 +1,36 @@
+package dev
+
+import (
+	"context"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func FuzzRunSingleValidation(f *testing.F) {
+	// including a fully populated valid-component.yaml to exercise a fuzzy version of passing valid-but-wrong-for-this-command oscal data.
+	for _, tc := range []string{"../../test/unit/common/oscal/valid-component.yaml", "../../test/unit/common/validation/multi.validation.yaml"} {
+		bytes, err := os.ReadFile(tc)
+		require.NoError(f, err)
+
+		f.Add(bytes)
+	}
+
+	f.Fuzz(func(t *testing.T, a []byte) {
+		RunSingleValidation(context.Background(), a)
+	})
+}
+
+func FuzzDevValidate(f *testing.F) {
+	bytes, err := os.ReadFile("../../test/unit/common/validation/multi.validation.yaml")
+	require.NoError(f, err)
+
+	drs, err := os.ReadFile("../../test/unit/common/resources/valid-resources.json")
+	require.NoError(f, err)
+	f.Add(bytes, drs)
+
+	f.Fuzz(func(t *testing.T, a []byte, b []byte) {
+		DevValidate(context.Background(), a, b, false, nil)
+	})
+}

src/pkg/common/common.go

@@ -12,15 +12,16 @@ import (
 	"strings"
 
 	oscalTypes "github.com/defenseunicorns/go-oscal/src/types/oscal-1-1-3"
+	goversion "github.com/hashicorp/go-version"
+	"k8s.io/apimachinery/pkg/util/yaml"
+
 	"github.com/defenseunicorns/lula/src/pkg/domains/api"
 	"github.com/defenseunicorns/lula/src/pkg/domains/files"
 	kube "github.com/defenseunicorns/lula/src/pkg/domains/kubernetes"
 	"github.com/defenseunicorns/lula/src/pkg/message"
 	"github.com/defenseunicorns/lula/src/pkg/providers/kyverno"
 	"github.com/defenseunicorns/lula/src/pkg/providers/opa"
 	"github.com/defenseunicorns/lula/src/types"
-	goversion "github.com/hashicorp/go-version"
-	"k8s.io/apimachinery/pkg/util/yaml"
 )
 
 const (

src/pkg/common/common_test.go

@@ -7,13 +7,15 @@ import (
 	"strings"
 	"testing"
 
+	kjson "github.com/kyverno/kyverno-json/pkg/apis/policy/v1alpha1"
+	"github.com/stretchr/testify/require"
+	"sigs.k8s.io/yaml"
+
 	"github.com/defenseunicorns/lula/src/pkg/common"
 	"github.com/defenseunicorns/lula/src/pkg/domains/api"
 	kube "github.com/defenseunicorns/lula/src/pkg/domains/kubernetes"
 	"github.com/defenseunicorns/lula/src/pkg/providers/kyverno"
 	"github.com/defenseunicorns/lula/src/pkg/providers/opa"
-	kjson "github.com/kyverno/kyverno-json/pkg/apis/policy/v1alpha1"
-	"sigs.k8s.io/yaml"
 )
 
 const multiValidationPath = "../../test/e2e/scenarios/remote-validations/multi-validations.yaml"
@@ -526,3 +528,24 @@ func TestIsVersionValid(t *testing.T) {
 		})
 	}
 }
+
+func FuzzPrefix(f *testing.F) {
+	f.Add("uuid")
+	f.Add("149f0049-7a3c-4e4d-8431-bec3a55f31d9")
+
+	f.Fuzz(func(t *testing.T, a string) {
+		withPrefix := common.AddIdPrefix(a)
+		removed := common.TrimIdPrefix(withPrefix)
+		require.Equal(t, a, removed)
+	})
+}
+
+func FuzzReadValidationsFromYaml(f *testing.F) {
+	bytes, err := os.ReadFile(multiValidationPath)
+	require.NoError(f, err)
+	f.Add(bytes)
+
+	f.Fuzz(func(t *testing.T, a []byte) {
+		common.ReadValidationsFromYaml(a)
+	})
+}

src/pkg/common/oscal/assessment-results_test.go

@@ -1,6 +1,7 @@
 package oscal_test
 
 import (
+	"os"
 	"slices"
 	"testing"
 	"time"
@@ -662,5 +663,19 @@ func TestGetObservationByUuid(t *testing.T) {
 		assert.Nil(t, observation)
 		require.ErrorContains(t, err, "observation with uuid invalid-uuid not found")
 	})
+}
+
+func FuzzNewAssessmentResults(f *testing.F) {
+	for _, tc := range []string{"../../../test/unit/common/oscal/valid-assessment-results-multi.yaml",
+		"../../../test/unit/common/oscal/valid-assessment-results-with-resources.yaml"} {
 
+		b, err := os.ReadFile(tc)
+		require.NoError(f, err)
+		f.Add(b)
+	}
+
+	f.Fuzz(func(t *testing.T, a []byte) {
+		// errors are ok, just watching for panics.
+		oscal.NewAssessmentResults(a)
+	})
 }

src/pkg/common/oscal/common_test.go

@@ -6,6 +6,7 @@ import (
 	"testing"
 
 	oscalTypes "github.com/defenseunicorns/go-oscal/src/types/oscal-1-1-3"
+
 	"github.com/defenseunicorns/lula/src/pkg/common/oscal"
 )
 
@@ -290,3 +291,23 @@ func TestSortControls(t *testing.T) {
 		})
 	}
 }
+
+func FuzzCompareControls(f *testing.F) {
+	f.Add("apple", "anotherword")
+	f.Add("AC-1", "ac-1")
+	f.Add("ac-4.4", "ac-4.21")
+
+	f.Fuzz(func(t *testing.T, a string, b string) {
+		oscal.CompareControls(a, b)
+	})
+}
+
+func FuzzCompareControlsInt(f *testing.F) {
+	f.Add("apple", "anotherword")
+	f.Add("AC-1", "ac-1")
+	f.Add("ac-4.4", "ac-4.21")
+
+	f.Fuzz(func(t *testing.T, a string, b string) {
+		oscal.CompareControlsInt(a, b)
+	})
+}

src/pkg/common/oscal/multi-validate_test.go

@@ -3,6 +3,8 @@ package oscal
 import (
 	"os"
 	"testing"
+
+	"github.com/stretchr/testify/require"
 )
 
 func TestMultiValidate(t *testing.T) {
@@ -18,3 +20,13 @@ func TestMultiValidate(t *testing.T) {
 		}
 	})
 }
+
+func FuzzMultiModelValidate(f *testing.F) {
+	multiModelData, err := os.ReadFile("../../../test/unit/common/oscal/multi-model.yaml")
+	require.NoError(f, err)
+	f.Add(multiModelData)
+
+	f.Fuzz(func(t *testing.T, a []byte) {
+		multiModelValidate(a)
+	})
+}