TEST ONLY sample refactor of validation logic

src/cmd/validate/test-validate.go

@@ -0,0 +1,113 @@
+package validate
+
+import (
+	"os"
+
+	"github.com/spf13/cobra"
+
+	"github.com/defenseunicorns/lula/src/cmd/common"
+	"github.com/defenseunicorns/lula/src/pkg/message"
+	"github.com/defenseunicorns/lula/src/pkg/validation"
+)
+
+// Take input file, create a new producer and consumer?
+// If doing an compdef with several targets.. do that loop here?
+
+func TestValidateCommand() *cobra.Command {
+	v := common.GetViper()
+
+	var (
+		outputFile          string
+		inputFile           string
+		target              string
+		setOpts             []string
+		simple              bool
+		silent              bool
+		confirmExecution    bool
+		runNonInteractively bool
+		saveResources       bool
+	)
+
+	cmd := &cobra.Command{
+		Use:     "test-validate",
+		Aliases: []string{"tv"},
+		Short:   "test-validate <anything>",
+		Long:    "Lula Validation of <anything>",
+		Example: validateHelp,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			consumerType := "assessment-results" // Default consumer type
+			if simple {
+				consumerType = "simple"
+			}
+
+			// Get the inputs, set up the producer/consumer
+			inputFileBytes, err := os.ReadFile(inputFile)
+			if err != nil {
+				return err
+			}
+
+			if outputFile != "" {
+				outputFile = getDefaultOutputFile(inputFile)
+			}
+
+			producer, err := validation.ResolveProducer(inputFileBytes, inputFile, target)
+			if err != nil {
+				return err
+			}
+
+			consumer, err := validation.ResolveConsumer(consumerType, outputFile)
+			if err != nil {
+				return err
+			}
+
+			// Set up the validation
+			opts := []validation.Option{
+				validation.WithAllowExecution(confirmExecution, runNonInteractively),
+				// Other options...
+			}
+
+			validator, err := validation.New(producer, consumer, opts...)
+			if err != nil {
+				return err
+			}
+
+			// Get stats
+			if !silent {
+				numReqts, numVals, numExeVals := validator.GetStats()
+				message.Title("\n🔍 Collecting Requirements and Validations for: ", inputFile)
+				message.Infof("%d Requirements: ", numReqts)
+				message.Infof("%d Validations: ", numVals)
+				if numExeVals > 0 {
+					message.Warnf("%d Executable Validations: ", numExeVals)
+				}
+
+			}
+
+			// TODO: Request confirmation for execution, if needed
+			runExecutableValidations := true
+
+			err = validator.ExecuteValidations(cmd.Context(), runExecutableValidations)
+			if err != nil {
+				return err
+			}
+
+			return validator.YieldResults()
+		},
+	}
+
+	cmd.Flags().StringVarP(&outputFile, "output-file", "o", "", "the path to write assessment results. Creates a new file or appends to existing files")
+	cmd.Flags().StringVarP(&inputFile, "input-file", "f", "", "the path to the target OSCAL component definition")
+	err := cmd.MarkFlagRequired("input-file")
+	if err != nil {
+		message.Fatal(err, "error initializing upgrade command flags")
+	}
+	cmd.Flags().StringVarP(&target, "target", "t", v.GetString(common.VTarget), "the specific control implementations or framework to validate against")
+	cmd.Flags().BoolVar(&simple, "simple", false, "simple output")
+	cmd.Flags().BoolVar(&silent, "silent", false, "if set, no output will be printed")
+	cmd.Flags().BoolVar(&confirmExecution, "confirm-execution", false, "confirm execution scripts run as part of the validation")
+	cmd.Flags().BoolVar(&runNonInteractively, "non-interactive", false, "run the command non-interactively")
+	cmd.Flags().BoolVar(&saveResources, "save-resources", false, "saves the resources to 'resources' directory at assessment-results level")
+	cmd.Flags().StringSliceVarP(&setOpts, "set", "s", []string{}, "set a value in the template data")
+
+	return cmd
+}

src/pkg/common/oscal/assessment-results.go

@@ -24,6 +24,55 @@ type EvalResult struct {
 	Latest    *oscalTypes.Result
 }
 
+type AssessmentResults struct {
+	Model *oscalTypes.AssessmentResults
+}
+
+func NewAssessmentResults2() *AssessmentResults {
+	var ar AssessmentResults
+	ar.Model = nil
+	return &ar
+}
+
+func (a *AssessmentResults) NewModel(data []byte) error {
+	var oscalModels oscalTypes.OscalModels
+
+	err := multiModelValidate(data)
+	if err != nil {
+		return err
+	}
+
+	err = yaml.Unmarshal(data, &oscalModels)
+	if err != nil {
+		return err
+	}
+
+	a.Model = oscalModels.AssessmentResults
+	if a.Model == nil {
+		return fmt.Errorf("unable to find assessment results model")
+	}
+
+	return nil
+}
+
+func (*AssessmentResults) GetType() string {
+	return "assessment-results"
+}
+
+func (a *AssessmentResults) GetCompleteModel() *oscalTypes.OscalModels {
+	return &oscalTypes.OscalModels{
+		AssessmentResults: a.Model,
+	}
+}
+
+func (a *AssessmentResults) MakeDeterministic() error {
+	return nil
+}
+
+func (a *AssessmentResults) HandleExisting(path string) error {
+	return nil
+}
+
 // NewAssessmentResults creates a new assessment results object from the given data.
 func NewAssessmentResults(data []byte) (*oscalTypes.AssessmentResults, error) {
 	var oscalModels oscalTypes.OscalModels

src/pkg/common/oscal/component.go

@@ -30,6 +30,57 @@ type parameter struct {
 	Select *selection
 }
 
+type ComponentDefinition struct {
+	Model *oscalTypes.ComponentDefinition
+}
+
+func NewComponentDefinition() *ComponentDefinition {
+	var compdef ComponentDefinition
+	compdef.Model = nil
+	return &compdef
+}
+
+// Create a new ComponentDefinition model
+func (c *ComponentDefinition) NewModel(data []byte) error {
+
+	var oscalModels oscalTypes.OscalModels
+
+	err := multiModelValidate(data)
+	if err != nil {
+		return err
+	}
+
+	err = yaml.Unmarshal(data, &oscalModels)
+	if err != nil {
+		return err
+	}
+
+	c.Model = oscalModels.ComponentDefinition
+	if c.Model == nil {
+		return fmt.Errorf("unable to find component definition model")
+	}
+
+	return nil
+}
+
+func (*ComponentDefinition) GetType() string {
+	return "component-definition"
+}
+
+func (c *ComponentDefinition) GetCompleteModel() *oscalTypes.OscalModels {
+	return &oscalTypes.OscalModels{
+		ComponentDefinition: c.Model,
+	}
+}
+
+func (c *ComponentDefinition) ImportComponentDefinitions() error {
+	// TODO: get all imported component definitions and re-write any relative links?
+
+	return nil
+}
+
+// TODO: Add other interface methods(?)
+
 // NewOscalComponentDefinition consumes a byte array and returns a new single OscalComponentDefinitionModel object
 // Standard use is to read a file from the filesystem and pass the []byte to this function
 func NewOscalComponentDefinition(data []byte) (componentDefinition *oscalTypes.ComponentDefinition, err error) {

src/pkg/validation/README.md

@@ -0,0 +1,24 @@
+# Validation
+
+Logic to extract, execute, and evaluate Lula validations on a specified environment against targeted requirements.
+
+## Components
+
+### Producer
+
+The producer is responsible for providing the structure which stores Lula validations and associated requirements. Current producers are:
+- OSCAL Componet Definition
+- Simple (Lula Validation .yaml file)
+
+### Consumer
+
+The consumer is responsible for evaluating the validations and requirements from the producer and providing the results in the custom format. Current consumers are:
+- OSCAL Assessment Results
+- Simple (direct pass/fail from requirements)
+
+### Requirement
+
+The requirement is responsible for providing the structure which stores producer-generated requirements along with any associated validations. Current requirements are:
+- Component Definition Requirements
+- Simple (simple requirement to track aggregated validation pass/fail)
+

src/pkg/validation/consumer.go

@@ -0,0 +1,126 @@
+package validation
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/defenseunicorns/lula/src/pkg/common/oscal"
+	"github.com/defenseunicorns/lula/src/pkg/message"
+)
+
+// ResultConsumer is the interface that must be implemented by any consumer of the validation
+// store and results. It is responsible for evaluating the results and generating the output
+// speific to the consumer.
+type ResultConsumer interface {
+	// Evaluate Results are the custom implementation for the consumer, which should take the
+	// requirements, as specified by the producer, plus the data in the validation store
+	// and evaluate them + generate the output
+	EvaluateResults(store *ValidationStore) error
+
+	// Generate Output is the custom implementation for the consumer that should create
+	// a custom output
+	GenerateOutput() error
+}
+
+// AssessmentResultsConsumer is an implementation of the ResultConsumer interface
+// This consumer is responsible for generating an OSCAL Assessment Results model
+type AssessmentResultsConsumer struct {
+	assessmentResults *oscal.AssessmentResults
+	path              string
+}
+
+func NewAssessmentResultsConsumer(path string) *AssessmentResultsConsumer {
+	// Get asssessment results from file
+	data, err := os.ReadFile(path)
+	if err != nil {
+		return nil
+	}
+	ar := oscal.NewAssessmentResults2()
+
+	// Update the assessment results model if data is not nil
+	if len(data) != 0 {
+		err = ar.NewModel(data)
+		if err != nil {
+			return nil
+		}
+	}
+
+	return &AssessmentResultsConsumer{
+		assessmentResults: ar,
+		path:              path,
+	}
+}
+
+func (c *AssessmentResultsConsumer) EvaluateResults(store *ValidationStore) error {
+	// Update the oscal.AssessmentResults with the results from the store
+	// each requirement should be a finding
+	// each validation in the requirement should be an observation
+
+	// Create oscal results -> generate assessment results model (GenerateAssessmentResults)
+
+	// If the existing assessment results are nil (c.assessmentResults == nil), set them
+
+	// If they are populated, merge the results from the store into the existing assessment results
+
+	return nil
+}
+
+func (c *AssessmentResultsConsumer) GenerateOutput() error {
+	// Maybe should this consumer just create the results and then run a generate function
+	// to create the assessment results model? I feel like if this is from an assessment plan
+	// vs. a component definition, the assesment results model will be different... could/should
+	// this be handled prior to the consumer being created?
+
+	return oscal.WriteOscalModelNew(c.path, c.assessmentResults)
+}
+
+// SimpleConsumer is an implementation of the ResultConsumer interface
+// The consumer determines "Pass" is true iff all requirements are satisfied
+// Useful for quick determination of pass/fail status of the requirements
+type SimpleConsumer struct {
+	pass bool
+	msg  string
+}
+
+func NewSimpleConsumer() *SimpleConsumer {
+	return &SimpleConsumer{
+		pass: false,
+	}
+}
+
+func (c *SimpleConsumer) EvaluateResults(store *ValidationStore) error {
+	var output strings.Builder
+	requirements := store.GetRequirements()
+	passCount := 0
+
+	// Evaluate each requirement for pass/fail
+	for _, requirement := range requirements {
+		if requirement == nil {
+			continue
+		}
+
+		pass, msg := requirement.EvaluateSuccess()
+		if !pass {
+			passCount++
+		}
+		output.WriteString(msg)
+	}
+
+	if passCount > 0 && passCount == len(requirements) {
+		c.pass = true
+		return nil
+	}
+
+	c.msg = output.String()
+
+	return nil
+}
+
+func (c *SimpleConsumer) GenerateOutput() error {
+	if !c.pass {
+		return fmt.Errorf("requirements failed: %s", c.msg)
+	}
+	message.Infof("Requirements passed")
+	return nil
+}