Skip to content

Commit

Permalink
Merge pull request #1180 from equinor/master
Browse files Browse the repository at this point in the history
Added radix job option OverrideUseBuildCache (#1177)
  • Loading branch information
satr authored Aug 26, 2024
2 parents cfd7130 + 2c9726b commit faf640b
Show file tree
Hide file tree
Showing 20 changed files with 274 additions and 48 deletions.
4 changes: 2 additions & 2 deletions charts/radix-operator/Chart.yaml
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
apiVersion: v2
name: radix-operator
version: 1.37.9
appVersion: 1.57.17
version: 1.37.10
appVersion: 1.57.18
kubeVersion: ">=1.24.0"
description: Radix Operator
keywords:
Expand Down
9 changes: 9 additions & 0 deletions pipeline-runner/internal/commandbuilder/command.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,15 @@ func NewCommand(formattedCmd string, a ...any) *Command {

return &c
}

func (c *Command) AddArg(arg string) *Command {
if arg == "" {
return c
}
c.args = append(c.args, arg)
return c
}

func (c *Command) AddArgf(format string, a ...any) *Command {
if format == "" {
return c
Expand Down
6 changes: 5 additions & 1 deletion pipeline-runner/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@ import (
"github.com/spf13/cobra"
)

var overrideUseBuildCache model.BoolPtr

// Requirements to run, pipeline must have:
// - access to create Jobs in "app" namespace it runs under
// - access to create RD in all namespaces
Expand Down Expand Up @@ -119,6 +121,7 @@ func setPipelineArgsFromArguments(cmd *cobra.Command, pipelineArgs *model.Pipeli
cmd.Flags().StringVar(&pipelineArgs.Builder.ResourcesRequestsMemory, defaults.OperatorAppBuilderResourcesRequestsMemoryEnvironmentVariable, "500M", "Image builder resource requests memory")
var useCache string
cmd.Flags().StringVar(&useCache, defaults.RadixUseCacheEnvironmentVariable, "0", "Use cache")
cmd.Flags().Var(&overrideUseBuildCache, defaults.RadixOverrideUseBuildCacheVariable, "Optional. Overrides configured or default useBuildCache option. It is applicable when the useBuildKit option is set as true.")
var pushImage string
cmd.Flags().StringVar(&pushImage, defaults.RadixPushImageEnvironmentVariable, "0", "Push docker image to a repository")
var debug string
Expand All @@ -145,8 +148,9 @@ func setPipelineArgsFromArguments(cmd *cobra.Command, pipelineArgs *model.Pipeli
pipelineArgs.PushImage, _ = strconv.ParseBool(pushImage)
pipelineArgs.PushImage = pipelineArgs.PipelineType == string(v1.BuildDeploy) || pipelineArgs.PushImage // build and deploy require push
pipelineArgs.UseCache, _ = strconv.ParseBool(useCache)
pipelineArgs.OverrideUseBuildCache = overrideUseBuildCache.Get()
pipelineArgs.Debug, _ = strconv.ParseBool(debug)
if pipelineArgs.ImageTagNames != nil && len(pipelineArgs.ImageTagNames) > 0 {
if len(pipelineArgs.ImageTagNames) > 0 {
log.Info().Msg("Image tag names provided:")
for componentName, imageTagName := range pipelineArgs.ImageTagNames {
log.Info().Msgf("- %s:%s", componentName, imageTagName)
Expand Down
36 changes: 36 additions & 0 deletions pipeline-runner/model/bool_ptr.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
package model

import "strconv"

// BoolPtr Nillable bool type
type BoolPtr struct {
value *bool
}

// String returns the string representation of the nillable boolean
func (b *BoolPtr) String() string {
if b.value == nil {
return "nil"
}
return strconv.FormatBool(*b.value)
}

// Set a new value
func (b *BoolPtr) Set(s string) error {
v, err := strconv.ParseBool(s)
if err != nil {
return err
}
b.value = &v
return nil
}

// Get returns the value of the nillable boolean
func (b *BoolPtr) Get() *bool {
return b.value
}

// Type returns the type of the nillable boolean
func (b *BoolPtr) Type() string {
return "bool"
}
18 changes: 10 additions & 8 deletions pipeline-runner/model/pipelineInfo.go
Original file line number Diff line number Diff line change
Expand Up @@ -61,14 +61,16 @@ type PipelineArguments struct {
JobName string
Branch string
// CommitID is sent from GitHub webhook. not to be confused with PipelineInfo.GitCommitHash
CommitID string
ImageTag string
UseCache bool
PushImage bool
DeploymentName string
FromEnvironment string
ToEnvironment string
ComponentsToDeploy []string
CommitID string
ImageTag string
UseCache bool
// OverrideUseBuildCache override default or configured build cache option
OverrideUseBuildCache *bool
PushImage bool
DeploymentName string
FromEnvironment string
ToEnvironment string
ComponentsToDeploy []string

RadixConfigFile string

Expand Down
7 changes: 5 additions & 2 deletions pipeline-runner/steps/build/build_acr.go
Original file line number Diff line number Diff line change
Expand Up @@ -313,6 +313,9 @@ func getContainerCommand(pipelineInfo *model.PipelineInfo, containerRegistry str
}
cacheImagePath := utils.GetImageCachePath(pipelineInfo.PipelineArguments.AppContainerRegistry, pipelineInfo.RadixApplication.Name)
useBuildCache := pipelineInfo.RadixApplication.Spec.Build.UseBuildCache == nil || *pipelineInfo.RadixApplication.Spec.Build.UseBuildCache
if pipelineInfo.PipelineArguments.OverrideUseBuildCache != nil {
useBuildCache = *pipelineInfo.PipelineArguments.OverrideUseBuildCache
}
cacheContainerRegistry := pipelineInfo.PipelineArguments.AppContainerRegistry // Store application cache in the App Registry
return getBuildahContainerCommand(containerRegistry, secretMountsArgsString, componentImage, clusterTypeImage, clusterNameImage, cacheContainerRegistry, cacheImagePath, useBuildCache, pipelineInfo.PipelineArguments.PushImage)
}
Expand Down Expand Up @@ -540,7 +543,7 @@ func getBuildahContainerCommand(containerImageRegistry, secretArgsString string,
AddArgf("--isolation=chroot").
AddArgf("--jobs 0").
AddArgf("--ulimit nofile=4096:4096").
AddArgf(secretArgsString).
AddArg(secretArgsString).
AddArgf("--file %s%s", context, componentImage.Dockerfile).
AddArgf(`--build-arg RADIX_GIT_COMMIT_HASH="${RADIX_GIT_COMMIT_HASH}"`).
AddArgf(`--build-arg RADIX_GIT_TAGS="${RADIX_GIT_TAGS}"`).
Expand All @@ -562,7 +565,7 @@ func getBuildahContainerCommand(containerImageRegistry, secretArgsString string,
AddArgf("--tag %s", clusterNameImageTag)
}

buildah.AddArgf(context)
buildah.AddArg(context)

if pushImage {
commandList.
Expand Down
149 changes: 148 additions & 1 deletion pipeline-runner/steps/build/build_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,10 @@ func (s *buildTestSuite) SetupTest() {
}

func (s *buildTestSuite) SetupSubTest() {
s.setupTest()
}

func (s *buildTestSuite) setupTest() {
s.kubeClient = kubefake.NewSimpleClientset()
s.radixClient = radixfake.NewSimpleClientset()
s.kedaClient = kedafake.NewSimpleClientset()
Expand All @@ -75,7 +79,6 @@ func (s *buildTestSuite) Test_BranchIsNotMapped_ShouldSkip() {
anyJobName = "any-job-name"
anyImageTag = "anytag"
anyCommitID = "4faca8595c5283a9d0f17a623b9255a0d9866a2e"
anyGitTags = "some tags go here"
anyBranch = "master"
anyEnvironment = "dev"
anyComponentName = "app"
Expand Down Expand Up @@ -1655,6 +1658,150 @@ func (s *buildTestSuite) Test_BuildJobSpec_BuildKit() {
s.ElementsMatch(job.Spec.Template.Spec.Containers[0].VolumeMounts, expectedVolumeMounts)
}

func (s *buildTestSuite) Test_BuildJobSpec_OverrideUseBuildCacheInBuildKit() {
appName, rjName, compName, sourceFolder, dockerFile, envName := "anyapp", "anyrj", "c1", "../path1/./../../path2", "anydockerfile", "dev"
prepareConfigMapName := "preparecm"
gitConfigMapName, gitHash, gitTags := "gitcm", "githash", "gittags"
type scenario struct {
name string
configUseBuildCache *bool
jobOverrideUseBuildCache *bool
expectedUseBuildCache bool
}
scenarios := []scenario{
{
name: "configured UseBuildCache true, not overridden",
configUseBuildCache: pointers.Ptr(true),
expectedUseBuildCache: true,
},
{
name: "configured UseBuildCache false, not overridden",
configUseBuildCache: pointers.Ptr(false),
expectedUseBuildCache: false,
},
{
name: "configured UseBuildCache true, overridden with false",
configUseBuildCache: pointers.Ptr(true),
jobOverrideUseBuildCache: pointers.Ptr(false),
expectedUseBuildCache: false,
},
{
name: "configured UseBuildCache false, overridden with true",
configUseBuildCache: pointers.Ptr(false),
jobOverrideUseBuildCache: pointers.Ptr(true),
expectedUseBuildCache: true,
},
{
name: "not configured UseBuildCache true, overridden with false",
jobOverrideUseBuildCache: pointers.Ptr(false),
expectedUseBuildCache: false,
},
{
name: "not configured UseBuildCache false, overridden with true",
jobOverrideUseBuildCache: pointers.Ptr(true),
expectedUseBuildCache: true,
},
}
for _, ts := range scenarios {
s.T().Run(ts.name, func(t *testing.T) {
s.setupTest()
rr := utils.ARadixRegistration().WithName(appName).BuildRR()
_, _ = s.radixClient.RadixV1().RadixRegistrations().Create(context.Background(), rr, metav1.CreateOptions{})
rj := utils.ARadixBuildDeployJob().WithJobName(rjName).WithAppName(appName).WithOverrideUseBuildCache(ts.jobOverrideUseBuildCache).BuildRJ()
_, _ = s.radixClient.RadixV1().RadixJobs(utils.GetAppNamespace(appName)).Create(context.Background(), rj, metav1.CreateOptions{})
ra := utils.NewRadixApplicationBuilder().
WithAppName(appName).
WithBuildKit(pointers.Ptr(true)).WithBuildCache(ts.configUseBuildCache).
WithEnvironment("dev", "main").
WithComponent(utils.NewApplicationComponentBuilder().WithPort("any", 8080).WithName(compName).WithDockerfileName(dockerFile).WithSourceFolder(sourceFolder)).
BuildRA()
s.Require().NoError(internaltest.CreatePreparePipelineConfigMapResponse(s.kubeClient, prepareConfigMapName, appName, ra, nil), "CreatePreparePipelineConfigMapResponse")
s.Require().NoError(internaltest.CreateGitInfoConfigMapResponse(s.kubeClient, gitConfigMapName, appName, gitHash, gitTags), "CreateGitInfoConfigMapResponse")
pipeline := model.PipelineInfo{
PipelineArguments: model.PipelineArguments{
PipelineType: "build-deploy",
Branch: "main",
JobName: rjName,
BuildKitImageBuilder: "anybuildkitimage:tag",
ImageTag: "anyimagetag",
ContainerRegistry: "anyregistry",
AppContainerRegistry: "anyappregistry",
Clustertype: "anyclustertype",
Clustername: "anyclustername",
GitCloneNsLookupImage: "any",
GitCloneGitImage: "any",
GitCloneBashImage: "any",
Builder: model.Builder{ResourcesLimitsMemory: "100M", ResourcesRequestsCPU: "50m", ResourcesRequestsMemory: "50M"},
OverrideUseBuildCache: ts.jobOverrideUseBuildCache,
},
RadixConfigMapName: prepareConfigMapName,
GitConfigMapName: gitConfigMapName,
}
jobWaiter := internalwait.NewMockJobCompletionWaiter(s.ctrl)
jobWaiter.EXPECT().Wait(gomock.Any()).Return(nil).Times(1)

applyStep := applyconfig.NewApplyConfigStep()
applyStep.Init(context.Background(), s.kubeClient, s.radixClient, s.kubeUtil, s.promClient, rr)
buildStep := NewBuildStep(jobWaiter)
buildStep.Init(context.Background(), s.kubeClient, s.radixClient, s.kubeUtil, s.promClient, rr)
s.Require().NoError(applyStep.Run(context.Background(), &pipeline))
s.Require().NoError(buildStep.Run(context.Background(), &pipeline))
jobs, _ := s.kubeClient.BatchV1().Jobs(utils.GetAppNamespace(appName)).List(context.Background(), metav1.ListOptions{})
s.Require().Len(jobs.Items, 1)
job := jobs.Items[0]
s.Require().Len(job.Spec.Template.Spec.Containers, 1)
s.Equal(pipeline.PipelineArguments.BuildKitImageBuilder, job.Spec.Template.Spec.Containers[0].Image)
expectedCommandElements := []string{
"mkdir /var/tmp && cp /radix-private-image-hubs/.dockerconfigjson /home/build/auth.json && ",
fmt.Sprintf("/usr/bin/buildah login --username ${BUILDAH_USERNAME} --password ${BUILDAH_PASSWORD} %s && ", pipeline.PipelineArguments.ContainerRegistry),
}
if ts.expectedUseBuildCache {
expectedCommandElements = append(expectedCommandElements, fmt.Sprintf("/usr/bin/buildah login --username ${BUILDAH_CACHE_USERNAME} --password ${BUILDAH_CACHE_PASSWORD} %s && ", pipeline.PipelineArguments.AppContainerRegistry))
}
expectedCommandElements = append(expectedCommandElements, []string{
"/usr/bin/buildah build --storage-driver=overlay --isolation=chroot --jobs 0 --ulimit nofile=4096:4096 ",
fmt.Sprintf("--file %s%s ", "/workspace/path2/", dockerFile),
"--build-arg RADIX_GIT_COMMIT_HASH=\"${RADIX_GIT_COMMIT_HASH}\" ",
"--build-arg RADIX_GIT_TAGS=\"${RADIX_GIT_TAGS}\" ",
"--build-arg BRANCH=\"${BRANCH}\" ",
"--build-arg TARGET_ENVIRONMENTS=\"${TARGET_ENVIRONMENTS}\" ",
}...)
if ts.expectedUseBuildCache {
expectedCommandElements = append(expectedCommandElements, "--layers ")
expectedCommandElements = append(expectedCommandElements, fmt.Sprintf("--cache-to=%s/%s/cache ", pipeline.PipelineArguments.AppContainerRegistry, appName))
expectedCommandElements = append(expectedCommandElements, fmt.Sprintf("--cache-from=%s/%s/cache ", pipeline.PipelineArguments.AppContainerRegistry, appName))
}
expectedCommandElements = append(expectedCommandElements, "/workspace/path2/")
expectedCommand := []string{"/bin/bash", "-c", strings.Join(expectedCommandElements, "")}
s.Equal(expectedCommand, job.Spec.Template.Spec.Containers[0].Command)
expectedEnv := []corev1.EnvVar{
{Name: "DOCKER_FILE_NAME", Value: dockerFile},
{Name: "DOCKER_REGISTRY", Value: pipeline.PipelineArguments.ContainerRegistry},
{Name: "IMAGE", Value: fmt.Sprintf("%s/%s-%s-%s:%s", pipeline.PipelineArguments.ContainerRegistry, appName, envName, compName, pipeline.PipelineArguments.ImageTag)},
{Name: "CONTEXT", Value: "/workspace/path2/"},
{Name: "PUSH", Value: ""},
{Name: "AZURE_CREDENTIALS", Value: "/radix-image-builder/.azure/sp_credentials.json"},
{Name: "SUBSCRIPTION_ID", Value: pipeline.PipelineArguments.SubscriptionId},
{Name: "CLUSTERTYPE_IMAGE", Value: fmt.Sprintf("%s/%s-%s-%s:%s-%s", pipeline.PipelineArguments.ContainerRegistry, appName, envName, compName, pipeline.PipelineArguments.Clustertype, pipeline.PipelineArguments.ImageTag)},
{Name: "CLUSTERNAME_IMAGE", Value: fmt.Sprintf("%s/%s-%s-%s:%s-%s", pipeline.PipelineArguments.ContainerRegistry, appName, envName, compName, pipeline.PipelineArguments.Clustername, pipeline.PipelineArguments.ImageTag)},
{Name: "REPOSITORY_NAME", Value: fmt.Sprintf("%s-%s-%s", appName, envName, compName)},
{Name: "CACHE", Value: "--no-cache"},
{Name: "RADIX_ZONE", Value: pipeline.PipelineArguments.RadixZone},
{Name: "BRANCH", Value: pipeline.PipelineArguments.Branch},
{Name: "TARGET_ENVIRONMENTS", Value: "dev"},
{Name: "RADIX_GIT_COMMIT_HASH", Value: gitHash},
{Name: "RADIX_GIT_TAGS", Value: gitTags},
{Name: "BUILDAH_USERNAME", ValueFrom: &corev1.EnvVarSource{SecretKeyRef: &corev1.SecretKeySelector{Key: "username", LocalObjectReference: corev1.LocalObjectReference{Name: defaults.AzureACRServicePrincipleBuildahSecretName}}}},
{Name: "BUILDAH_PASSWORD", ValueFrom: &corev1.EnvVarSource{SecretKeyRef: &corev1.SecretKeySelector{Key: "password", LocalObjectReference: corev1.LocalObjectReference{Name: defaults.AzureACRServicePrincipleBuildahSecretName}}}},
{Name: "BUILDAH_CACHE_USERNAME", ValueFrom: &corev1.EnvVarSource{SecretKeyRef: &corev1.SecretKeySelector{Key: "username", LocalObjectReference: corev1.LocalObjectReference{Name: defaults.AzureACRTokenPasswordAppRegistrySecretName}}}},
{Name: "BUILDAH_CACHE_PASSWORD", ValueFrom: &corev1.EnvVarSource{SecretKeyRef: &corev1.SecretKeySelector{Key: "password", LocalObjectReference: corev1.LocalObjectReference{Name: defaults.AzureACRTokenPasswordAppRegistrySecretName}}}},
{Name: "REGISTRY_AUTH_FILE", Value: "/home/build/auth.json"},
}
s.ElementsMatch(expectedEnv, job.Spec.Template.Spec.Containers[0].Env)
})
}
}

func (s *buildTestSuite) Test_BuildJobSpec_BuildKit_PushImage() {
appName, rjName, compName, sourceFolder, dockerFile, envName := "anyapp", "anyrj", "c1", "../path1/./../../path2", "anydockerfile", "dev"
prepareConfigMapName := "preparecm"
Expand Down
2 changes: 1 addition & 1 deletion pkg/apis/batch/kubejob.go
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ func (s *syncer) validatePayloadSecretReference(ctx context.Context, batchJob *r
if !radixlabels.GetRadixBatchDescendantsSelector(jobComponent.GetName()).Matches(labels.Set(payloadSecret.GetLabels())) {
return fmt.Errorf("secret %s, referenced in the job %s of the batch %s is not valid payload secret", batchJob.PayloadSecretRef.Name, batchJob.Name, s.radixBatch.GetName())
}
if payloadSecret.Data == nil || len(payloadSecret.Data) == 0 {
if len(payloadSecret.Data) == 0 {
return fmt.Errorf("payload secret %s, in the job %s of the batch %s is empty", batchJob.PayloadSecretRef.Name, batchJob.Name, s.radixBatch.GetName())
}
if _, ok := payloadSecret.Data[batchJob.PayloadSecretRef.Key]; !ok {
Expand Down
3 changes: 3 additions & 0 deletions pkg/apis/defaults/environment_variables.go
Original file line number Diff line number Diff line change
Expand Up @@ -135,6 +135,9 @@ const (
// RadixUseCacheEnvironmentVariable Use cache for the built component
RadixUseCacheEnvironmentVariable = "USE_CACHE"

// RadixOverrideUseBuildCacheVariable override default or configured build cache option
RadixOverrideUseBuildCacheVariable = "OVERRIDE_USE_BUILD_CACHE"

// RadixPipelineJobEnvironmentVariable Radix pipeline job name
RadixPipelineJobEnvironmentVariable = "JOB_NAME"

Expand Down
2 changes: 1 addition & 1 deletion pkg/apis/deployment/radixcommoncomponent.go
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ func getImagePath(componentName string, componentImage pipeline.DeployComponentI
}

func errorMissingExpectedDynamicImageTagName(componentName string) error {
return fmt.Errorf(fmt.Sprintf("component %s is missing an expected dynamic imageTagName for its image", componentName))
return fmt.Errorf("component %s is missing an expected dynamic imageTagName for its image", componentName)
}

func getImageTagName(componentImage pipeline.DeployComponentImage, componentImageTagName string, environmentSpecificConfig v1.RadixCommonEnvironmentConfig) string {
Expand Down
4 changes: 2 additions & 2 deletions pkg/apis/deployment/volumemount_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1306,7 +1306,7 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureKeyVaultResources(
for _, azureKeyVault := range scenario.azureKeyVaults {
spc, err := deployment.createAzureKeyVaultSecretProviderClassForRadixDeployment(context.Background(), namespace, appName, radixDeployComponent.GetName(), azureKeyVault)
if err != nil {
t.Logf(err.Error())
t.Log(err.Error())
} else {
t.Logf("created secret provider class %s", spc.Name)
}
Expand Down Expand Up @@ -1354,7 +1354,7 @@ func (suite *VolumeMountTestSuite) Test_CreateOrUpdateCsiAzureKeyVaultResources(
for _, azureKeyVault := range scenario.azureKeyVaults {
spc, err := deployment.createAzureKeyVaultSecretProviderClassForRadixDeployment(context.Background(), namespace, appName, radixDeployComponent.GetName(), azureKeyVault)
if err != nil {
t.Logf(err.Error())
t.Log(err.Error())
} else {
t.Logf("created secret provider class %s", spc.Name)
}
Expand Down
3 changes: 3 additions & 0 deletions pkg/apis/job/kubejob.go
Original file line number Diff line number Diff line change
Expand Up @@ -199,6 +199,9 @@ func (job *Job) getPipelineJobArguments(ctx context.Context, appName, jobName st
args = append(args, fmt.Sprintf("--%s=%s", defaults.RadixCommitIdEnvironmentVariable, jobSpec.Build.CommitID))
args = append(args, fmt.Sprintf("--%s=%s", defaults.RadixPushImageEnvironmentVariable, getPushImageTag(jobSpec.Build.PushImage)))
args = append(args, fmt.Sprintf("--%s=%s", defaults.RadixUseCacheEnvironmentVariable, useImageBuilderCache))
if jobSpec.Build.OverrideUseBuildCache != nil {
args = append(args, fmt.Sprintf("--%s=%v", defaults.RadixOverrideUseBuildCacheVariable, *jobSpec.Build.OverrideUseBuildCache))
}
case radixv1.Promote:
args = append(args, fmt.Sprintf("--%s=%s", defaults.RadixPromoteDeploymentEnvironmentVariable, jobSpec.Promote.DeploymentName))
args = append(args, fmt.Sprintf("--%s=%s", defaults.RadixPromoteFromEnvironmentEnvironmentVariable, jobSpec.Promote.FromEnvironment))
Expand Down
5 changes: 5 additions & 0 deletions pkg/apis/radix/v1/radixjobtypes.go
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,11 @@ type RadixBuildSpec struct {
//
// required: false
PushImage bool `json:"pushImage" yaml:"pushImage"`

// OverrideUseBuildCache override default or configured build cache option
//
// required: false
OverrideUseBuildCache *bool `json:"overrideUseBuildCache,omitempty" yaml:"overrideUseBuildCache,omitempty"`
}

// RadixPromoteSpec is the spec for a promote job
Expand Down
Loading

0 comments on commit faf640b

Please sign in to comment.