diff --git a/.env b/.env index 73ed245..f39fbfd 100644 --- a/.env +++ b/.env @@ -4,5 +4,5 @@ ZOT_URL="127.0.0.1:8585" TOKEN="" ENV=dev USE_UNSECURE=true -GROUP_NAME=test-satellite-group -STATE_ARTIFACT_NAME=state-artifact +GROUP_NAME=satellite-test-group-state +STATE_ARTIFACT_NAME=state diff --git a/config.toml b/config.toml index bd7f9d5..30614fc 100644 --- a/config.toml +++ b/config.toml @@ -7,7 +7,7 @@ own_registry_port = "8585" # URL of remote registry OR local file path # url_or_file = "https://demo.goharbor.io/v2/myproject/album-server" -url_or_file = "https://demo.goharbor.io" +url_or_file = "https://registry.bupd.xyz" ## for testing for local file # url_or_file = "./image-list/images.json" diff --git a/image-list/images.json b/image-list/images.json index cd8ac06..86cf321 100644 --- a/image-list/images.json +++ b/image-list/images.json @@ -1,15 +1,25 @@ { - "registry": "http://demo.goharbor.io/", - "artifacts": [ - { - "repository": "satellite-test-alpine/alpine", - "tag": "latest", - "hash": "sha256:9cee2b38" + "registry": "Satellite", + "artifacts": [ + { + "repository": "satellite-test-group-state/alpine", + "tag": [ + "latest" + ], + "labels": null, + "type": "IMAGE", + "digest": "sha256:9cee2b382fe2412cd77d5d437d15a93da8de373813621f2e4d406e3df0cf0e7c", + "deleted": false }, - { - "repository": "satellite-test-postgres/postgres", - "tag": "latest", - "hash": "sha256:9cee2b38" - } + { + "repository": "satellite-test-group-state/postgres", + "tag": [ + "latest" + ], + "labels": null, + "type": "IMAGE", + "digest": "sha256:dde924f70bc972261013327c480adf402ea71487b5750e40569a0b74fa90c74a", + "deleted": false + } ] } diff --git a/internal/state/fetcher.go b/internal/state/fetcher.go index 60a22f9..d028087 100644 --- a/internal/state/fetcher.go +++ b/internal/state/fetcher.go @@ -1,22 +1,20 @@ package state import ( - "context" + "archive/tar" + "bytes" "encoding/json" "fmt" + "io" "os" - "path/filepath" "strings" "container-registry.com/harbor-satellite/internal/config" - "oras.land/oras-go/v2" - "oras.land/oras-go/v2/content/file" - "oras.land/oras-go/v2/registry/remote" - "oras.land/oras-go/v2/registry/remote/auth" - "oras.land/oras-go/v2/registry/remote/retry" + "container-registry.com/harbor-satellite/internal/utils" + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/crane" ) - type StateFetcher interface { // Fetches the state artifact from the registry FetchStateArtifact() (StateReader, error) @@ -75,73 +73,68 @@ func (f *FileStateArtifactFetcher) FetchStateArtifact() (StateReader, error) { } func (f *URLStateFetcher) FetchStateArtifact() (StateReader, error) { - cwd, err := os.Getwd() - if err != nil { - return nil, fmt.Errorf("failed to get current working directory: %v", err) - } - // Creating a file store in the current working directory will be deleted later after reading the state artifact - fs, err := file.New(fmt.Sprintf("%s/state-artifact", cwd)) - if err != nil { - return nil, fmt.Errorf("failed to create file store: %v", err) + + auth := authn.FromConfig(authn.AuthConfig{ + Username: config.GetHarborUsername(), + Password: config.GetHarborPassword(), + }) + + options := []crane.Option{crane.WithAuth(auth)} + if config.UseUnsecure() { + options = append(options, crane.Insecure) } - defer fs.Close() - ctx := context.Background() + sourceRegistry := utils.FormatRegistryUrl(config.GetRemoteRegistryURL()) + group := config.GetGroupName() + stateArtifactName := config.GetStateArtifactName() + var tag string = "latest" + fmt.Printf("Pulling state artifact from %s/%s/%s:%s\n", sourceRegistry, group, stateArtifactName, tag) + fmt.Printf("Auth: %v\n", auth) - repo, err := remote.NewRepository(fmt.Sprintf("%s/%s/%s", f.url, f.group_name, f.state_artifact_name)) + // pull the state artifact from the central registry + img, err := crane.Pull(fmt.Sprintf("%s/%s/%s:%s", sourceRegistry, group, stateArtifactName, tag), options...) if err != nil { - return nil, fmt.Errorf("failed to create remote repository: %v", err) + return nil, fmt.Errorf("failed to pull the state artifact: %v", err) } - // Setting up the authentication for the remote registry - repo.Client = &auth.Client{ - Client: retry.DefaultClient, - Cache: auth.NewCache(), - Credential: auth.StaticCredential( - f.url, - auth.Credential{ - Username: config.GetHarborUsername(), - Password: config.GetHarborPassword(), - }, - ), - } - // Copy from the remote repository to the file store - tag := "latest" - _, err = oras.Copy(ctx, repo, tag, fs, tag, oras.DefaultCopyOptions) - if err != nil { - return nil, fmt.Errorf("failed to copy from remote repository to file store: %v", err) + tarContent := new(bytes.Buffer) + if err := crane.Export(img, tarContent); err != nil { + return nil, fmt.Errorf("failed to export the state artifact: %v", err) } - stateArtifactDir := filepath.Join(cwd, "state-artifact") - var state_reader StateReader - // Find the state artifact file in the state-artifact directory that is created temporarily - err = filepath.Walk(stateArtifactDir, func(path string, info os.FileInfo, err error) error { + // parse the state artifact + tr := tar.NewReader(tarContent) + var artifactsJSON []byte + + for { + hdr, err := tr.Next() + if err == io.EOF { + break // End of tar archive + } if err != nil { - return err + return nil, fmt.Errorf("failed to read the tar archive: %v", err) } - if filepath.Ext(info.Name()) == ".json" { - content, err := os.ReadFile(path) - if err != nil { - return err - } - state_reader, err = FromJSON(content, f.state_artifact_reader) + + if hdr.Name == "artifacts.json" { + // Found `artifacts.json`, read the content + artifactsJSON, err = io.ReadAll(tr) if err != nil { - return fmt.Errorf("failed to parse the state artifact file: %v", err) + return nil, fmt.Errorf("failed to read the artifacts.json file: %v", err) } - return nil + break } - return nil - }) + } - if err != nil { - return nil, fmt.Errorf("failed to read the state artifact file: %v", err) + if artifactsJSON == nil { + return nil, fmt.Errorf("artifacts.json not found in the state artifact") } - // Clean up everything inside the state-artifact folder - err = os.RemoveAll(stateArtifactDir) + + err = json.Unmarshal(artifactsJSON, &f.state_artifact_reader) if err != nil { - return nil, fmt.Errorf("failed to remove state-artifact directory: %v", err) + return nil, fmt.Errorf("failed to parse the artifacts.json file: %v", err) } - return state_reader, nil + + return f.state_artifact_reader, nil } // FromJSON parses the input JSON data into a StateArtifactReader @@ -151,7 +144,7 @@ func FromJSON(data []byte, reg StateReader) (StateReader, error) { return nil, err } // Validation - if reg.GetRegistryURL()== "" { + if reg.GetRegistryURL() == "" { return nil, fmt.Errorf("registry URL is required") } return reg, nil diff --git a/internal/state/replicator.go b/internal/state/replicator.go index 6dec2f0..9ac1312 100644 --- a/internal/state/replicator.go +++ b/internal/state/replicator.go @@ -10,6 +10,8 @@ import ( "container-registry.com/harbor-satellite/logger" "github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/crane" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/types" ) type Replicator interface { @@ -34,7 +36,7 @@ func NewBasicReplicator(state_reader StateReader) Replicator { stateReader: state_reader, } } - +// Replicate replicates images from the source registry to the Zot registry. func (r *BasicReplicator) Replicate(ctx context.Context) error { log := logger.FromContext(ctx) auth := authn.FromConfig(authn.AuthConfig{ @@ -46,33 +48,42 @@ func (r *BasicReplicator) Replicate(ctx context.Context) error { if r.useUnsecure { options = append(options, crane.Insecure) } - sourceRegistry := r.stateReader.GetRegistryURL() + sourceRegistry := utils.FormatRegistryUrl(config.GetRemoteRegistryURL()) for _, artifact := range r.stateReader.GetArtifacts() { - // Extract the image name from the repository of the artifact + // Extract the image name and repository from the artifact repo, image, err := utils.GetRepositoryAndImageNameFromArtifact(artifact.GetRepository()) if err != nil { log.Error().Msgf("Error getting repository and image name: %v", err) return err } - log.Info().Msgf("Pulling image %s from repository %s at registry %s", image, repo, sourceRegistry) - // Pull the image at the given repository at the source registry - srcImage, err := crane.Pull(fmt.Sprintf("%s/%s/%s", sourceRegistry, repo, image), options...) - if err != nil { - log.Error().Msgf("Failed to pull image: %v", err) - return err - } - // Push the image to the local registry - err = crane.Push(srcImage, fmt.Sprintf("%s/%s", r.zotURL, image), options...) - if err != nil { - log.Error().Msgf("Failed to push image: %v", err) - return err + allTags := artifact.GetTags() + + // Pull and replicate all tags of the image + for _, tag := range allTags { + log.Info().Msgf("Pulling image %s from repository %s at registry %s with tag %s", image, repo, sourceRegistry, tag) + + // Pull the image from the source registry + srcImage, err := crane.Pull(fmt.Sprintf("%s/%s/%s:%s", sourceRegistry, image, image, tag), options...) + if err != nil { + log.Error().Msgf("Failed to pull image: %v", err) + return err + } + + // Convert Docker manifest to OCI manifest + ociImage := mutate.MediaType(srcImage, types.OCIManifestSchema1) + + // Push the converted OCI image to the Zot registry + err = crane.Push(ociImage, fmt.Sprintf("%s/%s", r.zotURL, image), options...) + if err != nil { + log.Error().Msgf("Failed to push image: %v", err) + return err + } + log.Info().Msgf("Image %s pushed successfully", image) } - log.Info().Msgf("Image %s pushed successfully", image) } - // Delete ./local-oci-layout directory - // This is required because it is a temporary directory used by crane to pull and push images to and from - // And crane does not automatically clean it + + // Clean up the temporary directory if err := os.RemoveAll("./local-oci-layout"); err != nil { log.Error().Msgf("Failed to remove directory: %v", err) return fmt.Errorf("failed to remove directory: %w", err) diff --git a/internal/utils/utils.go b/internal/utils/utils.go index b427564..f77db03 100644 --- a/internal/utils/utils.go +++ b/internal/utils/utils.go @@ -116,3 +116,11 @@ func FormatDuration(input string) (string , error) { return result, nil } + +// FormatRegistryUrl formats the registry URL by trimming the "https://" or "http://" prefix if present +func FormatRegistryUrl(url string) string { + // Trim the "https://" or "http://" prefix if present + url = strings.TrimPrefix(url, "https://") + url = strings.TrimPrefix(url, "http://") + return url +}