adding config to process new state artifact file

.env

@@ -4,5 +4,5 @@ ZOT_URL="127.0.0.1:8585"
 TOKEN=""
 ENV=dev
 USE_UNSECURE=true
-GROUP_NAME=test-satellite-group
-STATE_ARTIFACT_NAME=state-artifact
+GROUP_NAME=satellite-test-group-state
+STATE_ARTIFACT_NAME=state

config.toml

@@ -7,7 +7,7 @@ own_registry_port = "8585"
 
 # URL of remote registry OR local file path
 # url_or_file = "https://demo.goharbor.io/v2/myproject/album-server"
-url_or_file = "https://demo.goharbor.io"
+url_or_file = "https://registry.bupd.xyz"
 ## for testing for local file
 # url_or_file = "./image-list/images.json"
 

image-list/images.json

@@ -1,15 +1,25 @@
 {
-  "registry": "http://demo.goharbor.io/",
-   "artifacts": [
-     {
-      "repository": "satellite-test-alpine/alpine",
-      "tag": "latest",
-      "hash": "sha256:9cee2b38"
+  "registry": "Satellite",
+  "artifacts": [
+    {
+      "repository": "satellite-test-group-state/alpine",
+      "tag": [
+        "latest"
+      ],
+      "labels": null,
+      "type": "IMAGE",
+      "digest": "sha256:9cee2b382fe2412cd77d5d437d15a93da8de373813621f2e4d406e3df0cf0e7c",
+      "deleted": false
     },
-   {
-    "repository": "satellite-test-postgres/postgres",
-    "tag": "latest",
-    "hash": "sha256:9cee2b38"
-   }
+    {
+      "repository": "satellite-test-group-state/postgres",
+      "tag": [
+        "latest"
+      ],
+      "labels": null,
+      "type": "IMAGE",
+      "digest": "sha256:dde924f70bc972261013327c480adf402ea71487b5750e40569a0b74fa90c74a",
+      "deleted": false
+    }
   ]
 }

internal/state/fetcher.go

@@ -1,22 +1,20 @@
 package state
 
 import (
-	"context"
+	"archive/tar"
+	"bytes"
 	"encoding/json"
 	"fmt"
+	"io"
 	"os"
-	"path/filepath"
 	"strings"
 
 	"container-registry.com/harbor-satellite/internal/config"
-	"oras.land/oras-go/v2"
-	"oras.land/oras-go/v2/content/file"
-	"oras.land/oras-go/v2/registry/remote"
-	"oras.land/oras-go/v2/registry/remote/auth"
-	"oras.land/oras-go/v2/registry/remote/retry"
+	"container-registry.com/harbor-satellite/internal/utils"
+	"github.com/google/go-containerregistry/pkg/authn"
+	"github.com/google/go-containerregistry/pkg/crane"
 )
 
-
 type StateFetcher interface {
 	// Fetches the state artifact from the registry
 	FetchStateArtifact() (StateReader, error)
@@ -75,73 +73,68 @@ func (f *FileStateArtifactFetcher) FetchStateArtifact() (StateReader, error) {
 }
 
 func (f *URLStateFetcher) FetchStateArtifact() (StateReader, error) {
-	cwd, err := os.Getwd()
-	if err != nil {
-		return nil, fmt.Errorf("failed to get current working directory: %v", err)
-	}
-	// Creating a file store in the current working directory will be deleted later after reading the state artifact
-	fs, err := file.New(fmt.Sprintf("%s/state-artifact", cwd))
-	if err != nil {
-		return nil, fmt.Errorf("failed to create file store: %v", err)
+
+	auth := authn.FromConfig(authn.AuthConfig{
+		Username: config.GetHarborUsername(),
+		Password: config.GetHarborPassword(),
+	})
+
+	options := []crane.Option{crane.WithAuth(auth)}
+	if config.UseUnsecure() {
+		options = append(options, crane.Insecure)
 	}
-	defer fs.Close()
 
-	ctx := context.Background()
+	sourceRegistry := utils.FormatRegistryUrl(config.GetRemoteRegistryURL())
+	group := config.GetGroupName()
+	stateArtifactName := config.GetStateArtifactName()
+	var tag string = "latest"
+	fmt.Printf("Pulling state artifact from %s/%s/%s:%s\n", sourceRegistry, group, stateArtifactName, tag)
+	fmt.Printf("Auth: %v\n", auth)
 
-	repo, err := remote.NewRepository(fmt.Sprintf("%s/%s/%s", f.url, f.group_name, f.state_artifact_name))
+	// pull the state artifact from the central registry
+	img, err := crane.Pull(fmt.Sprintf("%s/%s/%s:%s", sourceRegistry, group, stateArtifactName, tag), options...)
 	if err != nil {
-		return nil, fmt.Errorf("failed to create remote repository: %v", err)
+		return nil, fmt.Errorf("failed to pull the state artifact: %v", err)
 	}
 
-	// Setting up the authentication for the remote registry
-	repo.Client = &auth.Client{
-		Client: retry.DefaultClient,
-		Cache:  auth.NewCache(),
-		Credential: auth.StaticCredential(
-			f.url,
-			auth.Credential{
-				Username: config.GetHarborUsername(),
-				Password: config.GetHarborPassword(),
-			},
-		),
-	}
-	// Copy from the remote repository to the file store
-	tag := "latest"
-	_, err = oras.Copy(ctx, repo, tag, fs, tag, oras.DefaultCopyOptions)
-	if err != nil {
-		return nil, fmt.Errorf("failed to copy from remote repository to file store: %v", err)
+	tarContent := new(bytes.Buffer)
+	if err := crane.Export(img, tarContent); err != nil {
+		return nil, fmt.Errorf("failed to export the state artifact: %v", err)
 	}
-	stateArtifactDir := filepath.Join(cwd, "state-artifact")
 
-	var state_reader StateReader
-	// Find the state artifact file in the state-artifact directory that is created temporarily
-	err = filepath.Walk(stateArtifactDir, func(path string, info os.FileInfo, err error) error {
+	// parse the state artifact
+	tr := tar.NewReader(tarContent)
+	var artifactsJSON []byte
+
+	for {
+		hdr, err := tr.Next()
+		if err == io.EOF {
+			break // End of tar archive
+		}
 		if err != nil {
-			return err
+			return nil, fmt.Errorf("failed to read the tar archive: %v", err)
 		}
-		if filepath.Ext(info.Name()) == ".json" {
-			content, err := os.ReadFile(path)
-			if err != nil {
-				return err
-			}
-			state_reader, err = FromJSON(content, f.state_artifact_reader)
+
+		if hdr.Name == "artifacts.json" {
+			// Found `artifacts.json`, read the content
+			artifactsJSON, err = io.ReadAll(tr)
 			if err != nil {
-				return fmt.Errorf("failed to parse the state artifact file: %v", err)
+				return nil, fmt.Errorf("failed to read the artifacts.json file: %v", err)
 			}
-			return nil
+			break
 		}
-		return nil
-	})
+	}
 
-	if err != nil {
-		return nil, fmt.Errorf("failed to read the state artifact file: %v", err)
+	if artifactsJSON == nil {
+		return nil, fmt.Errorf("artifacts.json not found in the state artifact")
 	}
-	// Clean up everything inside the state-artifact folder
-	err = os.RemoveAll(stateArtifactDir)
+
+	err = json.Unmarshal(artifactsJSON, &f.state_artifact_reader)
 	if err != nil {
-		return nil, fmt.Errorf("failed to remove state-artifact directory: %v", err)
+		return nil, fmt.Errorf("failed to parse the artifacts.json file: %v", err)
 	}
-	return state_reader, nil
+
+	return f.state_artifact_reader, nil
 }
 
 // FromJSON parses the input JSON data into a StateArtifactReader
@@ -151,7 +144,7 @@ func FromJSON(data []byte, reg StateReader) (StateReader, error) {
 		return nil, err
 	}
 	// Validation
-	if reg.GetRegistryURL()== "" {
+	if reg.GetRegistryURL() == "" {
 		return nil, fmt.Errorf("registry URL is required")
 	}
 	return reg, nil

internal/state/replicator.go

@@ -10,6 +10,8 @@ import (
 	"container-registry.com/harbor-satellite/logger"
 	"github.com/google/go-containerregistry/pkg/authn"
 	"github.com/google/go-containerregistry/pkg/crane"
+	"github.com/google/go-containerregistry/pkg/v1/mutate"
+	"github.com/google/go-containerregistry/pkg/v1/types"
 )
 
 type Replicator interface {
@@ -34,7 +36,7 @@ func NewBasicReplicator(state_reader StateReader) Replicator {
 		stateReader: state_reader,
 	}
 }
-
+// Replicate replicates images from the source registry to the Zot registry.
 func (r *BasicReplicator) Replicate(ctx context.Context) error {
 	log := logger.FromContext(ctx)
 	auth := authn.FromConfig(authn.AuthConfig{
@@ -46,33 +48,42 @@ func (r *BasicReplicator) Replicate(ctx context.Context) error {
 	if r.useUnsecure {
 		options = append(options, crane.Insecure)
 	}
-	sourceRegistry := r.stateReader.GetRegistryURL()
+	sourceRegistry := utils.FormatRegistryUrl(config.GetRemoteRegistryURL())
 
 	for _, artifact := range r.stateReader.GetArtifacts() {
-		// Extract the image name from the repository of the artifact
+		// Extract the image name and repository from the artifact
 		repo, image, err := utils.GetRepositoryAndImageNameFromArtifact(artifact.GetRepository())
 		if err != nil {
 			log.Error().Msgf("Error getting repository and image name: %v", err)
 			return err
 		}
-		log.Info().Msgf("Pulling image %s from repository %s at registry %s", image, repo, sourceRegistry)
-		// Pull the image at the given repository at the source registry
-		srcImage, err := crane.Pull(fmt.Sprintf("%s/%s/%s", sourceRegistry, repo, image), options...)
-		if err != nil {
-			log.Error().Msgf("Failed to pull image: %v", err)
-			return err
-		}
-		// Push the image to the local registry
-		err = crane.Push(srcImage, fmt.Sprintf("%s/%s", r.zotURL, image), options...)
-		if err != nil {
-			log.Error().Msgf("Failed to push image: %v", err)
-			return err
+		allTags := artifact.GetTags()
+
+		// Pull and replicate all tags of the image
+		for _, tag := range allTags {
+			log.Info().Msgf("Pulling image %s from repository %s at registry %s with tag %s", image, repo, sourceRegistry, tag)
+
+			// Pull the image from the source registry
+			srcImage, err := crane.Pull(fmt.Sprintf("%s/%s/%s:%s", sourceRegistry, image, image, tag), options...)
+			if err != nil {
+				log.Error().Msgf("Failed to pull image: %v", err)
+				return err
+			}
+
+			// Convert Docker manifest to OCI manifest
+			ociImage := mutate.MediaType(srcImage, types.OCIManifestSchema1)
+
+			// Push the converted OCI image to the Zot registry
+			err = crane.Push(ociImage, fmt.Sprintf("%s/%s", r.zotURL, image), options...)
+			if err != nil {
+				log.Error().Msgf("Failed to push image: %v", err)
+				return err
+			}
+			log.Info().Msgf("Image %s pushed successfully", image)
 		}
-		log.Info().Msgf("Image %s pushed successfully", image)
 	}
-	// Delete ./local-oci-layout directory
-	// This is required because it is a temporary directory used by crane to pull and push images to and from
-	// And crane does not automatically clean it
+
+	// Clean up the temporary directory
 	if err := os.RemoveAll("./local-oci-layout"); err != nil {
 		log.Error().Msgf("Failed to remove directory: %v", err)
 		return fmt.Errorf("failed to remove directory: %w", err)

internal/utils/utils.go

@@ -116,3 +116,11 @@ func FormatDuration(input string) (string , error) {
 
 	return result, nil
 }
+
+// FormatRegistryUrl formats the registry URL by trimming the "https://" or "http://" prefix if present
+func FormatRegistryUrl(url string) string {
+	// Trim the "https://" or "http://" prefix if present
+	url = strings.TrimPrefix(url, "https://")
+	url = strings.TrimPrefix(url, "http://")
+	return url
+}