diff --git a/config.toml b/config.toml index 3a2eebd..d9ab5a1 100644 --- a/config.toml +++ b/config.toml @@ -1,11 +1,9 @@ # Wether to us the built-in Zot registry or not -bring_own_registry = true +bring_own_registry = false # IP address and port of own registry own_registry_adr = "127.0.0.1" own_registry_port = "8585" -# URL of own registry -own_registry_adr = "127.0.0.1:5000" # URL of remote registry OR local file path url_or_file = "https://demo.goharbor.io/v2/myproject/album-server" @@ -14,9 +12,7 @@ url_or_file = "https://demo.goharbor.io/v2/myproject/album-server" zotConfigPath = "./registry/config.json" # Set logging level -log_level = "warn" -# url_or_file = "https://demo.goharbor.io/v2/myproject/album-server" - url_or_file = "http://localhost:5001/v2/library/busybox" +log_level = "info" # For testing purposes : # https://demo.goharbor.io/v2/myproject/album-server diff --git a/image-list/images.json b/image-list/images.json index 93da74a..271496a 100644 --- a/image-list/images.json +++ b/image-list/images.json @@ -5,13 +5,10 @@ "repository": "myproject", "images": [ { - "name": "album-server@sha256:71df27326a806ef2946ce502d26212efa11d70e4dcea06ceae612eb29cba398b" + "name": "album-server@sha256:39879890008f12c25ea14125aa8e9ec8ef3e167f0b0ed88057e955a8fa32c430" }, { - "name": "album-server" - }, - { - "name": "album-server:v1-manifest-app" + "name": "album-server:busybox" } ] } diff --git a/internal/replicate/replicate.go b/internal/replicate/replicate.go index 783a28a..5a2267a 100644 --- a/internal/replicate/replicate.go +++ b/internal/replicate/replicate.go @@ -59,19 +59,18 @@ func stripPrefix(imageName string) string { func (r *BasicReplicator) DeleteExtraImages(ctx context.Context, imgs []store.Image) error { log := logger.FromContext(ctx) - errLog := logger.ErrorLoggerFromContext(ctx) zotUrl := os.Getenv("ZOT_URL") - host := os.Getenv("HOST") registry := os.Getenv("REGISTRY") repository := os.Getenv("REPOSITORY") + image := os.Getenv("IMAGE") - localRegistry := fmt.Sprintf("%s/%s/%s/%s", zotUrl, host, registry, repository) + localRegistry := fmt.Sprintf("%s/%s/%s/%s", zotUrl, registry, repository, image) log.Info().Msgf("Local registry: %s", localRegistry) // Get the list of images from the local registry localImages, err := crane.ListTags(localRegistry) if err != nil { - errLog.Error().Msgf("failed to list tags: %v", err) + log.Error().Msgf("failed to list tags: %v", err) return err } @@ -90,7 +89,7 @@ func (r *BasicReplicator) DeleteExtraImages(ctx context.Context, imgs []store.Im log.Info().Msgf("Deleting image: %s", localImage) err := crane.Delete(fmt.Sprintf("%s:%s", localRegistry, localImage)) if err != nil { - errLog.Error().Msgf("failed to delete image: %v", err) + log.Error().Msgf("failed to delete image: %v", err) return err } log.Info().Msgf("Image deleted: %s", localImage) @@ -101,35 +100,35 @@ func (r *BasicReplicator) DeleteExtraImages(ctx context.Context, imgs []store.Im } func getPullSource(ctx context.Context, image string) string { - errLog := logger.ErrorLoggerFromContext(ctx) + log := logger.FromContext(ctx) input := os.Getenv("USER_INPUT") scheme := os.Getenv("SCHEME") if strings.HasPrefix(scheme, "http://") || strings.HasPrefix(scheme, "https://") { - url := os.Getenv("HOST") + "/" + os.Getenv("REGISTRY") + "/" + image + url := os.Getenv("REGISTRY") + "/" + os.Getenv("REPOSITORY") + "/" + image return url } else { registryInfo, err := getFileInfo(ctx, input) if err != nil { - errLog.Error().Msgf("Error getting file info: %v", err) + log.Error().Msgf("Error getting file info: %v", err) return "" } registryURL := registryInfo.RegistryUrl registryURL = strings.TrimPrefix(registryURL, "https://") - registryURL = strings.TrimSuffix(registryURL, "v2/") + registryURL = strings.TrimSuffix(registryURL, "/v2/") // TODO: Handle multiple repositories repositoryName := registryInfo.Repositories[0].Repository - return registryURL + repositoryName + "/" + image + return registryURL + "/" + repositoryName + "/" + image } } func getFileInfo(ctx context.Context, input string) (*RegistryInfo, error) { - errLog := logger.ErrorLoggerFromContext(ctx) + log := logger.FromContext(ctx) // Get the current working directory workingDir, err := os.Getwd() if err != nil { - errLog.Error().Msgf("Error getting current directory: %v", err) + log.Error().Msgf("Error getting current directory: %v", err) return nil, err } @@ -139,14 +138,14 @@ func getFileInfo(ctx context.Context, input string) (*RegistryInfo, error) { // Read the file jsonData, err := os.ReadFile(fullPath) if err != nil { - errLog.Error().Msgf("Error reading file: %v", err) + log.Error().Msgf("Error reading file: %v", err) return nil, err } var registryInfo RegistryInfo err = json.Unmarshal(jsonData, ®istryInfo) if err != nil { - errLog.Error().Msgf("Error unmarshalling JSON data: %v", err) + log.Error().Msgf("Error unmarshalling JSON data: %v", err) return nil, err } @@ -155,24 +154,22 @@ func getFileInfo(ctx context.Context, input string) (*RegistryInfo, error) { func CopyImage(ctx context.Context, imageName string) error { log := logger.FromContext(ctx) - errLog := logger.ErrorLoggerFromContext(ctx) log.Info().Msgf("Copying image: %s", imageName) zotUrl := os.Getenv("ZOT_URL") if zotUrl == "" { - errLog.Error().Msg("ZOT_URL environment variable is not set") + log.Error().Msg("ZOT_URL environment variable is not set") return fmt.Errorf("ZOT_URL environment variable is not set") } - // Clean up the image name by removing any host part - cleanedImageName := removeHostName(imageName) - destRef := fmt.Sprintf("%s/%s", zotUrl, cleanedImageName) - fmt.Println("Destination reference:", destRef) + // Build the destination reference + destRef := fmt.Sprintf("%s/%s", zotUrl, imageName) + log.Info().Msgf("Destination reference: %s", destRef) // Get credentials from environment variables username := os.Getenv("HARBOR_USERNAME") password := os.Getenv("HARBOR_PASSWORD") if username == "" || password == "" { - errLog.Error().Msg("HARBOR_USERNAME or HARBOR_PASSWORD environment variable is not set") + log.Error().Msg("HARBOR_USERNAME or HARBOR_PASSWORD environment variable is not set") return fmt.Errorf("HARBOR_USERNAME or HARBOR_PASSWORD environment variable is not set") } @@ -184,7 +181,7 @@ func CopyImage(ctx context.Context, imageName string) error { // Pull the image with authentication srcImage, err := crane.Pull(imageName, crane.WithAuth(auth), crane.Insecure) if err != nil { - errLog.Error().Msgf("Failed to pull image: %v", err) + log.Error().Msgf("Failed to pull image: %v", err) return fmt.Errorf("failed to pull image: %w", err) } else { log.Info().Msg("Image pulled successfully") @@ -193,7 +190,7 @@ func CopyImage(ctx context.Context, imageName string) error { // Push the image to the destination registry err = crane.Push(srcImage, destRef, crane.Insecure) if err != nil { - errLog.Error().Msgf("Failed to push image: %v", err) + log.Error().Msgf("Failed to push image: %v", err) return fmt.Errorf("failed to push image: %w", err) } else { log.Info().Msg("Image pushed successfully") @@ -203,19 +200,9 @@ func CopyImage(ctx context.Context, imageName string) error { // This is required because it is a temporary directory used by crane to pull and push images to and from // And crane does not automatically clean it if err := os.RemoveAll("./local-oci-layout"); err != nil { - errLog.Error().Msgf("Failed to remove directory: %v", err) + log.Error().Msgf("Failed to remove directory: %v", err) return fmt.Errorf("failed to remove directory: %w", err) } return nil } - -// take only the parts after the hostname -func removeHostName(imageName string) string { - parts := strings.Split(imageName, "/") - if len(parts) > 1 { - return strings.Join(parts[1:], "/") - } - - return imageName -} diff --git a/internal/satellite/satellite.go b/internal/satellite/satellite.go index 91eb644..b2f6b0a 100644 --- a/internal/satellite/satellite.go +++ b/internal/satellite/satellite.go @@ -23,12 +23,11 @@ func NewSatellite(ctx context.Context, storer store.Storer, replicator replicate func (s *Satellite) Run(ctx context.Context) error { log := logger.FromContext(ctx) - errLog := logger.ErrorLoggerFromContext(ctx) // Execute the initial operation immediately without waiting for the ticker imgs, err := s.storer.List(ctx) if err != nil { - errLog.Error().Err(err).Msg("Error listing images") + log.Error().Err(err).Msg("Error listing images") return err } if len(imgs) == 0 { @@ -37,7 +36,7 @@ func (s *Satellite) Run(ctx context.Context) error { for _, img := range imgs { err = s.replicator.Replicate(ctx, img.Name) if err != nil { - errLog.Error().Err(err).Msg("Error replicating image") + log.Error().Err(err).Msg("Error replicating image") return err } } @@ -56,7 +55,7 @@ func (s *Satellite) Run(ctx context.Context) error { case <-ticker.C: imgs, err := s.storer.List(ctx) if err != nil { - errLog.Error().Err(err).Msg("Error listing images") + log.Error().Err(err).Msg("Error listing images") return err } if len(imgs) == 0 { @@ -65,7 +64,7 @@ func (s *Satellite) Run(ctx context.Context) error { for _, img := range imgs { err = s.replicator.Replicate(ctx, img.Name) if err != nil { - errLog.Error().Err(err).Msg("Error replicating image") + log.Error().Err(err).Msg("Error replicating image") return err } } diff --git a/internal/store/file-fetch.go b/internal/store/file-fetch.go index 78e0690..fe32f63 100644 --- a/internal/store/file-fetch.go +++ b/internal/store/file-fetch.go @@ -30,11 +30,11 @@ func (f *FileImageList) Type(ctx context.Context) string { } func FileImageListFetcher(ctx context.Context, relativePath string) *FileImageList { - errLog := logger.ErrorLoggerFromContext(ctx) + log := logger.FromContext(ctx) // Get the current working directory dir, err := os.Getwd() if err != nil { - errLog.Error().Err(err).Msg("Error getting current directory") + log.Error().Err(err).Msg("Error getting current directory") return nil } @@ -47,13 +47,13 @@ func FileImageListFetcher(ctx context.Context, relativePath string) *FileImageLi } func (client *FileImageList) List(ctx context.Context) ([]Image, error) { - errLog := logger.ErrorLoggerFromContext(ctx) + log := logger.FromContext(ctx) var images []Image // Read the file data, err := os.ReadFile(client.Path) if err != nil { - errLog.Error().Err(err).Msg("Error reading file") + log.Error().Err(err).Msg("Error reading file") return nil, err } @@ -61,7 +61,7 @@ func (client *FileImageList) List(ctx context.Context) ([]Image, error) { // Parse the JSON data err = json.Unmarshal(data, &imageData) if err != nil { - errLog.Error().Err(err).Msg("Error unmarshalling JSON data") + log.Error().Err(err).Msg("Error unmarshalling JSON data") return nil, err } diff --git a/internal/store/http-fetch.go b/internal/store/http-fetch.go index 352a315..922a454 100644 --- a/internal/store/http-fetch.go +++ b/internal/store/http-fetch.go @@ -37,7 +37,6 @@ func (r *RemoteImageList) Type(ctx context.Context) string { func (client *RemoteImageList) List(ctx context.Context) ([]Image, error) { log := logger.FromContext(ctx) - errLog := logger.ErrorLoggerFromContext(ctx) // Construct the URL for fetching tags url := client.BaseURL + "/tags/list" @@ -49,7 +48,7 @@ func (client *RemoteImageList) List(ctx context.Context) ([]Image, error) { // Create a new HTTP request req, err := http.NewRequest("GET", url, nil) if err != nil { - errLog.Error().Msgf("failed to create request: %v", err) + log.Error().Msgf("failed to create request: %v", err) return nil, err } @@ -65,7 +64,7 @@ func (client *RemoteImageList) List(ctx context.Context) ([]Image, error) { log.Info().Msgf("Sending request to %s", url) resp, err := httpClient.Do(req) if err != nil { - errLog.Error().Msgf("failed to send request: %v", err) + log.Error().Msgf("failed to send request: %v", err) return nil, err } defer resp.Body.Close() @@ -73,14 +72,14 @@ func (client *RemoteImageList) List(ctx context.Context) ([]Image, error) { // Read the response body body, err := io.ReadAll(resp.Body) if err != nil { - errLog.Error().Msgf("failed to read response body: %v", err) + log.Error().Msgf("failed to read response body: %v", err) return nil, err } // Unmarshal the JSON response var tagListResponse TagListResponse if err := json.Unmarshal(body, &tagListResponse); err != nil { - errLog.Error().Msgf("failed to unmarshal response: %v", err) + log.Error().Msgf("failed to unmarshal response: %v", err) return nil, err } @@ -97,7 +96,7 @@ func (client *RemoteImageList) List(ctx context.Context) ([]Image, error) { } func (client *RemoteImageList) GetDigest(ctx context.Context, tag string) (string, error) { - errLog := logger.ErrorLoggerFromContext(ctx) + log := logger.FromContext(ctx) // Construct the image reference imageRef := fmt.Sprintf("%s:%s", client.BaseURL, tag) // Remove extra characters from the URL @@ -114,7 +113,7 @@ func (client *RemoteImageList) GetDigest(ctx context.Context, tag string) (strin Password: password, }), crane.Insecure) if err != nil { - errLog.Error().Msgf("failed to get digest using crane: %v", err) + log.Error().Msgf("failed to get digest using crane: %v", err) return "", nil } diff --git a/internal/store/in-memory-store.go b/internal/store/in-memory-store.go index c0913e2..da25d3f 100644 --- a/internal/store/in-memory-store.go +++ b/internal/store/in-memory-store.go @@ -41,7 +41,6 @@ func NewInMemoryStore(ctx context.Context, fetcher ImageFetcher) (context.Contex func (s *inMemoryStore) List(ctx context.Context) ([]Image, error) { log := logger.FromContext(ctx) - errLog := logger.ErrorLoggerFromContext(ctx) var imageList []Image var change bool err := error(nil) @@ -108,7 +107,7 @@ func (s *inMemoryStore) List(ctx context.Context) ([]Image, error) { tagParts := strings.Split(img.Name, ":") // Check if there is a tag part, min length is 1 char if len(tagParts) < 2 { - errLog.Error().Msgf("Invalid image reference: %s", img.Name) + log.Error().Msgf("Invalid image reference: %s", img.Name) } // Use the last part as the tag tag := tagParts[len(tagParts)-1] @@ -208,7 +207,7 @@ func (s *inMemoryStore) RemoveImage(ctx context.Context, image string) error { // TODO: Rework complicated logic and add support for multiple repositories // checkImageAndDigest checks if the image exists in the store and if the digest matches the image reference func (s *inMemoryStore) checkImageAndDigest(ctx context.Context, digest string, image string) bool { - errLog := logger.ErrorLoggerFromContext(ctx) + log := logger.FromContext(ctx) // Check if the received image exists in the store for storeDigest, storeImage := range s.images { @@ -220,7 +219,7 @@ func (s *inMemoryStore) checkImageAndDigest(ctx context.Context, digest string, tag := strings.Split(image, ":")[1] localRegistryDigest, err := GetLocalDigest(context.Background(), tag) if err != nil { - errLog.Error().Msgf("Error getting digest from local registry: %v", err) + log.Error().Msgf("Error getting digest from local registry: %v", err) return false } else { // Check if the digest from the local registry matches the digest from the store @@ -247,34 +246,24 @@ func (s *inMemoryStore) checkImageAndDigest(ctx context.Context, digest string, } func GetLocalDigest(ctx context.Context, tag string) (string, error) { - errLog := logger.ErrorLoggerFromContext(ctx) + log := logger.FromContext(ctx) zotUrl := os.Getenv("ZOT_URL") userURL := os.Getenv("USER_INPUT") + // Remove extra characters from the URLs userURL = userURL[strings.Index(userURL, "//")+2:] userURL = strings.ReplaceAll(userURL, "/v2", "") - regUrl := removeHostName(userURL) // Construct the URL for fetching the digest - url := zotUrl + "/" + regUrl + ":" + tag + url := zotUrl + "/" + userURL + ":" + tag // Use crane.Digest to get the digest of the image digest, err := crane.Digest(url) if err != nil { - errLog.Error().Msgf("Error getting digest using crane: %v", err) + log.Error().Msgf("Error getting digest using crane: %v", err) return "", fmt.Errorf("failed to get digest using crane: %w", err) } return digest, nil } - -// Split the imageName by "/" and take only the parts after the hostname -func removeHostName(imageName string) string { - parts := strings.Split(imageName, "/") - if len(parts) > 1 { - return strings.Join(parts[1:], "/") - } - - return imageName -} diff --git a/logger/logger.go b/logger/logger.go index d6784d0..78664dc 100644 --- a/logger/logger.go +++ b/logger/logger.go @@ -10,7 +10,6 @@ import ( type contextKey string const loggerKey contextKey = "logger" -const errorLoggerKey contextKey = "errorLogger" // AddLoggerToContext creates a new context with a zerolog logger for stdout adn stderr and sets the global log level. func AddLoggerToContext(ctx context.Context, logLevel string) context.Context { @@ -31,13 +30,9 @@ func AddLoggerToContext(ctx context.Context, logLevel string) context.Context { default: zerolog.SetGlobalLevel(zerolog.InfoLevel) } - // Use os.Stdout for the main logger - logger := zerolog.New(os.Stdout).With().Timestamp().Logger() - ctx = context.WithValue(ctx, loggerKey, &logger) - // Use os.Stderr for the error logger - errorLogger := zerolog.New(os.Stderr).With().Timestamp().Logger() - ctx = context.WithValue(ctx, errorLoggerKey, &errorLogger) + logger := zerolog.New(os.Stderr).With().Timestamp().Logger() + ctx = context.WithValue(ctx, loggerKey, &logger) return ctx } @@ -53,15 +48,3 @@ func FromContext(ctx context.Context) *zerolog.Logger { } return logger } - -// ErrorLoggerFromContext extracts the error logger from the context. -func ErrorLoggerFromContext(ctx context.Context) *zerolog.Logger { - errorLogger, ok := ctx.Value(errorLoggerKey).(*zerolog.Logger) - if !ok { - // Fallback to a default logger if none is found in the context. - defaultErrorLogger := zerolog.New(os.Stderr).With().Timestamp().Logger() - defaultErrorLogger.Error().Msg("Failed to extract error logger from context") - return &defaultErrorLogger - } - return errorLogger -} diff --git a/main.go b/main.go index c717d49..78ff515 100644 --- a/main.go +++ b/main.go @@ -2,6 +2,7 @@ package main import ( "context" + "encoding/json" "errors" "fmt" "net" @@ -30,6 +31,16 @@ import ( "github.com/joho/godotenv" ) +type ImageList struct { + RegistryURL string `json:"registryUrl"` + Repositories []struct { + Repository string `json:"repository"` + Images []struct { + Name string `json:"name"` + } `json:"images"` + } `json:"repositories"` +} + func main() { viper.SetConfigName("config") viper.SetConfigType("toml") @@ -57,7 +68,6 @@ func run() error { ctx = logger.AddLoggerToContext(ctx, logLevel) log := logger.FromContext(ctx) - errLog := logger.ErrorLoggerFromContext(ctx) log.Info().Msg("Satellite starting") mux := http.NewServeMux() @@ -95,13 +105,13 @@ func run() error { // Validate registryAdr format ip := net.ParseIP(registryAdr) if ip == nil { - errLog.Error().Msg("Invalid IP address") + log.Error().Msg("Invalid IP address") return errors.New("invalid IP address") } if ip.To4() != nil { log.Info().Msg("IP address is valid IPv4") } else { - errLog.Error().Msg("IP address is IPv6 format and unsupported") + log.Error().Msg("IP address is IPv6 format and unsupported") return errors.New("IP address is IPv6 format and unsupported") } registryPort := viper.GetString("own_registry_port") @@ -116,7 +126,7 @@ func run() error { } if err != nil { cancel() - errLog.Error().Err(err).Msg("Failed to launch default registry") + log.Error().Err(err).Msg("Failed to launch default registry") return err } return nil @@ -127,22 +137,53 @@ func run() error { parsedURL, err := url.Parse(input) if err != nil || parsedURL.Scheme == "" { if strings.ContainsAny(input, "\\:*?\"<>|") { - errLog.Error().Msg("Path contains invalid characters. Please check the configuration.") + log.Error().Msg("Path contains invalid characters. Please check the configuration.") return err } dir, err := os.Getwd() if err != nil { - errLog.Error().Err(err).Msg("Error getting current directory") + log.Error().Err(err).Msg("Error getting current directory") return err } absPath := filepath.Join(dir, input) if _, err := os.Stat(absPath); os.IsNotExist(err) { - errLog.Error().Err(err).Msg("No URL or file found. Please check the configuration.") + log.Error().Err(err).Msg("No URL or file found. Please check the configuration.") return err } log.Info().Msg("Input is a valid file path.") fetcher = store.FileImageListFetcher(ctx, input) os.Setenv("USER_INPUT", input) + + // Parse images.json and set environment variables + file, err := os.Open(absPath) + if err != nil { + log.Error().Err(err).Msg("Error opening images.json file") + return err + } + defer file.Close() + + var imageList ImageList + if err := json.NewDecoder(file).Decode(&imageList); err != nil { + log.Error().Err(err).Msg("Error decoding images.json file") + return err + } + + registryURL := imageList.RegistryURL + registryParts := strings.Split(registryURL, "/") + if len(registryParts) < 3 { + log.Error().Msg("Invalid registryUrl format in images.json") + return errors.New("invalid registryUrl format in images.json") + } + registry := registryParts[2] + os.Setenv("REGISTRY", registry) + + if len(imageList.Repositories) > 0 { + repository := imageList.Repositories[0].Repository + os.Setenv("REPOSITORY", repository) + } else { + log.Error().Msg("No repositories found in images.json") + return errors.New("no repositories found in images.json") + } } else { log.Info().Msg("Input is a valid URL.") fetcher = store.RemoteImageListFetcher(ctx, input) @@ -150,21 +191,21 @@ func run() error { parts := strings.SplitN(input, "://", 2) scheme := parts[0] + "://" os.Setenv("SCHEME", scheme) - hostAndPath := parts[1] - hostParts := strings.Split(hostAndPath, "/") - host := hostParts[0] - os.Setenv("HOST", host) - apiVersion := hostParts[1] - os.Setenv("API_VERSION", apiVersion) - registry := hostParts[2] + registryAndPath := parts[1] + registryParts := strings.Split(registryAndPath, "/") + registry := registryParts[0] os.Setenv("REGISTRY", registry) - repository := hostParts[3] + apiVersion := registryParts[1] + os.Setenv("API_VERSION", apiVersion) + repository := registryParts[2] os.Setenv("REPOSITORY", repository) + image := registryParts[3] + os.Setenv("IMAGE", image) } err = godotenv.Load() if err != nil { - errLog.Error().Err(err).Msg("Error loading.env file") + log.Error().Err(err).Msg("Error loading.env file") return err } @@ -179,7 +220,7 @@ func run() error { err = g.Wait() if err != nil { - errLog.Error().Err(err).Msg("Error running satellite") + log.Error().Err(err).Msg("Error running satellite") return err } return nil diff --git a/registry/config.json b/registry/config.json index 7b8f91a..3401d70 100644 --- a/registry/config.json +++ b/registry/config.json @@ -1,10 +1,13 @@ { - "distSpecVersion": "1.1.0", - "storage": { - "rootDirectory": "./zot" - }, - "http": { - "address": "127.0.0.1", - "port": "8585" - } + "distSpecVersion": "1.1.0", + "storage": { + "rootDirectory": "./zot" + }, + "http": { + "address": "127.0.0.1", + "port": "8585" + }, + "log": { + "level": "info" + } }