From ce6a19e9b833945e31bf7578eef7a46ed3a639a2 Mon Sep 17 00:00:00 2001 From: Mehul-Kumar-27 Date: Sat, 14 Sep 2024 00:40:33 +0530 Subject: [PATCH] adding utils for simplification --- internal/config/config.go | 122 +++++++++++++++-- internal/images/get-images.go | 11 ++ internal/utils/utils.go | 131 +++++++++++++++++++ main.go | 237 +++++++++++++++------------------- 4 files changed, 353 insertions(+), 148 deletions(-) create mode 100644 internal/utils/utils.go diff --git a/internal/config/config.go b/internal/config/config.go index 44331b6..8948647 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -6,13 +6,102 @@ import ( "github.com/spf13/viper" ) +var AppConfig *Config + type Config struct { - LogLevel string - OwnRegistry bool - OwnRegistryAdr string - OwnRegistryPort string - ZotConfigPath string - Input string + logLevel string + ownRegistry bool + ownRegistryAdr string + ownRegistryPort string + zotConfigPath string + input string + zot_url string + registry string + repository string + user_input string + scheme string + api_version string + image string +} + +func GetLogLevel() string { + return AppConfig.logLevel +} + +func GetOwnRegistry() bool { + return AppConfig.ownRegistry +} + +func GetOwnRegistryAdr() string { + return AppConfig.ownRegistryAdr +} + +func GetOwnRegistryPort() string { + return AppConfig.ownRegistryPort +} + +func GetZotConfigPath() string { + return AppConfig.zotConfigPath +} + +func GetInput() string { + return AppConfig.input +} + +func SetZotURL(url string) { + AppConfig.zot_url = url +} + +func GetZotURL() string { + return AppConfig.zot_url +} + +func SetRegistry(registry string) { + AppConfig.registry = registry +} + +func GetRegistry() string { + return AppConfig.registry +} + +func SetRepository(repository string) { + AppConfig.repository = repository +} + +func GetRepository() string { + return AppConfig.repository +} + +func SetUserInput(user_input string) { + AppConfig.user_input = user_input +} + +func GetUserInput() string { + return AppConfig.user_input +} + +func SetScheme(scheme string) { + AppConfig.scheme = scheme +} + +func GetScheme() string { + return AppConfig.scheme +} + +func SetAPIVersion(api_version string) { + AppConfig.api_version = api_version +} + +func GetAPIVersion() string { + return AppConfig.api_version +} + +func SetImage(image string) { + AppConfig.image = image +} + +func GetImage() string { + return AppConfig.image } func LoadConfig() (*Config, error) { @@ -24,11 +113,20 @@ func LoadConfig() (*Config, error) { } return &Config{ - LogLevel: viper.GetString("log_level"), - OwnRegistry: viper.GetBool("bring_own_registry"), - OwnRegistryAdr: viper.GetString("own_registry_adr"), - OwnRegistryPort: viper.GetString("own_registry_port"), - ZotConfigPath: viper.GetString("zotConfigPath"), - Input: viper.GetString("url_or_file"), + logLevel: viper.GetString("log_level"), + ownRegistry: viper.GetBool("bring_own_registry"), + ownRegistryAdr: viper.GetString("own_registry_adr"), + ownRegistryPort: viper.GetString("own_registry_port"), + zotConfigPath: viper.GetString("zotConfigPath"), + input: viper.GetString("url_or_file"), }, nil } + +func InitConfig() error { + var err error + AppConfig, err = LoadConfig() + if err != nil { + return err + } + return nil +} diff --git a/internal/images/get-images.go b/internal/images/get-images.go index bac6cdc..e20e38b 100644 --- a/internal/images/get-images.go +++ b/internal/images/get-images.go @@ -10,6 +10,17 @@ import ( "time" ) +type ImageList struct { + RegistryURL string `json:"registryUrl"` + Repositories []struct { + Repository string `json:"repository"` + Images []struct { + Name string `json:"name"` + } `json:"images"` + } `json:"repositories"` +} + + type Image struct { ID int `json:"ID"` Registry string `json:"Registry"` diff --git a/internal/utils/utils.go b/internal/utils/utils.go new file mode 100644 index 0000000..1b647d6 --- /dev/null +++ b/internal/utils/utils.go @@ -0,0 +1,131 @@ +package utils + +import ( + "encoding/json" + "errors" + "fmt" + "net" + "net/url" + "os" + "path/filepath" + "strings" + + "container-registry.com/harbor-satellite/internal/config" + "container-registry.com/harbor-satellite/internal/images" + "container-registry.com/harbor-satellite/registry" +) + +// / ValidateRegistryAddress validates the registry address and port and returns the URL +func ValidateRegistryAddress(registryAdr, registryPort string) (string, error) { + ip := net.ParseIP(registryAdr) + if ip == nil { + return "", errors.New("invalid IP address") + } + if ip.To4() != nil { + } else { + return "", errors.New("IP address is IPv6 format and unsupported") + } + + return fmt.Sprintf("%s:%s", registryAdr, registryPort), nil +} + +// / HandleOwnRegistry handles the own registry address and port and sets the Zot URL +func HandleOwnRegistry() error { + zotURL, err := ValidateRegistryAddress(config.GetOwnRegistryAdr(), config.GetOwnRegistryPort()) + if err != nil { + return err + } + config.SetZotURL(zotURL) + return nil +} + +// LaunchDefaultZotRegistry launches the default Zot registry using the Zot config path +func LaunchDefaultZotRegistry() error { + launch, err := registry.LaunchRegistry(config.GetZotConfigPath()) + if !launch { + return fmt.Errorf("error launching registry: %w", err) + } + if err != nil { + return fmt.Errorf("error launching registry: %w", err) + } + return nil +} + +// Helper function to determine if input is a valid URL +func IsValidURL(input string) bool { + parsedURL, err := url.Parse(input) + return err == nil && parsedURL.Scheme != "" +} + +// GetAbsFilePath gets the absolute file path of the input file path and checks if it exists +func GetAbsFilePath(input string) error { + dir, err := os.Getwd() + if err != nil { + return err + } + absPath := filepath.Join(dir, input) + if _, err := os.Stat(absPath); os.IsNotExist(err) { + return err + } + return nil +} + +// Check if path contains invalid characters +func HasInvalidPathChars(input string) bool { + return strings.ContainsAny(input, "\\:*?\"<>|") +} + +// ParseImagesJsonFile parses the images.json file and decodes it into the ImageList struct +func ParseImagesJsonFile(absPath string, imagesList *images.ImageList) error { + file, err := os.Open(absPath) + if err != nil { + return err + } + defer file.Close() + + if err := json.NewDecoder(file).Decode(imagesList); err != nil { + return err + } + return nil +} + +// Set registry environment variables +func SetRegistryEnvVars(imageList images.ImageList) error { + registryURL := imageList.RegistryURL + registryParts := strings.Split(registryURL, "/") + if len(registryParts) < 3 { + return fmt.Errorf("invalid registryUrl format in images.json") + } + + os.Setenv("REGISTRY", registryParts[2]) + config.SetRegistry(registryParts[2]) + + if len(imageList.Repositories) > 0 { + os.Setenv("REPOSITORY", imageList.Repositories[0].Repository) + config.SetRepository(imageList.Repositories[0].Repository) + } else { + return fmt.Errorf("no repositories found in images.json") + } + + return nil +} + +// SetUrlConfig sets the URL configuration for the input URL and sets the environment variables +func SetUrlConfig(input string) { + os.Setenv("USER_INPUT", input) + config.SetUserInput(input) + parts := strings.SplitN(input, "://", 2) + scheme := parts[0] + "://" + os.Setenv("SCHEME", scheme) + config.SetScheme(scheme) + registryAndPath := parts[1] + registryParts := strings.Split(registryAndPath, "/") + os.Setenv("REGISTRY", registryParts[0]) + config.SetRegistry(registryParts[0]) + os.Setenv("API_VERSION", registryParts[1]) + config.SetAPIVersion(registryParts[1]) + os.Setenv("REPOSITORY", registryParts[2]) + config.SetRepository(registryParts[2]) + os.Setenv("IMAGE", registryParts[3]) + config.SetImage(registryParts[3]) +} diff --git a/main.go b/main.go index e5cb0aa..a477a34 100644 --- a/main.go +++ b/main.go @@ -2,41 +2,26 @@ package main import ( "context" - "encoding/json" - "errors" "fmt" - "net" - "net/url" "os" "os/signal" - "path/filepath" - "strings" "syscall" "container-registry.com/harbor-satellite/internal/config" + "container-registry.com/harbor-satellite/internal/images" "container-registry.com/harbor-satellite/internal/replicate" "container-registry.com/harbor-satellite/internal/satellite" "container-registry.com/harbor-satellite/internal/server" "container-registry.com/harbor-satellite/internal/store" + "container-registry.com/harbor-satellite/internal/utils" "container-registry.com/harbor-satellite/logger" - "container-registry.com/harbor-satellite/registry" "golang.org/x/sync/errgroup" - "github.com/spf13/viper" - "github.com/joho/godotenv" + "github.com/rs/zerolog" + "github.com/spf13/viper" ) -type ImageList struct { - RegistryURL string `json:"registryUrl"` - Repositories []struct { - Repository string `json:"repository"` - Images []struct { - Name string `json:"name"` - } `json:"images"` - } `json:"repositories"` -} - func main() { if err := run(); err != nil { fmt.Fprintf(os.Stderr, "Error: %v\n", err) @@ -45,158 +30,138 @@ func main() { } func run() error { - config, err := config.LoadConfig() - if err != nil { + // Initialize Config and Logger + if err := initConfig(); err != nil { return err } - var fetcher store.ImageFetcher - ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGTERM) + ctx, cancel := setupContext() defer cancel() - g, ctx := errgroup.WithContext(ctx) - log := logger.SetupLogger(ctx, config.LogLevel) + g, ctx := errgroup.WithContext(ctx) + log := logger.SetupLogger(ctx, config.GetLogLevel()) log.Info().Msg("Satellite starting") + // Set up router and app + app := setupServerApp(ctx, log) + app.SetupRoutes() + app.SetupServer(g) + + // Handle registry setup + if err := handleRegistrySetup(g, log, cancel); err != nil { + return err + } + + // Process Input (file or URL) + fetcher, err := processInput(ctx, log) + if err != nil { + return err + } + + // Load environment and start satellite + if err := godotenv.Load(); err != nil { + log.Error().Err(err).Msg("Error loading .env file") + return err + } + + ctx, storer := store.NewInMemoryStore(ctx, fetcher) + replicator := replicate.NewReplicator(ctx) + satelliteService := satellite.NewSatellite(ctx, storer, replicator) + + g.Go(func() error { + return satelliteService.Run(ctx) + }) + + log.Info().Msg("Satellite running") + return g.Wait() +} + +func initConfig() error { + if err := config.InitConfig(); err != nil { + return fmt.Errorf("error initializing config: %w", err) + } + return nil +} + +func setupContext() (context.Context, context.CancelFunc) { + ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT) + return ctx, cancel +} + +func setupServerApp(ctx context.Context, log *zerolog.Logger) *server.App { router := server.NewDefaultRouter("/api/v1") router.Use(server.LoggingMiddleware) - app := server.NewApp( + return server.NewApp( router, ctx, log, - config, + config.AppConfig, &server.MetricsRegistrar{}, &server.DebugRegistrar{}, &satellite.SatelliteRegistrar{}, ) +} - app.SetupRoutes() - app.SetupServer(g) - bringOwnRegistry := viper.GetBool("bring_own_registry") - if bringOwnRegistry { - registryAdr := viper.GetString("own_registry_adr") - - // Validate registryAdr format - ip := net.ParseIP(registryAdr) - if ip == nil { - log.Error().Msg("Invalid IP address") - return errors.New("invalid IP address") - } - if ip.To4() != nil { - log.Info().Msg("IP address is valid IPv4") - } else { - log.Error().Msg("IP address is IPv6 format and unsupported") - return errors.New("IP address is IPv6 format and unsupported") +func handleRegistrySetup(g *errgroup.Group, log *zerolog.Logger, cancel context.CancelFunc) error { + if config.GetOwnRegistry() { + if err := utils.HandleOwnRegistry(); err != nil { + log.Error().Err(err).Msg("Error handling own registry") + return err } - registryPort := viper.GetString("own_registry_port") - os.Setenv("ZOT_URL", registryAdr+":"+registryPort) } else { log.Info().Msg("Launching default registry") g.Go(func() error { - launch, err := registry.LaunchRegistry(viper.GetString("zotConfigPath")) - if launch { - cancel() - return err - } - if err != nil { + if err := utils.LaunchDefaultZotRegistry(); err != nil { + log.Error().Err(err).Msg("Error launching default registry") cancel() - log.Error().Err(err).Msg("Failed to launch default registry") return err } + cancel() return nil }) } + return nil +} +func processInput(ctx context.Context, log *zerolog.Logger) (store.ImageFetcher, error) { input := viper.GetString("url_or_file") - parsedURL, err := url.Parse(input) - if err != nil || parsedURL.Scheme == "" { - if strings.ContainsAny(input, "\\:*?\"<>|") { - log.Error().Msg("Path contains invalid characters. Please check the configuration.") - return err + if !utils.IsValidURL(config.GetInput()) { + log.Info().Msg("Input is not a valid URL, checking if it is a file path") + if err := validateFilePath(config.GetInput(), log); err != nil { + return nil, err } - dir, err := os.Getwd() - if err != nil { - log.Error().Err(err).Msg("Error getting current directory") - return err - } - absPath := filepath.Join(dir, input) - if _, err := os.Stat(absPath); os.IsNotExist(err) { - log.Error().Err(err).Msg("No URL or file found. Please check the configuration.") - return err - } - log.Info().Msg("Input is a valid file path.") - fetcher = store.FileImageListFetcher(ctx, input) - os.Setenv("USER_INPUT", input) - - // Parse images.json and set environment variables - file, err := os.Open(absPath) - if err != nil { - log.Error().Err(err).Msg("Error opening images.json file") - return err - } - defer file.Close() + return setupFileFetcher(ctx, log) + } - var imageList ImageList - if err := json.NewDecoder(file).Decode(&imageList); err != nil { - log.Error().Err(err).Msg("Error decoding images.json file") - return err - } + log.Info().Msg("Input is a valid URL") + fetcher := store.RemoteImageListFetcher(ctx, input) + utils.SetUrlConfig(input) + return fetcher, nil +} - registryURL := imageList.RegistryURL - registryParts := strings.Split(registryURL, "/") - if len(registryParts) < 3 { - log.Error().Msg("Invalid registryUrl format in images.json") - return errors.New("invalid registryUrl format in images.json") - } - registry := registryParts[2] - os.Setenv("REGISTRY", registry) - - if len(imageList.Repositories) > 0 { - repository := imageList.Repositories[0].Repository - os.Setenv("REPOSITORY", repository) - } else { - log.Error().Msg("No repositories found in images.json") - return errors.New("no repositories found in images.json") - } - } else { - log.Info().Msg("Input is a valid URL.") - fetcher = store.RemoteImageListFetcher(ctx, input) - os.Setenv("USER_INPUT", input) - parts := strings.SplitN(input, "://", 2) - scheme := parts[0] + "://" - os.Setenv("SCHEME", scheme) - registryAndPath := parts[1] - registryParts := strings.Split(registryAndPath, "/") - registry := registryParts[0] - os.Setenv("REGISTRY", registry) - apiVersion := registryParts[1] - os.Setenv("API_VERSION", apiVersion) - repository := registryParts[2] - os.Setenv("REPOSITORY", repository) - image := registryParts[3] - os.Setenv("IMAGE", image) +func validateFilePath(path string, log *zerolog.Logger) error { + if utils.HasInvalidPathChars(path) { + log.Error().Msg("Path contains invalid characters") + return fmt.Errorf("invalid file path: %s", path) } - - err = godotenv.Load() - if err != nil { - log.Error().Err(err).Msg("Error loading.env file") - return err + if err := utils.GetAbsFilePath(path); err != nil { + log.Error().Err(err).Msg("No file found") + return fmt.Errorf("no file found: %s", path) } + return nil +} - ctx, storer := store.NewInMemoryStore(ctx, fetcher) - replicator := replicate.NewReplicator(ctx) - s := satellite.NewSatellite(ctx, storer, replicator) - - g.Go(func() error { - return s.Run(ctx) - }) - log.Info().Msg("Satellite running") - - err = g.Wait() - if err != nil { - log.Error().Err(err).Msg("Error running satellite") - return err +func setupFileFetcher(ctx context.Context, log *zerolog.Logger) (store.ImageFetcher, error) { + fetcher := store.FileImageListFetcher(ctx, config.GetInput()) + var imagesList images.ImageList + if err := utils.ParseImagesJsonFile(config.GetInput(), &imagesList); err != nil { + log.Error().Err(err).Msg("Error parsing images.json file") + return nil, err } - return nil + if err := utils.SetRegistryEnvVars(imagesList); err != nil { + log.Error().Err(err).Msg("Error setting registry environment variables") + return nil, err + } + return fetcher, nil }