Skip to content

Commit

Permalink
refactor: Implement optional service image and sync without remote do…
Browse files Browse the repository at this point in the history 
…cker registry
  • Loading branch information
@yarlson
yarlson committed Nov 20, 2024
1 parent d87707f commit deb2bf7
Show file tree
Hide file tree
Showing 4 changed files with 66 additions and 44 deletions.
10 changes: 9 additions & 1 deletion cmd/deploy.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,15 @@ func deployToServer(project string, cfg *config.Config, server config.Server) er
}
defer runner.Close()

syncer := imagesync.NewImageSync(imagesync.Config{}, runner)
localStore, err := os.MkdirTemp("", "dockersync-local")
if err != nil {
return fmt.Errorf("failed to create local store: %w", err)
}

syncer := imagesync.NewImageSync(imagesync.Config{
LocalStore: localStore,
MaxParallel: 1,
}, runner)
deploy := deployment.NewDeployment(runner, syncer)

ctx, cancel := context.WithCancel(context.Background())
Expand Down
28 changes: 22 additions & 6 deletions pkg/deployment/deployment.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ type Runner interface {
}

type ImageSyncer interface {
Sync(ctx context.Context) error
Sync(ctx context.Context, image string) error
}

type Deployment struct {
Expand Down Expand Up @@ -361,8 +361,14 @@ func (d *Deployment) startDependency(project string, dependency *config.Dependen
}

func (d *Deployment) installService(project string, service *config.Service) error {
if _, err := d.pullImage(service.Image); err != nil {
return fmt.Errorf("failed to pull image for %s: %v", service.Image, err)
if service.Image != "" {
if _, err := d.pullImage(service.Image); err != nil {
return fmt.Errorf("failed to pull image for %s: %v", service.Image, err)
}
} else {
if err := d.syncer.Sync(context.Background(), fmt.Sprintf("%s-%s", project, service.Name)); err != nil {
return fmt.Errorf("failed to sync service %s for %s: %v", service.Name, service.Image, err)
}
}

if err := d.startContainer(project, service, ""); err != nil {
Expand All @@ -381,8 +387,14 @@ func (d *Deployment) installService(project string, service *config.Service) err
func (d *Deployment) updateService(project string, service *config.Service) error {
svcName := service.Name

if _, err := d.pullImage(service.Image); err != nil {
return fmt.Errorf("failed to pull new image for %s: %v", svcName, err)
if service.Image != "" {
if _, err := d.pullImage(service.Image); err != nil {
return fmt.Errorf("failed to pull new image for %s: %v", svcName, err)
}
} else {
if err := d.syncer.Sync(context.Background(), fmt.Sprintf("%s-%s", project, service.Name)); err != nil {
return fmt.Errorf("failed to sync service %s for %s: %v", service.Name, service.Image, err)
}
}

if service.Recreate {
Expand Down Expand Up @@ -537,7 +549,11 @@ func (d *Deployment) startContainer(project string, service *config.Service, suf
args = append(args, "--entrypoint", strings.Join(service.Entrypoint, " "))
}

args = append(args, service.Image)
image := service.Image
if image == "" {
image = fmt.Sprintf("%s-%s", project, service.Name)
}
args = append(args, image)

if service.Command != "" {
args = append(args, service.Command)
Expand Down
65 changes: 32 additions & 33 deletions pkg/imagesync/imagesync.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ import (

// Config holds the configuration for the Docker image sync operation.
type Config struct {
ImageName string
LocalStore string
RemoteStore string
MaxParallel int
Expand Down Expand Up @@ -46,8 +45,8 @@ func NewImageSync(cfg Config, runner *remote.Runner) *ImageSync {
}

// Sync performs the Docker image synchronization process.
func (s *ImageSync) Sync(ctx context.Context) error {
needsSync, err := s.compareImages(ctx)
func (s *ImageSync) Sync(ctx context.Context, image string) error {
needsSync, err := s.compareImages(ctx, image)
if err != nil {
return fmt.Errorf("failed to compare images: %w", err)
}
Expand All @@ -60,33 +59,33 @@ func (s *ImageSync) Sync(ctx context.Context) error {
return fmt.Errorf("failed to prepare directories: %w", err)
}

if err := s.exportAndExtractImage(ctx); err != nil {
if err := s.exportAndExtractImage(ctx, image); err != nil {
return fmt.Errorf("failed to export and extract image: %w", err)
}

if err := s.transferMetadata(ctx); err != nil {
if err := s.transferMetadata(ctx, image); err != nil {
return fmt.Errorf("failed to transfer metadata: %w", err)
}

if err := s.syncBlobs(ctx); err != nil {
if err := s.syncBlobs(ctx, image); err != nil {
return fmt.Errorf("failed to sync blobs: %w", err)
}

if err := s.loadRemoteImage(ctx); err != nil {
if err := s.loadRemoteImage(ctx, image); err != nil {
return fmt.Errorf("failed to load remote image: %w", err)
}

return nil
}

// compareImages checks if the image needs to be synced by comparing local and remote versions.
func (s *ImageSync) compareImages(ctx context.Context) (bool, error) {
localInspect, err := s.inspectLocalImage()
func (s *ImageSync) compareImages(ctx context.Context, image string) (bool, error) {
localInspect, err := s.inspectLocalImage(image)
if err != nil {
return false, fmt.Errorf("failed to inspect local image: %w", err)
}

remoteInspect, err := s.inspectRemoteImage(ctx)
remoteInspect, err := s.inspectRemoteImage(ctx, image)
if err != nil {
return true, nil // Assume sync needed if remote inspection fails
}
Expand Down Expand Up @@ -126,8 +125,8 @@ type ImageData struct {
Os string `json:"Os"`
}

func (s *ImageSync) inspectLocalImage() (*ImageData, error) {
cmd := exec.Command("docker", "inspect", s.cfg.ImageName)
func (s *ImageSync) inspectLocalImage(image string) (*ImageData, error) {
cmd := exec.Command("docker", "inspect", image)
output, err := cmd.Output()
if err != nil {
return nil, fmt.Errorf("docker inspect failed: %w", err)
Expand All @@ -145,8 +144,8 @@ func (s *ImageSync) inspectLocalImage() (*ImageData, error) {
return &data[0], nil
}

func (s *ImageSync) inspectRemoteImage(ctx context.Context) (*ImageData, error) {
outputReader, err := s.runner.RunCommand(ctx, "docker", "inspect", s.cfg.ImageName)
func (s *ImageSync) inspectRemoteImage(ctx context.Context, image string) (*ImageData, error) {
outputReader, err := s.runner.RunCommand(ctx, "docker", "inspect", image)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -181,16 +180,16 @@ func (s *ImageSync) prepareDirectories(ctx context.Context) error {
return nil
}

func (s *ImageSync) exportAndExtractImage(ctx context.Context) error {
imageDir := normalizeImageName(s.cfg.ImageName)
func (s *ImageSync) exportAndExtractImage(ctx context.Context, image string) error {
imageDir := normalizeImageName(image)
localPath := filepath.Join(s.cfg.LocalStore, imageDir)

if err := os.MkdirAll(localPath, 0755); err != nil {
return fmt.Errorf("failed to create image directory: %w", err)
}

tarPath := filepath.Join(localPath, "image.tar")
cmd := exec.Command("docker", "save", s.cfg.ImageName, "-o", tarPath)
cmd := exec.Command("docker", "save", image, "-o", tarPath)
if err := cmd.Run(); err != nil {
return fmt.Errorf("failed to save image: %w", err)
}
Expand Down Expand Up @@ -271,13 +270,13 @@ func extractFile(tr *tar.Reader, target string) error {
return nil
}

func (s *ImageSync) syncBlobs(ctx context.Context) error {
localBlobs, err := s.listLocalBlobs()
func (s *ImageSync) syncBlobs(ctx context.Context, image string) error {
localBlobs, err := s.listLocalBlobs(image)
if err != nil {
return err
}

remoteBlobs, err := s.listRemoteBlobs(ctx)
remoteBlobs, err := s.listRemoteBlobs(ctx, image)
if err != nil {
return err
}
Expand All @@ -291,10 +290,10 @@ func (s *ImageSync) syncBlobs(ctx context.Context) error {
}

// Transfer blobs in parallel batches
return s.transferBlobs(ctx, blobsToTransfer)
return s.transferBlobs(ctx, image, blobsToTransfer)
}

func (s *ImageSync) transferBlobs(ctx context.Context, blobs []string) error {
func (s *ImageSync) transferBlobs(ctx context.Context, image string, blobs []string) error {
if len(blobs) == 0 {
return nil
}
Expand All @@ -310,7 +309,7 @@ func (s *ImageSync) transferBlobs(ctx context.Context, blobs []string) error {
semaphore <- struct{}{}
defer func() { <-semaphore }()

if err := s.transferBlob(ctx, blob); err != nil {
if err := s.transferBlob(ctx, image, blob); err != nil {
errChan <- fmt.Errorf("failed to transfer blob %s: %w", blob, err)
}
}(blob)
Expand All @@ -330,9 +329,9 @@ func (s *ImageSync) transferBlobs(ctx context.Context, blobs []string) error {
return nil
}

func (s *ImageSync) loadRemoteImage(ctx context.Context) error {
func (s *ImageSync) loadRemoteImage(ctx context.Context, image string) error {
cmd := fmt.Sprintf("cd %s && tar -cf - . | docker load",
filepath.Join(s.cfg.RemoteStore, normalizeImageName(s.cfg.ImageName)))
filepath.Join(s.cfg.RemoteStore, normalizeImageName(image)))

outputReader, err := s.runner.RunCommand(ctx, cmd)
if err != nil {
Expand Down Expand Up @@ -374,8 +373,8 @@ func contains(slice []string, item string) bool {
}

// listLocalBlobs returns a list of blob hashes from the local blob directory.
func (s *ImageSync) listLocalBlobs() ([]string, error) {
imageDir := normalizeImageName(s.cfg.ImageName)
func (s *ImageSync) listLocalBlobs(image string) ([]string, error) {
imageDir := normalizeImageName(image)
blobPath := filepath.Join(s.cfg.LocalStore, imageDir, "blobs", "sha256")

entries, err := os.ReadDir(blobPath)
Expand All @@ -394,8 +393,8 @@ func (s *ImageSync) listLocalBlobs() ([]string, error) {
}

// listRemoteBlobs returns a list of blob hashes from the remote blob directory.
func (s *ImageSync) listRemoteBlobs(ctx context.Context) ([]string, error) {
imageDir := normalizeImageName(s.cfg.ImageName)
func (s *ImageSync) listRemoteBlobs(ctx context.Context, image string) ([]string, error) {
imageDir := normalizeImageName(image)
blobPath := filepath.Join(s.cfg.RemoteStore, imageDir, "blobs", "sha256")

output, err := s.runner.RunCommand(ctx, "ls", blobPath)
Expand All @@ -413,8 +412,8 @@ func (s *ImageSync) listRemoteBlobs(ctx context.Context) ([]string, error) {
}

// transferBlob copies a single blob to the remote host.
func (s *ImageSync) transferBlob(ctx context.Context, blob string) error {
imageDir := normalizeImageName(s.cfg.ImageName)
func (s *ImageSync) transferBlob(ctx context.Context, image string, blob string) error {
imageDir := normalizeImageName(image)
localPath := filepath.Join(s.cfg.LocalStore, imageDir, "blobs", "sha256", blob)
remotePath := filepath.Join(s.cfg.RemoteStore, imageDir, "blobs", "sha256", blob)

Expand All @@ -427,8 +426,8 @@ func (s *ImageSync) transferBlob(ctx context.Context, blob string) error {
}

// transferMetadata copies the image metadata files to the remote host.
func (s *ImageSync) transferMetadata(ctx context.Context) error {
imageDir := normalizeImageName(s.cfg.ImageName)
func (s *ImageSync) transferMetadata(ctx context.Context, image string) error {
imageDir := normalizeImageName(image)
localDir := filepath.Join(s.cfg.LocalStore, imageDir)
remoteDir := filepath.Join(s.cfg.RemoteStore, imageDir)

Expand Down
7 changes: 3 additions & 4 deletions pkg/imagesync/imagesync_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,6 @@ func TestImageSync(t *testing.T) {

// Initialize ImageSync
cfg := Config{
ImageName: testImage,
LocalStore: localStore,
RemoteStore: remoteStore,
MaxParallel: 4,
Expand All @@ -84,7 +83,7 @@ func TestImageSync(t *testing.T) {
// Run sync
t.Log("Running sync...")
ctx := context.Background()
err = sync.Sync(ctx)
err = sync.Sync(ctx, testImage)
require.NoError(t, err)

// Verify image exists on remote
Expand All @@ -99,12 +98,12 @@ func TestImageSync(t *testing.T) {

// Test image comparison
t.Log("Comparing images...")
needsSync, err := sync.compareImages(ctx)
needsSync, err := sync.compareImages(ctx, testImage)
require.NoError(t, err)
require.False(t, needsSync, "Images should be identical after sync")

// Test re-sync with no changes
t.Log("Re-syncing...")
err = sync.Sync(ctx)
err = sync.Sync(ctx, testImage)
require.NoError(t, err)
}

0 comments on commit deb2bf7

Please sign in to comment.