diff --git a/packages/api/.gitignore b/packages/api/.gitignore index e2de397ac..add3c1d81 100644 --- a/packages/api/.gitignore +++ b/packages/api/.gitignore @@ -1,3 +1,4 @@ bin mkfcenv.tar.gz -.env \ No newline at end of file +.env +.shared diff --git a/packages/nomad/loki.hcl b/packages/nomad/loki.hcl index 9970eabe3..5eeab44c4 100644 --- a/packages/nomad/loki.hcl +++ b/packages/nomad/loki.hcl @@ -57,8 +57,8 @@ job "loki" { resources { memory_max = 2048 - memory = 1024 - cpu = 512 + memory = 256 + cpu = 256 } template { diff --git a/packages/nomad/otel-collector.hcl b/packages/nomad/otel-collector.hcl index 7c70e82fb..40a460c02 100644 --- a/packages/nomad/otel-collector.hcl +++ b/packages/nomad/otel-collector.hcl @@ -194,6 +194,15 @@ processors: - "nomad_client_unallocated_memory" - "orchestrator.*" - "api.*" + metricstransform: + transforms: + - include: "nomad_client_host_cpu_idle" + match_type: strict + action: update + operations: + - action: aggregate_labels + aggregation_type: sum + label_set: [instance, node_id, node_status, node_pool] attributes/session-proxy: actions: - key: service.name @@ -249,7 +258,7 @@ service: receivers: - prometheus - otlp - processors: [filter, batch] + processors: [filter, batch, metricstransform] exporters: - prometheusremotewrite/grafana_cloud_metrics # metrics/session-proxy: diff --git a/packages/nomad/proxies/client.conf b/packages/nomad/proxies/client.conf index 048ef39f4..3f03ca742 100644 --- a/packages/nomad/proxies/client.conf +++ b/packages/nomad/proxies/client.conf @@ -73,7 +73,7 @@ server { location / { if ($node_ip = "") { # If you set any text, the header will be set to `application/octet-stream` and then browser won't be able to render the content - return 404; + return 404; # Invalid sandbox url } @@ -85,6 +85,14 @@ server { } } +# Mock for sandbox server when the sandbox is not running, 127.0.0.1 is returned by the DNS resolver +server { + listen 3003; + + default_type text/plain; + return 404 'Sandbox is not running or not found.'; +} + server { listen 3001; location /health { diff --git a/packages/orchestrator/cmd/inspect-header/main.go b/packages/orchestrator/cmd/inspect-header/main.go index 1ff50400b..aca20dd85 100644 --- a/packages/orchestrator/cmd/inspect-header/main.go +++ b/packages/orchestrator/cmd/inspect-header/main.go @@ -69,14 +69,7 @@ func main() { fmt.Printf("=======\n") for _, mapping := range h.Mapping { - rangeMessage := fmt.Sprintf("%d-%d", mapping.Offset/h.Metadata.BlockSize, (mapping.Offset+mapping.Length-1)/h.Metadata.BlockSize) - - fmt.Printf( - "%-14s [%11d,%11d) = [%11d,%11d) in %s, %d B\n", - rangeMessage, - mapping.Offset, mapping.Offset+mapping.Length, - mapping.BuildStorageOffset, mapping.BuildStorageOffset+mapping.Length, mapping.BuildId.String(), mapping.Length, - ) + fmt.Println(mapping.Format(h.Metadata.BlockSize)) } fmt.Printf("\nMAPPING SUMMARY\n") diff --git a/packages/orchestrator/cmd/simulate-headers-merge/main.go b/packages/orchestrator/cmd/simulate-headers-merge/main.go new file mode 100644 index 000000000..4a2a1ffee --- /dev/null +++ b/packages/orchestrator/cmd/simulate-headers-merge/main.go @@ -0,0 +1,158 @@ +package main + +import ( + "context" + "flag" + "fmt" + "log" + "os" + + "github.com/e2b-dev/infra/packages/shared/pkg/storage" + "github.com/e2b-dev/infra/packages/shared/pkg/storage/gcs" + "github.com/e2b-dev/infra/packages/shared/pkg/storage/header" + "github.com/google/uuid" +) + +func main() { + baseBuildId := flag.String("base", "", "base build id") + diffBuildId := flag.String("diff", "", "diff build id") + kind := flag.String("kind", "", "'memfile' or 'rootfs'") + visualize := flag.Bool("visualize", false, "visualize the headers") + + flag.Parse() + + baseTemplate := storage.NewTemplateFiles( + "", + *baseBuildId, + "", + "", + false, + ) + + diffTemplate := storage.NewTemplateFiles( + "", + *diffBuildId, + "", + "", + false, + ) + + var baseStoragePath string + var diffStoragePath string + + if *kind == "memfile" { + baseStoragePath = baseTemplate.StorageMemfileHeaderPath() + diffStoragePath = diffTemplate.StorageMemfileHeaderPath() + } else if *kind == "rootfs" { + baseStoragePath = baseTemplate.StorageRootfsHeaderPath() + diffStoragePath = diffTemplate.StorageRootfsHeaderPath() + } else { + log.Fatalf("invalid kind: %s", *kind) + } + + ctx := context.Background() + + baseObj := gcs.NewObject(ctx, gcs.TemplateBucket, baseStoragePath) + diffObj := gcs.NewObject(ctx, gcs.TemplateBucket, diffStoragePath) + + baseHeader, err := header.Deserialize(baseObj) + if err != nil { + log.Fatalf("failed to deserialize base header: %s", err) + } + + diffHeader, err := header.Deserialize(diffObj) + if err != nil { + log.Fatalf("failed to deserialize diff header: %s", err) + } + + fmt.Printf("\nBASE METADATA\n") + fmt.Printf("Storage path %s/%s\n", gcs.TemplateBucket.BucketName(), baseStoragePath) + fmt.Printf("========\n") + + for _, mapping := range baseHeader.Mapping { + fmt.Println(mapping.Format(baseHeader.Metadata.BlockSize)) + } + + if *visualize { + bottomLayers := header.Layers(baseHeader.Mapping) + delete(*bottomLayers, baseHeader.Metadata.BaseBuildId) + + fmt.Println("") + fmt.Println( + header.Visualize( + baseHeader.Mapping, + baseHeader.Metadata.Size, + baseHeader.Metadata.BlockSize, + 128, + bottomLayers, + &map[uuid.UUID]struct{}{ + baseHeader.Metadata.BuildId: {}, + }, + ), + ) + } + + if err := header.ValidateMappings(baseHeader.Mapping, baseHeader.Metadata.Size, baseHeader.Metadata.BlockSize); err != nil { + log.Fatalf("failed to validate base header: %s", err) + } + + fmt.Printf("\nDIFF METADATA\n") + fmt.Printf("Storage path %s/%s\n", gcs.TemplateBucket.BucketName(), diffStoragePath) + fmt.Printf("========\n") + + onlyDiffMappings := make([]*header.BuildMap, 0) + + for _, mapping := range diffHeader.Mapping { + if mapping.BuildId == diffHeader.Metadata.BuildId { + onlyDiffMappings = append(onlyDiffMappings, mapping) + } + } + + for _, mapping := range onlyDiffMappings { + fmt.Println(mapping.Format(baseHeader.Metadata.BlockSize)) + } + + if *visualize { + fmt.Println("") + fmt.Println( + header.Visualize( + onlyDiffMappings, + baseHeader.Metadata.Size, + baseHeader.Metadata.BlockSize, + 128, + nil, + header.Layers(onlyDiffMappings), + ), + ) + } + + mergedHeader := header.MergeMappings(baseHeader.Mapping, onlyDiffMappings) + + fmt.Printf("\n\nMERGED METADATA\n") + fmt.Printf("========\n") + + for _, mapping := range mergedHeader { + fmt.Println(mapping.Format(baseHeader.Metadata.BlockSize)) + } + + if *visualize { + bottomLayers := header.Layers(baseHeader.Mapping) + delete(*bottomLayers, baseHeader.Metadata.BaseBuildId) + + fmt.Println("") + fmt.Println( + header.Visualize( + mergedHeader, + baseHeader.Metadata.Size, + baseHeader.Metadata.BlockSize, + 128, + bottomLayers, + header.Layers(onlyDiffMappings), + ), + ) + } + + if err := header.ValidateMappings(mergedHeader, baseHeader.Metadata.Size, baseHeader.Metadata.BlockSize); err != nil { + fmt.Fprintf(os.Stderr, "\n\n[VALIDATION ERROR]: failed to validate merged header: %s", err) + } +} diff --git a/packages/orchestrator/go.mod b/packages/orchestrator/go.mod index a4b6574f6..0d157927c 100644 --- a/packages/orchestrator/go.mod +++ b/packages/orchestrator/go.mod @@ -15,7 +15,6 @@ require ( github.com/hashicorp/consul/api v1.30.0 github.com/jellydator/ttlcache/v3 v3.3.0 github.com/loopholelabs/userfaultfd-go v0.1.2 - github.com/miekg/dns v1.1.62 github.com/pojntfx/go-nbd v0.3.2 github.com/shirou/gopsutil/v4 v4.24.10 github.com/vishvananda/netlink v1.3.0 @@ -93,6 +92,7 @@ require ( github.com/mdlayher/genetlink v1.3.2 // indirect github.com/mdlayher/netlink v1.7.2 // indirect github.com/mdlayher/socket v0.5.1 // indirect + github.com/miekg/dns v1.1.62 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/mapstructure v1.5.1-0.20220423185008-bf980b35cac4 // indirect github.com/oklog/ulid v1.3.1 // indirect diff --git a/packages/orchestrator/internal/sandbox/build/build.go b/packages/orchestrator/internal/sandbox/build/build.go index bb46cff2c..fe3a5c415 100644 --- a/packages/orchestrator/internal/sandbox/build/build.go +++ b/packages/orchestrator/internal/sandbox/build/build.go @@ -102,6 +102,7 @@ func (b *File) Slice(off, length int64) ([]byte, error) { return nil, fmt.Errorf("failed to get mapping: %w", err) } + // Pass empty huge page when the build id is nil. if *buildID == uuid.Nil { return header.EmptyHugePage, nil } diff --git a/packages/orchestrator/internal/sandbox/build/cache.go b/packages/orchestrator/internal/sandbox/build/cache.go index 999aa0969..e5115b396 100644 --- a/packages/orchestrator/internal/sandbox/build/cache.go +++ b/packages/orchestrator/internal/sandbox/build/cache.go @@ -11,7 +11,7 @@ import ( "github.com/e2b-dev/infra/packages/shared/pkg/storage/gcs" ) -const buildExpiration = time.Hour * 25 +const buildExpiration = time.Hour * 48 const cachePath = "/orchestrator/build" diff --git a/packages/orchestrator/internal/sandbox/sandbox.go b/packages/orchestrator/internal/sandbox/sandbox.go index 29ca78edb..17126de41 100644 --- a/packages/orchestrator/internal/sandbox/sandbox.go +++ b/packages/orchestrator/internal/sandbox/sandbox.go @@ -84,7 +84,6 @@ func NewSandbox( config.KernelVersion, config.FirecrackerVersion, config.HugePages, - isSnapshot, ) if err != nil { return nil, cleanup, fmt.Errorf("failed to get template snapshot data: %w", err) @@ -393,7 +392,13 @@ func (s *Sandbox) Snapshot( return nil, fmt.Errorf("failed to create memfile diff file: %w", err) } - err = header.CreateDiff(sourceFile, s.files.MemfilePageSize(), memfileDirtyPages, memfileDiffFile) + memfileDirtyPages, emptyDirtyPages, err := header.CreateDiff( + sourceFile, + s.files.MemfilePageSize(), + memfileDirtyPages, + originalMemfile, + memfileDiffFile, + ) if err != nil { return nil, fmt.Errorf("failed to create memfile diff: %w", err) } @@ -402,15 +407,32 @@ func (s *Sandbox) Snapshot( releaseLock() - memfileMapping := header.CreateMapping( - memfileMetadata, + var memfileMappings []*header.BuildMap + + memfileEmptyMapping := header.CreateMapping( + &uuid.Nil, + emptyDirtyPages, + memfileMetadata.BlockSize, + ) + + if memfileEmptyMapping != nil { + memfileMappings = header.MergeMappings( + originalMemfile.Header().Mapping, + memfileEmptyMapping, + ) + + memfileMappings = header.NormalizeMappings(memfileMappings) + } + + memfileDirtyMappings := header.CreateMapping( &buildId, memfileDirtyPages, + memfileMetadata.BlockSize, ) - memfileMappings := header.MergeMappings( + memfileMappings = header.MergeMappings( originalMemfile.Header().Mapping, - memfileMapping, + memfileDirtyMappings, ) snapfile, err := template.NewLocalFile(snapshotTemplateFiles.CacheSnapfilePath()) @@ -469,9 +491,9 @@ func (s *Sandbox) Snapshot( } rootfsMapping := header.CreateMapping( - rootfsMetadata, &buildId, rootfsDirtyBlocks, + rootfsMetadata.BlockSize, ) rootfsMappings := header.MergeMappings( diff --git a/packages/orchestrator/internal/sandbox/template/cache.go b/packages/orchestrator/internal/sandbox/template/cache.go index 45383138a..991913757 100644 --- a/packages/orchestrator/internal/sandbox/template/cache.go +++ b/packages/orchestrator/internal/sandbox/template/cache.go @@ -14,7 +14,7 @@ import ( // How long to keep the template in the cache since the last access. // Should be longer than the maximum possible sandbox lifetime. -const templateExpiration = time.Hour * 25 +const templateExpiration = time.Hour * 72 type Cache struct { cache *ttlcache.Cache[string, Template] @@ -62,7 +62,6 @@ func (c *Cache) GetTemplate( kernelVersion, firecrackerVersion string, hugePages bool, - isSnapshot bool, ) (Template, error) { storageTemplate, err := newTemplateFromStorage( templateId, @@ -70,7 +69,6 @@ func (c *Cache) GetTemplate( kernelVersion, firecrackerVersion, hugePages, - isSnapshot, nil, nil, c.bucket, @@ -125,7 +123,6 @@ func (c *Cache) AddSnapshot( kernelVersion, firecrackerVersion, hugePages, - true, memfileHeader, rootfsHeader, c.bucket, diff --git a/packages/orchestrator/internal/sandbox/template/storage.go b/packages/orchestrator/internal/sandbox/template/storage.go index 0057c5f96..fc594dae2 100644 --- a/packages/orchestrator/internal/sandbox/template/storage.go +++ b/packages/orchestrator/internal/sandbox/template/storage.go @@ -2,6 +2,7 @@ package template import ( "context" + "errors" "fmt" "github.com/google/uuid" @@ -23,20 +24,23 @@ func NewStorage( buildId string, fileType build.DiffType, blockSize int64, - isSnapshot bool, h *header.Header, bucket *gcs.BucketHandle, ) (*Storage, error) { - if isSnapshot && h == nil { + if h == nil { headerObject := gcs.NewObject(ctx, bucket, buildId+"/"+string(fileType)+storage.HeaderSuffix) diffHeader, err := header.Deserialize(headerObject) - if err != nil { + if err != nil && !errors.As(gcs.ErrObjectNotExist, err) { return nil, fmt.Errorf("failed to deserialize header: %w", err) } - h = diffHeader - } else if h == nil { + if err == nil { + h = diffHeader + } + } + + if h == nil { object := gcs.NewObject(ctx, bucket, buildId+"/"+string(fileType)) size, err := object.Size() diff --git a/packages/orchestrator/internal/sandbox/template/storage_template.go b/packages/orchestrator/internal/sandbox/template/storage_template.go index 62b817151..6dc92b5b3 100644 --- a/packages/orchestrator/internal/sandbox/template/storage_template.go +++ b/packages/orchestrator/internal/sandbox/template/storage_template.go @@ -20,8 +20,6 @@ type storageTemplate struct { rootfs *utils.SetOnce[*Storage] snapfile *utils.SetOnce[File] - isSnapshot bool - memfileHeader *header.Header rootfsHeader *header.Header localSnapfile *LocalFile @@ -35,7 +33,6 @@ func newTemplateFromStorage( kernelVersion, firecrackerVersion string, hugePages bool, - isSnapshot bool, memfileHeader *header.Header, rootfsHeader *header.Header, bucket *gcs.BucketHandle, @@ -55,7 +52,6 @@ func newTemplateFromStorage( return &storageTemplate{ files: files, localSnapfile: localSnapfile, - isSnapshot: isSnapshot, memfileHeader: memfileHeader, rootfsHeader: rootfsHeader, bucket: bucket, @@ -111,7 +107,6 @@ func (t *storageTemplate) Fetch(ctx context.Context, buildStore *build.DiffStore t.files.BuildId, build.Memfile, t.files.MemfilePageSize(), - t.isSnapshot, t.memfileHeader, t.bucket, ) @@ -134,7 +129,6 @@ func (t *storageTemplate) Fetch(ctx context.Context, buildStore *build.DiffStore t.files.BuildId, build.Rootfs, t.files.RootfsBlockSize(), - t.isSnapshot, t.rootfsHeader, t.bucket, ) diff --git a/packages/orchestrator/internal/server/sandboxes.go b/packages/orchestrator/internal/server/sandboxes.go index f61d3029a..df1160e59 100644 --- a/packages/orchestrator/internal/server/sandboxes.go +++ b/packages/orchestrator/internal/server/sandboxes.go @@ -163,20 +163,22 @@ func (s *server) Delete(ctx context.Context, in *orchestrator.SandboxDeleteReque return nil, status.New(codes.NotFound, errMsg.Error()).Err() } - sbx.Healthcheck(ctx, true) - // Don't allow connecting to the sandbox anymore. s.dns.Remove(in.SandboxId, sbx.Slot.HostIP()) + // Remove the sandbox from the cache to prevent loading it again in API during the time the instance is stopping. + // Old comment: + // Ensure the sandbox is removed from cache. + // Ideally we would rely only on the goroutine defer. + s.sandboxes.Remove(in.SandboxId) + + sbx.Healthcheck(ctx, true) + err := sbx.Stop() if err != nil { fmt.Fprintf(os.Stderr, "error stopping sandbox '%s': %v\n", in.SandboxId, err) } - // Ensure the sandbox is removed from cache. - // Ideally we would rely only on the goroutine defer. - s.sandboxes.Remove(in.SandboxId) - return &emptypb.Empty{}, nil } diff --git a/packages/shared/go.mod b/packages/shared/go.mod index 702d82bc1..eba9030bd 100644 --- a/packages/shared/go.mod +++ b/packages/shared/go.mod @@ -85,6 +85,7 @@ require ( github.com/opentracing/opentracing-go v1.2.0 // indirect github.com/pelletier/go-toml/v2 v2.2.3 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/stretchr/testify v1.10.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.12 // indirect github.com/zclconf/go-cty v1.12.1 // indirect diff --git a/packages/shared/go.sum b/packages/shared/go.sum index e29311cf8..0746b89e0 100644 --- a/packages/shared/go.sum +++ b/packages/shared/go.sum @@ -302,6 +302,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= diff --git a/packages/shared/pkg/dns/server.go b/packages/shared/pkg/dns/server.go index 8f065d291..98f2b6306 100644 --- a/packages/shared/pkg/dns/server.go +++ b/packages/shared/pkg/dns/server.go @@ -14,6 +14,8 @@ import ( const ttl = 0 +const defaultRoutingIP = "127.0.0.1" + type DNS struct { mu sync.Mutex records *smap.Map[string] @@ -51,21 +53,24 @@ func (d *DNS) handleDNSRequest(w resolver.ResponseWriter, r *resolver.Msg) { for _, q := range m.Question { if q.Qtype == resolver.TypeA { + a := &resolver.A{ + Hdr: resolver.RR_Header{ + Name: q.Name, + Rrtype: resolver.TypeA, + Class: resolver.ClassINET, + Ttl: ttl, + }, + } + sandboxID := strings.Split(q.Name, "-")[0] ip, found := d.get(sandboxID) if found { - a := &resolver.A{ - Hdr: resolver.RR_Header{ - Name: q.Name, - Rrtype: resolver.TypeA, - Class: resolver.ClassINET, - Ttl: ttl, - }, - A: net.ParseIP(ip).To4(), - } - - m.Answer = append(m.Answer, a) + a.A = net.ParseIP(ip).To4() + } else { + a.A = net.ParseIP(defaultRoutingIP).To4() } + + m.Answer = append(m.Answer, a) } } diff --git a/packages/shared/pkg/storage/gcs/object.go b/packages/shared/pkg/storage/gcs/object.go index ad9062918..3578d8de7 100644 --- a/packages/shared/pkg/storage/gcs/object.go +++ b/packages/shared/pkg/storage/gcs/object.go @@ -27,6 +27,8 @@ type Object struct { ctx context.Context } +var ErrObjectNotExist = storage.ErrObjectNotExist + func NewObject(ctx context.Context, bucket *storage.BucketHandle, objectPath string) *Object { obj := bucket.Object(objectPath).Retryer( storage.WithMaxAttempts(maxAttempts), @@ -50,7 +52,7 @@ func (o *Object) WriteTo(dst io.Writer) (int64, error) { reader, err := o.object.NewReader(ctx) if err != nil { - return 0, fmt.Errorf("failed to create GCS reader: %w", err) + return 0, err } defer reader.Close() diff --git a/packages/shared/pkg/storage/header/diff.go b/packages/shared/pkg/storage/header/diff.go index 38d61fb95..272b7565a 100644 --- a/packages/shared/pkg/storage/header/diff.go +++ b/packages/shared/pkg/storage/header/diff.go @@ -1,6 +1,7 @@ package header import ( + "bytes" "fmt" "io" @@ -18,20 +19,61 @@ var ( EmptyBlock = make([]byte, RootfsBlockSize) ) -func CreateDiff(source io.ReaderAt, blockSize int64, dirty *bitset.BitSet, diff io.Writer) error { +type Slicer interface { + Slice(off, length int64) ([]byte, error) +} + +func CreateDiff( + source io.ReaderAt, + blockSize int64, + dirty *bitset.BitSet, + base Slicer, + diff io.Writer, +) (*bitset.BitSet, *bitset.BitSet, error) { b := make([]byte, blockSize) + emptyBuf := EmptyBlock + if blockSize == HugepageSize { + emptyBuf = EmptyHugePage + } + + empty := bitset.New(0) + for i, e := dirty.NextSet(0); e; i, e = dirty.NextSet(i + 1) { _, err := source.ReadAt(b, int64(i)*blockSize) if err != nil { - return fmt.Errorf("error reading from source: %w", err) + return nil, nil, fmt.Errorf("error reading from source: %w", err) + } + + if base != nil { + // At this moment the template should be cached locally, because it was used when starting or during running the sandbox—that's why it is dirty. + cacheBlock, err := base.Slice(int64(i)*blockSize, blockSize) + if err != nil { + return nil, nil, fmt.Errorf("error reading from cache: %w", err) + } + + // If the block is the same as in the base it is not dirty. + if bytes.Equal(b, cacheBlock) { + dirty.Clear(uint(i)) + + continue + } + } + + // If the block is empty, we don't need to write it to the diff. + // Because we checked it does not equal to the base, so we keep it separately. + if bytes.Equal(b, emptyBuf) { + dirty.Clear(uint(i)) + empty.Set(uint(i)) + + continue } _, err = diff.Write(b) if err != nil { - return fmt.Errorf("error writing to diff: %w", err) + return nil, nil, fmt.Errorf("error writing to diff: %w", err) } } - return nil + return dirty, empty, nil } diff --git a/packages/shared/pkg/storage/header/mapping.go b/packages/shared/pkg/storage/header/mapping.go index 35f82fc71..d645c9398 100644 --- a/packages/shared/pkg/storage/header/mapping.go +++ b/packages/shared/pkg/storage/header/mapping.go @@ -1,6 +1,10 @@ package header import ( + "fmt" + "os" + "strings" + "github.com/bits-and-blooms/bitset" "github.com/google/uuid" ) @@ -17,9 +21,9 @@ type BuildMap struct { } func CreateMapping( - metadata *Metadata, buildId *uuid.UUID, dirty *bitset.BitSet, + blockSize uint64, ) []*BuildMap { var mappings []*BuildMap @@ -36,9 +40,9 @@ func CreateMapping( if blockLength > 0 { m := &BuildMap{ - Offset: uint64(int64(startBlock) * int64(metadata.BlockSize)), + Offset: uint64(int64(startBlock) * int64(blockSize)), BuildId: *buildId, - Length: uint64(blockLength) * uint64(metadata.BlockSize), + Length: uint64(blockLength) * uint64(blockSize), BuildStorageOffset: buildStorageOffset, } @@ -53,9 +57,9 @@ func CreateMapping( if blockLength > 0 { mappings = append(mappings, &BuildMap{ - Offset: uint64(startBlock) * metadata.BlockSize, + Offset: uint64(startBlock) * blockSize, BuildId: *buildId, - Length: uint64(blockLength) * uint64(metadata.BlockSize), + Length: uint64(blockLength) * blockSize, BuildStorageOffset: buildStorageOffset, }) } @@ -63,8 +67,12 @@ func CreateMapping( return mappings } +// MergeMappings merges two sets of mappings. +// // The mapping are stored in a sorted order. // The baseMapping must cover the whole size. +// +// It returns a new set of mappings that covers the whole size. func MergeMappings( baseMapping []*BuildMap, diffMapping []*BuildMap, @@ -88,13 +96,13 @@ func MergeMappings( base := baseMapping[baseIdx] diff := diffMapping[diffIdx] - if base.Length == 0 { + if base.Length <= 0 { baseIdx++ continue } - if diff.Length == 0 { + if diff.Length <= 0 { diffIdx++ continue @@ -134,15 +142,17 @@ func MergeMappings( // add diff to the result // if right part is not empty, update baseMapping with it, otherwise remove it from the baseMapping if diff.Offset >= base.Offset && diff.Offset+diff.Length <= base.Offset+base.Length { - leftBase := &BuildMap{ - Offset: base.Offset, - Length: diff.Offset - base.Offset, - BuildId: base.BuildId, - // the build storage offset is the same as the base mapping - BuildStorageOffset: base.BuildStorageOffset, - } + leftBaseLength := int64(diff.Offset) - int64(base.Offset) + + if leftBaseLength > 0 { + leftBase := &BuildMap{ + Offset: base.Offset, + Length: uint64(leftBaseLength), + BuildId: base.BuildId, + // the build storage offset is the same as the base mapping + BuildStorageOffset: base.BuildStorageOffset, + } - if leftBase.Length > 0 { mappings = append(mappings, leftBase) } @@ -150,16 +160,17 @@ func MergeMappings( diffIdx++ - rightBaseShift := diff.Offset + diff.Length - base.Offset + rightBaseShift := int64(diff.Offset) + int64(diff.Length) - int64(base.Offset) + rightBaseLength := int64(base.Length) - rightBaseShift - rightBase := &BuildMap{ - Offset: base.Offset + rightBaseShift, - Length: base.Length - rightBaseShift, - BuildId: base.BuildId, - BuildStorageOffset: base.BuildStorageOffset + rightBaseShift, - } + if rightBaseLength > 0 { + rightBase := &BuildMap{ + Offset: base.Offset + uint64(rightBaseShift), + Length: uint64(rightBaseLength), + BuildId: base.BuildId, + BuildStorageOffset: base.BuildStorageOffset + uint64(rightBaseShift), + } - if rightBase.Length > 0 { baseMapping[baseIdx] = rightBase } else { baseIdx++ @@ -171,18 +182,25 @@ func MergeMappings( // base is after diff and there is overlap // add diff to the result // add the right part of base to the baseMapping, it should not be empty because of the check above - if base.Offset+base.Length > diff.Offset { + if base.Offset > diff.Offset { mappings = append(mappings, diff) diffIdx++ - rightBaseShift := diff.Offset + diff.Length - base.Offset + rightBaseShift := int64(diff.Offset) + int64(diff.Length) - int64(base.Offset) + rightBaseLength := int64(base.Length) - rightBaseShift - baseMapping[baseIdx] = &BuildMap{ - Offset: base.Offset + rightBaseShift, - Length: base.Length - rightBaseShift, - BuildId: base.BuildId, - BuildStorageOffset: base.BuildStorageOffset + rightBaseShift, + if rightBaseLength > 0 { + rightBase := &BuildMap{ + Offset: base.Offset + uint64(rightBaseShift), + Length: uint64(rightBaseLength), + BuildId: base.BuildId, + BuildStorageOffset: base.BuildStorageOffset + uint64(rightBaseShift), + } + + baseMapping[baseIdx] = rightBase + } else { + baseIdx++ } continue @@ -190,18 +208,26 @@ func MergeMappings( // diff is after base and there is overlap // add the left part of base to the result, it should not be empty because of the check above - if diff.Offset+diff.Length > base.Offset { - mappings = append(mappings, &BuildMap{ - Offset: base.Offset, - Length: diff.Offset - base.Offset, - BuildId: base.BuildId, - BuildStorageOffset: base.BuildStorageOffset, - }) + if diff.Offset > base.Offset { + leftBaseLength := int64(diff.Offset) - int64(base.Offset) + + if leftBaseLength > 0 { + leftBase := &BuildMap{ + Offset: base.Offset, + Length: uint64(leftBaseLength), + BuildId: base.BuildId, + BuildStorageOffset: base.BuildStorageOffset, + } + + mappings = append(mappings, leftBase) + } baseIdx++ continue } + + fmt.Fprintf(os.Stderr, "invalid case during merge mappings: %+v %+v\n", base, diff) } mappings = append(mappings, baseMapping[baseIdx:]...) @@ -209,3 +235,137 @@ func MergeMappings( return mappings } + +// Join adjanced mappings that have the same buildId. +func NormalizeMappings(mappings []*BuildMap) []*BuildMap { + for i := 0; i < len(mappings); i++ { + if i+1 < len(mappings) && mappings[i].BuildId == mappings[i+1].BuildId { + mappings[i].Length += mappings[i+1].Length + mappings = append(mappings[:i+1], mappings[i+2:]...) + } + } + + return mappings +} + +// Format returns a string representation of the mapping as: +// +// startBlock-endBlock [offset, offset+length) := [buildStorageOffset, buildStorageOffset+length) ⊂ buildId, length in bytes +// +// It is used for debugging and visualization. +func (mapping *BuildMap) Format(blockSize uint64) string { + rangeMessage := fmt.Sprintf("%d-%d", mapping.Offset/blockSize, (mapping.Offset+mapping.Length)/blockSize) + + return fmt.Sprintf( + "%-14s [%11d,%11d) := [%11d,%11d) ⊂ %s, %d B", + rangeMessage, + mapping.Offset, mapping.Offset+mapping.Length, + mapping.BuildStorageOffset, mapping.BuildStorageOffset+mapping.Length, mapping.BuildId.String(), mapping.Length, + ) +} + +const ( + SkippedBlockChar = '░' + DirtyBlockChar1 = '▓' + DirtyBlockChar2 = '█' +) + +// Layers returns a map of buildIds that are present in the mappings. +func Layers(mappings []*BuildMap) *map[uuid.UUID]struct{} { + layers := make(map[uuid.UUID]struct{}) + + for _, mapping := range mappings { + layers[mapping.BuildId] = struct{}{} + } + + return &layers +} + +// Visualize returns a string representation of the mappings as a grid of blocks. +// It is used for debugging and visualization. +// +// You can pass maps to visualize different groups of buildIds. +func Visualize(mappings []*BuildMap, size, blockSize, cols uint64, bottomGroup, topGroup *map[uuid.UUID]struct{}) string { + output := make([]rune, size/blockSize) + + for outputIdx := range output { + output[outputIdx] = SkippedBlockChar + } + + for _, mapping := range mappings { + for block := uint64(0); block < mapping.Length/blockSize; block++ { + if bottomGroup != nil { + if _, ok := (*bottomGroup)[mapping.BuildId]; ok { + output[mapping.Offset/blockSize+block] = DirtyBlockChar1 + } + } + + if topGroup != nil { + if _, ok := (*topGroup)[mapping.BuildId]; ok { + output[mapping.Offset/blockSize+block] = DirtyBlockChar2 + } + } + } + } + + lineOutput := make([]string, 0) + + for i := uint64(0); i < size/blockSize; i += cols { + if i+cols <= uint64(len(output)) { + lineOutput = append(lineOutput, string(output[i:i+cols])) + } else { + lineOutput = append(lineOutput, string(output[i:])) + } + } + + return strings.Join(lineOutput, "\n") +} + +// ValidateMappings validates the mappings. +// It is used to check if the mappings are valid. +// +// It checks if the mappings are contiguous and if the length of each mapping is a multiple of the block size. +// It also checks if the mappings cover the whole size. +func ValidateMappings(mappings []*BuildMap, size, blockSize uint64) error { + var currentOffset uint64 + + for _, mapping := range mappings { + if currentOffset != mapping.Offset { + return fmt.Errorf("mapping validation failed: the following mapping\n- %s\ndoes not start at the correct offset: expected %d (block %d), got %d (block %d)\n", mapping.Format(blockSize), currentOffset, currentOffset/blockSize, mapping.Offset, mapping.Offset/blockSize) + } + + if mapping.Length%blockSize != 0 { + return fmt.Errorf("mapping validation failed: the following mapping\n- %s\nhas an invalid length: %d. It should be a multiple of block size: %d\n", mapping.Format(blockSize), mapping.Length, blockSize) + } + + if currentOffset+mapping.Length > size { + return fmt.Errorf("mapping validation failed: the following mapping\n- %s\ngoes beyond the size: %d (current offset) + %d (length) > %d (size)\n", mapping.Format(blockSize), currentOffset, mapping.Length, size) + } + + currentOffset += mapping.Length + } + + if currentOffset != size { + return fmt.Errorf("mapping validation failed: the following mapping\n- %s\ndoes not cover the whole size: %d (current offset) != %d (size)\n", mappings[len(mappings)-1].Format(blockSize), currentOffset, size) + } + + return nil +} + +func (mapping *BuildMap) Equal(other *BuildMap) bool { + return mapping.Offset == other.Offset && mapping.Length == other.Length && mapping.BuildId == other.BuildId +} + +func Equal(a, b []*BuildMap) bool { + if len(a) != len(b) { + return false + } + + for i := range a { + if !a[i].Equal(b[i]) { + return false + } + } + + return true +} diff --git a/packages/shared/pkg/storage/header/mapping_test.go b/packages/shared/pkg/storage/header/mapping_test.go new file mode 100644 index 000000000..c6ade5606 --- /dev/null +++ b/packages/shared/pkg/storage/header/mapping_test.go @@ -0,0 +1,303 @@ +package header + +import ( + "testing" + + "github.com/google/uuid" + + "github.com/stretchr/testify/require" +) + +var ignoreID = uuid.Nil +var baseID = uuid.New() +var diffID = uuid.New() + +var blockSize = uint64(2 << 20) + +var size = 8 * blockSize + +var simpleBase = []*BuildMap{ + { + Offset: 0, + Length: 2 * blockSize, + BuildId: ignoreID, + }, + { + Offset: 2 * blockSize, + Length: 4 * blockSize, + BuildId: baseID, + }, + { + Offset: 6 * blockSize, + Length: 2 * blockSize, + BuildId: ignoreID, + }, +} + +func TestMergeMappingsRemoveEmpty(t *testing.T) { + diff := []*BuildMap{ + { + Offset: 0, + Length: 0, + BuildId: ignoreID, + }, + } + + m := MergeMappings(simpleBase, diff) + + require.True(t, Equal(m, []*BuildMap{ + { + Offset: 0, + Length: 2 * blockSize, + BuildId: ignoreID, + }, + { + Offset: 2 * blockSize, + Length: 4 * blockSize, + BuildId: baseID, + }, + { + Offset: 6 * blockSize, + Length: 2 * blockSize, + BuildId: ignoreID, + }, + })) + + err := ValidateMappings(m, size, blockSize) + + require.NoError(t, err) +} + +func TestMergeMappingsBaseBeforeDiffNoOverlap(t *testing.T) { + diff := []*BuildMap{ + { + Offset: 7 * blockSize, + Length: 1 * blockSize, + BuildId: diffID, + }, + } + + m := MergeMappings(simpleBase, diff) + + require.True(t, Equal(m, []*BuildMap{ + { + Offset: 0, + Length: 2 * blockSize, + BuildId: ignoreID, + }, + { + Offset: 2 * blockSize, + Length: 4 * blockSize, + BuildId: baseID, + }, + { + Offset: 6 * blockSize, + Length: 1 * blockSize, + BuildId: ignoreID, + }, + { + Offset: 7 * blockSize, + Length: 1 * blockSize, + BuildId: diffID, + }, + })) + + err := ValidateMappings(m, size, blockSize) + + require.NoError(t, err) +} + +func TestMergeMappingsDiffBeforeBaseNoOverlap(t *testing.T) { + diff := []*BuildMap{ + { + Offset: 0, + Length: 1 * blockSize, + BuildId: diffID, + }, + } + + m := MergeMappings(simpleBase, diff) + + require.True(t, Equal(m, []*BuildMap{ + { + Offset: 0, + Length: 1 * blockSize, + BuildId: diffID, + }, + { + Offset: 1 * blockSize, + Length: 1 * blockSize, + BuildId: ignoreID, + }, + { + Offset: 2 * blockSize, + Length: 4 * blockSize, + BuildId: baseID, + }, + { + Offset: 6 * blockSize, + Length: 2 * blockSize, + BuildId: ignoreID, + }, + })) + + err := ValidateMappings(m, size, blockSize) + + require.NoError(t, err) +} + +func TestMergeMappingsBaseInsideDiff(t *testing.T) { + diff := []*BuildMap{ + { + Offset: 1 * blockSize, + Length: 5 * blockSize, + BuildId: diffID, + }, + } + + m := MergeMappings(simpleBase, diff) + + require.True(t, Equal(m, []*BuildMap{ + { + Offset: 0, + Length: 1 * blockSize, + BuildId: ignoreID, + }, + { + Offset: 1 * blockSize, + Length: 5 * blockSize, + BuildId: diffID, + }, + { + Offset: 6 * blockSize, + Length: 2 * blockSize, + BuildId: ignoreID, + }, + })) + + err := ValidateMappings(m, size, blockSize) + + require.NoError(t, err) +} + +func TestMergeMappingsDiffInsideBase(t *testing.T) { + diff := []*BuildMap{ + { + Offset: 3 * blockSize, + Length: 1 * blockSize, + BuildId: diffID, + }, + } + + m := MergeMappings(simpleBase, diff) + + require.True(t, Equal(m, []*BuildMap{ + { + Offset: 0, + Length: 2 * blockSize, + BuildId: ignoreID, + }, + { + Offset: 2 * blockSize, + Length: 1 * blockSize, + BuildId: baseID, + }, + { + Offset: 3 * blockSize, + Length: 1 * blockSize, + BuildId: diffID, + }, + { + Offset: 4 * blockSize, + Length: 2 * blockSize, + BuildId: baseID, + }, + { + Offset: 6 * blockSize, + Length: 2 * blockSize, + BuildId: ignoreID, + }, + })) + + err := ValidateMappings(m, size, blockSize) + + require.NoError(t, err) +} + +func TestMergeMappingsBaseAfterDiffWithOverlap(t *testing.T) { + diff := []*BuildMap{ + { + Offset: 1 * blockSize, + Length: 4 * blockSize, + BuildId: diffID, + }, + } + + m := MergeMappings(simpleBase, diff) + + require.True(t, Equal(m, []*BuildMap{ + { + Offset: 0, + Length: 1 * blockSize, + BuildId: ignoreID, + }, + { + Offset: 1 * blockSize, + Length: 4 * blockSize, + BuildId: diffID, + }, + { + Offset: 5 * blockSize, + Length: 1 * blockSize, + BuildId: baseID, + }, + { + Offset: 6 * blockSize, + Length: 2 * blockSize, + BuildId: ignoreID, + }, + })) + + err := ValidateMappings(m, size, blockSize) + + require.NoError(t, err) +} + +func TestMergeMappingsDiffAfterBaseWithOverlap(t *testing.T) { + diff := []*BuildMap{ + { + Offset: 3 * blockSize, + Length: 4 * blockSize, + BuildId: diffID, + }, + } + + m := MergeMappings(simpleBase, diff) + + require.True(t, Equal(m, []*BuildMap{ + { + Offset: 0, + Length: 2 * blockSize, + BuildId: ignoreID, + }, + { + Offset: 2 * blockSize, + Length: 1 * blockSize, + BuildId: baseID, + }, + { + Offset: 3 * blockSize, + Length: 4 * blockSize, + BuildId: diffID, + }, + { + Offset: 7 * blockSize, + Length: 1 * blockSize, + BuildId: ignoreID, + }, + })) + + err := ValidateMappings(m, size, blockSize) + + require.NoError(t, err) +} diff --git a/packages/shared/pkg/storage/template.go b/packages/shared/pkg/storage/template.go index 64afadab0..19a36f6ae 100644 --- a/packages/shared/pkg/storage/template.go +++ b/packages/shared/pkg/storage/template.go @@ -123,6 +123,14 @@ func (t *TemplateFiles) BuildRootfsPath() string { return filepath.Join(t.BuildDir(), RootfsName) } +func (t *TemplateFiles) BuildMemfileDiffPath() string { + return filepath.Join(t.BuildDir(), fmt.Sprintf("%s.diff", MemfileName)) +} + +func (t *TemplateFiles) BuildRootfsDiffPath() string { + return filepath.Join(t.BuildDir(), fmt.Sprintf("%s.diff", RootfsName)) +} + func (t *TemplateFiles) BuildSnapfilePath() string { return filepath.Join(t.BuildDir(), SnapfileName) } diff --git a/packages/template-manager/internal/server/create_template.go b/packages/template-manager/internal/server/create_template.go index 4092d6a37..36a9ba88b 100644 --- a/packages/template-manager/internal/server/create_template.go +++ b/packages/template-manager/internal/server/create_template.go @@ -3,16 +3,20 @@ package server import ( "context" "fmt" + "os" "os/exec" "strconv" "strings" "time" + "github.com/bits-and-blooms/bitset" + "github.com/google/uuid" "go.opentelemetry.io/otel/attribute" "google.golang.org/grpc/metadata" template_manager "github.com/e2b-dev/infra/packages/shared/pkg/grpc/template-manager" "github.com/e2b-dev/infra/packages/shared/pkg/storage" + "github.com/e2b-dev/infra/packages/shared/pkg/storage/header" "github.com/e2b-dev/infra/packages/shared/pkg/telemetry" "github.com/e2b-dev/infra/packages/template-manager/internal/build" "github.com/e2b-dev/infra/packages/template-manager/internal/build/writer" @@ -59,7 +63,7 @@ func (s *serverStore) TemplateCreate(templateRequest *template_manager.TemplateC var err error - // Remove local template files if build fails + // Remove local template files after build ends. defer func() { removeCtx, cancel := context.WithTimeout(context.Background(), cleanupTimeout) defer cancel() @@ -92,14 +96,139 @@ func (s *serverStore) TemplateCreate(templateRequest *template_manager.TemplateC } }() + buildID, err := uuid.Parse(config.BuildID) + if err != nil { + return fmt.Errorf("error parsing build id: %w", err) + } + + // MEMFILE memfilePath := template.BuildMemfilePath() + memfileDiffPath := template.BuildMemfileDiffPath() + + memfileSource, err := os.Open(memfilePath) + if err != nil { + return fmt.Errorf("error opening memfile source: %w", err) + } + + memfileInfo, err := memfileSource.Stat() + if err != nil { + return fmt.Errorf("error getting memfile size: %w", err) + } + + memfileDiffFile, err := os.Create(memfileDiffPath) + if err != nil { + return fmt.Errorf("error creating memfile diff file: %w", err) + } + + memfileDirtyPages := bitset.New(0) + memfileDirtyPages.FlipRange(0, uint(header.TotalBlocks(memfileInfo.Size(), template.MemfilePageSize()))) + + memfileDirtyPages, emptyDirtyPages, err := header.CreateDiff( + memfileSource, + template.MemfilePageSize(), + memfileDirtyPages, + nil, + memfileDiffFile, + ) + + memfileDirtyMappings := header.CreateMapping( + &buildID, + memfileDirtyPages, + uint64(template.MemfilePageSize()), + ) + + memfileEmptyMappings := header.CreateMapping( + &uuid.Nil, + emptyDirtyPages, + uint64(template.MemfilePageSize()), + ) + + memfileMappings := header.MergeMappings(memfileDirtyMappings, memfileEmptyMappings) + + memfileMetadata := &header.Metadata{ + Version: 1, + Generation: 0, + BlockSize: uint64(template.MemfilePageSize()), + Size: uint64(memfileInfo.Size()), + BuildId: buildID, + BaseBuildId: buildID, + } + + memfileHeader := header.NewHeader( + memfileMetadata, + memfileMappings, + ) + + // ROOTFS rootfsPath := template.BuildRootfsPath() + rootfsDiffPath := template.BuildRootfsDiffPath() + + rootfsSource, err := os.Open(rootfsPath) + if err != nil { + return fmt.Errorf("error opening rootfs source: %w", err) + } + + rootfsInfo, err := rootfsSource.Stat() + if err != nil { + return fmt.Errorf("error getting rootfs size: %w", err) + } + + rootfsDiffFile, err := os.Create(rootfsDiffPath) + if err != nil { + return fmt.Errorf("error creating rootfs diff file: %w", err) + } + + rootfsDirtyBlocks := bitset.New(0) + rootfsDirtyBlocks.FlipRange(0, uint(header.TotalBlocks(rootfsInfo.Size(), template.RootfsBlockSize()))) + + rootfsDirtyBlocks, emptyDirtyBlocks, err := header.CreateDiff( + rootfsSource, + template.RootfsBlockSize(), + rootfsDirtyBlocks, + nil, + rootfsDiffFile, + ) + + rootfsDirtyMappings := header.CreateMapping( + &buildID, + rootfsDirtyBlocks, + uint64(template.RootfsBlockSize()), + ) + + rootfsEmptyMappings := header.CreateMapping( + &uuid.Nil, + emptyDirtyBlocks, + uint64(template.RootfsBlockSize()), + ) + + rootfsMappings := header.MergeMappings(rootfsDirtyMappings, rootfsEmptyMappings) + + rootfsMetadata := &header.Metadata{ + Version: 1, + Generation: 0, + BlockSize: uint64(template.RootfsBlockSize()), + Size: uint64(rootfsInfo.Size()), + BuildId: buildID, + BaseBuildId: buildID, + } + + rootfsHeader := header.NewHeader( + rootfsMetadata, + rootfsMappings, + ) + + // UPLOAD + b := storage.NewTemplateBuild( + memfileHeader, + rootfsHeader, + template.TemplateFiles, + ) - upload := buildStorage.Upload( + upload := b.Upload( childCtx, template.BuildSnapfilePath(), - &memfilePath, - &rootfsPath, + &memfileDiffPath, + &rootfsDiffPath, ) cmd := exec.Command(storage.HostEnvdPath, "-version")