Merge branch 'main' into importer

src/Paprika.Cli/Program.cs

@@ -39,7 +39,7 @@ public override int Execute(CommandContext context, StatisticsSettings settings)
 
         var stats = new Layout("Stats");
 
-        AnsiConsole.Write("Gathering stats...");
+        AnsiConsole.WriteLine("Gathering stats...");
 
         StatisticsForPagedDb.Report(stats, read);
 

src/Paprika.Runner.Pareto/Program.cs

@@ -185,7 +185,9 @@ public static async Task Main(String[] args)
 
             var state = new StatisticsReporter(TrieType.State);
             var storage = new StatisticsReporter(TrieType.Storage);
-            read.Report(state, storage);
+            var counting = new PageCountingReporter();
+
+            read.Report(state, storage, counting, out _);
 
             spectre.Cancel();
             await reportingTask;

src/Paprika.Runner/StatisticsForPagedDb.cs

@@ -17,13 +17,22 @@ public static void Report(Layout reportTo, IReporting read)
         {
             var state = new StatisticsReporter(TrieType.State);
             var storage = new StatisticsReporter(TrieType.Storage);
+            var ids = new PageCountingReporter();
 
-            read.Report(state, storage);
+            read.Report(state, storage, ids, out var totalAbandoned);
 
             var report = new Layout()
-                .SplitColumns(
-                    BuildReport(state, "State"),
-                    BuildReport(storage, "Storage"));
+                .SplitRows(
+                    new Layout("top")
+                        .SplitColumns(
+                            BuildReport(state, "State"),
+                            BuildReport(storage, "Storage")),
+                    new Layout("bottom")
+                        .Update(new Panel(new Paragraph(
+                            $"- pages used for id mapping: {Page.FormatAsGb(ids.Count)}\n" +
+                                $"- total pages abandoned: {Page.FormatAsGb(totalAbandoned)}\n" +
+                            "")).Header("Other stats").Expand())
+                );
 
             reportTo.Update(new Panel(report).Header("Paprika tree statistics").Expand());
         }

src/Paprika.Tests/Store/MemoryFencingTests.cs

@@ -0,0 +1,62 @@
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+using FluentAssertions;
+using Paprika.Store;
+
+namespace Paprika.Tests.Store;
+
+/// <summary>
+/// Tests ensuring that unmanaged structs do no leak out beyond their sizes.
+/// </summary>
+public class MemoryFencingTests
+{
+    [Test]
+    public void Int_for_sanity()
+    {
+        Alloc<int>(sizeof(int)).Should().Be(0);
+    }
+
+    [Test]
+    public void AbandonedPage()
+    {
+        ref var list = ref Alloc<AbandonedList>(AbandonedList.Size);
+        list.IsFullyEmpty.Should().BeTrue();
+    }
+
+    [TearDown]
+    public unsafe void TearDown()
+    {
+        while (_alignedAllocations.TryPop(out var alloc))
+        {
+            NativeMemory.AlignedFree(alloc.ToPointer());
+        }
+    }
+
+    private readonly Stack<UIntPtr> _alignedAllocations = new();
+
+    private unsafe ref T Alloc<T>(int size, int alignment = sizeof(int))
+        where T : struct
+    {
+        const int fenceSize = 32;
+
+        var sizeTotal = (UIntPtr)size + fenceSize * 2;
+        var memory = NativeMemory.AlignedAlloc(sizeTotal, (UIntPtr)alignment);
+        NativeMemory.Clear(memory, sizeTotal);
+
+        _alignedAllocations.Push((UIntPtr)memory);
+
+        var actualStart = new UIntPtr(memory) + fenceSize;
+
+        Fence(memory);
+        Fence((actualStart + (uint)size).ToPointer());
+
+        return ref Unsafe.AsRef<T>(actualStart.ToPointer());
+
+        static void Fence(void* ptr)
+        {
+            var span = new Span<byte>(ptr, fenceSize);
+            const byte fenceFilling = 0xFF;
+            span.Fill(fenceFilling);
+        }
+    }
+}

src/Paprika.Tests/Store/PagedDbTests.cs

@@ -186,7 +186,22 @@ static void Assert(PagedDb db)
             var state = new StatisticsReporter(TrieType.State);
             var storage = new StatisticsReporter(TrieType.Storage);
 
-            read.Report(state, storage);
+            read.Report(state, storage, new JustLookingReporter(), out _);
+        }
+    }
+
+    private class JustLookingReporter : IReporter
+    {
+        public void ReportDataUsage(PageType type, int pageLevel, int trimmedNibbles, in SlottedArray array)
+        {
+        }
+
+        public void ReportPage(uint ageInBatches, PageType type)
+        {
+        }
+
+        public void ReportLeafOverflowCount(byte count)
+        {
         }
     }
 

src/Paprika/Store/AbandonedList.cs

@@ -1,5 +1,6 @@
 using System.Diagnostics;
 using System.Diagnostics.CodeAnalysis;
+using System.Diagnostics.Contracts;
 using System.Runtime.InteropServices;
 
 namespace Paprika.Store;
@@ -10,15 +11,17 @@ namespace Paprika.Store;
 [StructLayout(LayoutKind.Explicit, Size = Size)]
 public struct AbandonedList
 {
+    /// <summary>
+    /// The start for spans of <see cref="BatchIds"/> and <see cref="Addresses"/>.
+    /// </summary>
     private const int EntriesStart = DbAddress.Size + sizeof(uint);
 
-    private const int Size = Page.PageSize - PageHeader.Size - RootPage.Payload.AbandonedStart - EntriesStart;
+    public const int Size = Page.PageSize - PageHeader.Size - RootPage.Payload.AbandonedStart - EntriesStart;
     private const int EntrySize = sizeof(uint) + DbAddress.Size;
-    private const int MaxCount = Size / EntrySize;
+    private const int MaxCount = (Size - EntriesStart) / EntrySize;
 
     [FieldOffset(0)] private DbAddress Current;
-
-    [FieldOffset(4)] private uint EntriesCount;
+    [FieldOffset(DbAddress.Size)] private uint EntriesCount;
 
     [FieldOffset(EntriesStart)] private uint BatchIdStart;
 
@@ -27,8 +30,6 @@ public struct AbandonedList
     [FieldOffset(MaxCount * sizeof(uint) + EntriesStart)]
     private DbAddress AddressStart;
 
-    private const int NotFound = -1;
-
     private Span<DbAddress> Addresses => MemoryMarshal.CreateSpan(ref AddressStart, MaxCount);
 
     /// <summary>
@@ -207,4 +208,38 @@ public void Accept(IPageVisitor visitor, IPageResolver resolver)
             }
         }
     }
-}
+
+    [Pure]
+    public long GatherTotalAbandoned(IPageResolver resolver)
+    {
+        resolver.Prefetch(Addresses);
+
+        long count = 0;
+
+        foreach (var addr in Addresses[..(int)EntriesCount])
+        {
+            var current = addr;
+            while (current.IsNull == false)
+            {
+                var abandoned = new AbandonedPage(resolver.GetAt(current));
+                count += abandoned.Count;
+                current = abandoned.Next;
+            }
+        }
+
+        return count;
+    }
+
+    public bool IsFullyEmpty
+    {
+        get
+        {
+            const int notFound = -1;
+
+            return Addresses.IndexOfAnyExcept(DbAddress.Null) == notFound &&
+                   BatchIds.IndexOfAnyExcept(default(uint)) == notFound &&
+                   EntriesCount == 0 &&
+                   Current == DbAddress.Null;
+        }
+    }
+}

src/Paprika/Store/DataPage.cs

@@ -303,6 +303,11 @@ private static bool TryGet(IReadOnlyBatchContext batch, scoped in NibblePath key
 
     public void Report(IReporter reporter, IPageResolver resolver, int pageLevel, int trimmedNibbles)
     {
+        resolver.Prefetch(Data.Buckets);
+
+        var slotted = new SlottedArray(Data.DataSpan);
+        reporter.ReportDataUsage(Header.PageType, pageLevel, trimmedNibbles, slotted);
+
         foreach (var bucket in Data.Buckets)
         {
             if (!bucket.IsNull)
@@ -315,8 +320,7 @@ public void Report(IReporter reporter, IPageResolver resolver, int pageLevel, in
             }
         }
 
-        var slotted = new SlottedArray(Data.DataSpan);
-        reporter.ReportDataUsage(Header.PageType, pageLevel, trimmedNibbles, slotted);
+
     }
 
     public void Accept(IPageVisitor visitor, IPageResolver resolver, DbAddress addr)

src/Paprika/Store/IBatchContext.cs

@@ -145,5 +145,17 @@ public interface IPageResolver
     /// Gets the page at given address.
     /// </summary>
     Page GetAt(DbAddress address);
+
     void Prefetch(DbAddress address);
+
+    void Prefetch(ReadOnlySpan<DbAddress> addresses)
+    {
+        foreach (var bucket in addresses)
+        {
+            if (!bucket.IsNull)
+            {
+                Prefetch(bucket);
+            }
+        }
+    }
 }

src/Paprika/Store/IReporter.cs

@@ -1,5 +1,3 @@
-using System.Diagnostics;
-using System.Runtime.InteropServices;
 using HdrHistogram;
 using Paprika.Crypto;
 using Paprika.Data;
@@ -26,7 +24,27 @@ public interface IReporter
 
 public interface IReporting
 {
-    void Report(IReporter state, IReporter storage);
+    void Report(IReporter state, IReporter storage, IReporter ids, out long totalAbandoned);
+}
+
+public class PageCountingReporter : IReporter
+{
+    public long Count { get; private set; }
+
+    public void ReportDataUsage(PageType type, int pageLevel, int trimmedNibbles, in SlottedArray array)
+    {
+        Count++;
+    }
+
+    public void ReportPage(uint ageInBatches, PageType type)
+    {
+        throw new NotImplementedException();
+    }
+
+    public void ReportLeafOverflowCount(byte count)
+    {
+        throw new NotImplementedException();
+    }
 }
 
 public class StatisticsReporter(TrieType trieType) : IReporter

src/Paprika/Store/LeafPage.cs

@@ -294,6 +294,8 @@ public void Report(IReporter reporter, IPageResolver resolver, int pageLevel, in
 
     public void Accept(IPageVisitor visitor, IPageResolver resolver, DbAddress addr)
     {
+        resolver.Prefetch(Data.Buckets);
+
         using var scope = visitor.On(this, addr);
 
         foreach (var bucket in Data.Buckets)

src/Paprika/Store/Page.cs

@@ -116,4 +116,10 @@ public struct PageHeader
 
     public static Page DevOnlyNativeAlloc() =>
         new((byte*)NativeMemory.AlignedAlloc(PageSize, PageSize));
+
+    public static string FormatAsGb(long pageCount)
+    {
+        var sizeInGb = (double)pageCount * PageSize / 1024 / 1024 / 1024;
+        return $"{sizeInGb:F2}GB";
+    }
 }

src/Paprika/Store/PagedDb.cs

@@ -484,14 +484,20 @@ public bool TryGet(scoped in Key key, out ReadOnlySpan<byte> result)
             return root.TryGet(key, this, out result);
         }
 
-        public void Report(IReporter state, IReporter storage)
+        public void Report(IReporter state, IReporter storage, IReporter ids, out long totalAbandoned)
         {
-            if (root.Data.StateRoot.IsNull == false)
+            ref readonly var data = ref root.Data;
+
+            totalAbandoned = 0;
+            totalAbandoned = data.AbandonedList.GatherTotalAbandoned(this);
+
+            if (data.StateRoot.IsNull == false)
             {
                 new Merkle.StateRootPage(GetAt(root.Data.StateRoot)).Report(state, this, 0, 0);
             }
 
-            root.Data.Storage.Report(storage, this, 0, 0);
+            data.Storage.Report(storage, this, 0, 0);
+            data.Ids.Report(ids, this, 0, 0);
         }
 
         public uint BatchId => root.Header.BatchId;

src/Paprika/Store/StorageFanOutPage.cs

@@ -108,6 +108,8 @@ public void Report(IReporter reporter, IPageResolver resolver, int pageLevel, in
 
     public void Accept(IPageVisitor visitor, IPageResolver resolver, DbAddress addr)
     {
+        resolver.Prefetch(Data.Addresses);
+
         using var scope = visitor.On(this, addr);
 
         foreach (var bucket in Data.Addresses)