Leaf + overflow (#264)

* Simple leaf page (#261) (#263) This reverts commit 9fbb48b. * Printing for tree structure * overflowing page added * overflowing * buckets slowly introduced * towards the overflow * stats * more * bit vectors and different lengths of overflow * format * overflow is optional * SlottedArray can store metadata * format * more * page lifecycle test asserted * more overflow pages; set refactored * DataPage enumerates leafs first, before flushing the biggest nibble down
NethermindEth · Mar 11, 2024 · 9bab523 · 9bab523
1 parent c98d61a
commit 9bab523
Show file tree

Hide file tree

Showing 38 changed files with 1,848 additions and 975 deletions.
diff --git a/src/Paprika.Importer/Paprika.Importer.csproj b/src/Paprika.Importer/Paprika.Importer.csproj
@@ -15,7 +15,7 @@
     </ItemGroup>
 
     <ItemGroup>
-      <PackageReference Include="Spectre.Console" Version="0.47.1-preview.0.23" />
+      <PackageReference Include="Spectre.Console" Version="0.48.1-preview.0.32" />
     </ItemGroup>
 
 </Project>
diff --git a/src/Paprika.Runner.Pareto/Paprika.Runner.Pareto.csproj b/src/Paprika.Runner.Pareto/Paprika.Runner.Pareto.csproj
@@ -22,7 +22,7 @@
     </ItemGroup>
 
     <ItemGroup>
-      <PackageReference Include="Spectre.Console" Version="0.47.1-preview.0.6" />
+      <PackageReference Include="Spectre.Console" Version="0.48.1-preview.0.32" />
     </ItemGroup>
 
 </Project>
diff --git a/src/Paprika.Runner/Paprika.Runner.csproj b/src/Paprika.Runner/Paprika.Runner.csproj
@@ -15,7 +15,7 @@
 
     <ItemGroup>
       <PackageReference Include="HdrHistogram" Version="2.5.0" />
-      <PackageReference Include="Spectre.Console" Version="0.47.1-preview.0.6" />
+      <PackageReference Include="Spectre.Console" Version="0.48.1-preview.0.32" />
     </ItemGroup>
 
 </Project>
diff --git a/src/Paprika.Runner/StatisticsForPagedDb.cs b/src/Paprika.Runner/StatisticsForPagedDb.cs
@@ -41,9 +41,10 @@ public static void Report(Layout reportTo, IReporting read)
     private static Layout BuildReport(StatisticsReporter reporter, string name)
     {
         var up = new Layout("up");
-        var down = new Layout("down");
+        var sizes = new Layout("down");
+        var leafs = new Layout("leafs");
 
-        var layout = new Layout().SplitRows(up, down);
+        var layout = new Layout().SplitRows(up, sizes, leafs);
 
         var general = $"Number of pages: {reporter.PageCount}";
         up.Update(new Panel(general).Header($"General stats for {name}").Expand());
@@ -67,7 +68,19 @@ private static Layout BuildReport(StatisticsReporter reporter, string name)
                 WriteHistogram(capacity));
         }
 
-        down.Update(t.Expand());
+        sizes.Update(t.Expand());
+
+        var leafsTable = new Table();
+        leafsTable.AddColumn(new TableColumn("Leaf capacity left"));
+        leafsTable.AddColumn(new TableColumn("Leaf->Overflow capacity left"));
+        leafsTable.AddColumn(new TableColumn("Leaf->Overflow count"));
+
+        leafsTable.AddRow(
+            WriteHistogram(reporter.LeafCapacityLeft),
+            WriteHistogram(reporter.LeafOverflowCapacityLeft),
+            WriteHistogram(reporter.LeafOverflowCount));
+
+        leafs.Update(leafsTable.Expand());
 
         return layout;
     }

diff --git a/src/Paprika.Tests/Chain/BlockchainTests.cs b/src/Paprika.Tests/Chain/BlockchainTests.cs
@@ -371,7 +371,7 @@ public async Task Account_destruction_database_flushed()
         blockchain.Finalize(hash);
 
         // Poor man's await on finalization flushed
-        await Task.Delay(500);
+        await blockchain.WaitTillFlush(hash);
 
         using var block2 = blockchain.StartNew(hash);
 

diff --git a/src/Paprika.Tests/Data/BitVector512Tests.cs b/src/Paprika.Tests/Data/BitVector512Tests.cs
@@ -0,0 +1,55 @@
+using FluentAssertions;
+using NUnit.Framework;
+using Paprika.Data;
+
+namespace Paprika.Tests.Data;
+
+public class BitVector1024Tests
+{
+    [Test]
+    public void Set_reset()
+    {
+        var v = new BitVector.Of1024();
+
+        for (int i = 0; i < BitVector.Of1024.Count; i++)
+        {
+            v[i].Should().BeFalse();
+            v[i] = true;
+            v[i].Should().BeTrue();
+            v[i] = false;
+            v[i].Should().BeFalse();
+        }
+    }
+
+    [TestCase(0)]
+    [TestCase(1)]
+    [TestCase(64)]
+    [TestCase(65)]
+    [TestCase(BitVector.Of1024.Count - 1)]
+    [TestCase(BitVector.Of1024.Count)]
+    public void First_not_set(int set)
+    {
+        var v = new BitVector.Of1024();
+
+        for (int i = 0; i < set; i++)
+        {
+            v[i] = true;
+        }
+
+        v.FirstNotSet.Should().Be((ushort)set);
+    }
+
+    [TestCase(1, true)]
+    [TestCase(BitVector.Of1024.Count, false)]
+    public void Any_not_set(int set, bool anyNotSet)
+    {
+        var v = new BitVector.Of1024();
+
+        for (int i = 0; i < set; i++)
+        {
+            v[i] = true;
+        }
+
+        v.HasEmptyBits.Should().Be(anyNotSet);
+    }
+}
diff --git a/src/Paprika.Tests/Data/UshortSlottedArrayTests.cs b/src/Paprika.Tests/Data/UshortSlottedArrayTests.cs
@@ -0,0 +1,108 @@
+using FluentAssertions;
+using NUnit.Framework;
+using Paprika.Crypto;
+using Paprika.Data;
+
+namespace Paprika.Tests.Data;
+
+public class UshortSlottedArrayTests
+{
+    private const ushort Key0 = 14;
+    private const ushort Key1 = 28;
+    private const ushort Key2 = 31;
+
+    private static ReadOnlySpan<byte> Data0 => new byte[] { 23 };
+    private static ReadOnlySpan<byte> Data1 => new byte[] { 29, 31 };
+
+    private static ReadOnlySpan<byte> Data2 => new byte[] { 37, 39 };
+
+    [Test]
+    public void Set_Get_Delete_Get_AnotherSet()
+    {
+        Span<byte> span = stackalloc byte[48];
+        var map = new UShortSlottedArray(span);
+
+        map.SetAssert(Key0, Data0);
+
+        map.GetAssert(Key0, Data0);
+
+        map.DeleteAssert(Key0);
+        map.GetShouldFail(Key0);
+
+        // should be ready to accept some data again
+        map.SetAssert(Key0, Data1, "Should have memory after previous delete");
+        map.GetAssert(Key0, Data1);
+    }
+
+    [Test]
+    public void Defragment_when_no_more_space()
+    {
+        // by trial and error, found the smallest value that will allow to put these two
+        Span<byte> span = stackalloc byte[24];
+        var map = new UShortSlottedArray(span);
+
+        map.SetAssert(Key0, Data0);
+        map.SetAssert(Key1, Data1);
+
+        map.DeleteAssert(Key0);
+
+        map.SetAssert(Key2, Data2, "Should retrieve space by running internally the defragmentation");
+
+        // should contains no key0, key1 and key2 now
+        map.GetShouldFail(Key0);
+
+        map.GetAssert(Key1, Data1);
+        map.GetAssert(Key2, Data2);
+    }
+
+    [Test]
+    public void Update_in_situ()
+    {
+        // by trial and error, found the smallest value that will allow to put these two
+        Span<byte> span = stackalloc byte[48];
+        var map = new UShortSlottedArray(span);
+
+        map.SetAssert(Key1, Data1);
+        map.SetAssert(Key1, Data2);
+
+        map.GetAssert(Key1, Data2);
+    }
+
+    [Test]
+    public void Update_in_resize()
+    {
+        // Update the value, with the next one being bigger.
+        Span<byte> span = stackalloc byte[40];
+        var map = new UShortSlottedArray(span);
+
+        map.SetAssert(Key0, Data0);
+        map.SetAssert(Key0, Data2);
+
+        map.GetAssert(Key0, Data2);
+    }
+}
+
+file static class Extensions
+{
+    public static void SetAssert(this UShortSlottedArray map, ushort key, ReadOnlySpan<byte> data,
+        string? because = null)
+    {
+        map.TrySet(key, data).Should().BeTrue(because ?? "TrySet should succeed");
+    }
+
+    public static void DeleteAssert(this UShortSlottedArray map, ushort key)
+    {
+        map.Delete(key).Should().BeTrue("Delete should succeed");
+    }
+
+    public static void GetAssert(this UShortSlottedArray map, ushort key, ReadOnlySpan<byte> expected)
+    {
+        map.TryGet(key, out var actual).Should().BeTrue();
+        actual.SequenceEqual(expected).Should().BeTrue("Actual data should equal expected");
+    }
+
+    public static void GetShouldFail(this UShortSlottedArray map, ushort key)
+    {
+        map.TryGet(key, out var actual).Should().BeFalse("The key should not exist");
+    }
+}
diff --git a/src/Paprika.Tests/Merkle/AdditionalTests.cs b/src/Paprika.Tests/Merkle/AdditionalTests.cs
@@ -17,7 +17,7 @@ public async Task Account_destruction_same_block()
         const int seed = 17;
         const int storageCount = 32 * 1024;
 
-        using var db = PagedDb.NativeMemoryDb(8 * 1024 * 1024, 2);
+        using var db = PagedDb.NativeMemoryDb(32 * 1024 * 1024, 2);
         var merkle = new ComputeMerkleBehavior(2, 2);
 
         await using var blockchain = new Blockchain(db, merkle);

diff --git a/src/Paprika.Tests/Paprika.Tests.csproj b/src/Paprika.Tests/Paprika.Tests.csproj
@@ -15,6 +15,7 @@
       <PackageReference Include="NUnit3TestAdapter" version="4.3.1" />
       <PackageReference Include="coverlet.collector" Version="3.2.0" />
       <PackageReference Include="NUnit.Analyzers" Version="3.5.0" />
+      <PackageReference Include="Spectre.Console" Version="0.48.1-preview.0.32" />
     </ItemGroup>
 
     <ItemGroup>

diff --git a/src/Paprika.Tests/PrinterTests.cs b/src/Paprika.Tests/PrinterTests.cs