Merge branch 'main' into importer

NethermindEth · Feb 23, 2024 · 53ad39b · 53ad39b
2 parents fc283a1 + 969ef7d
commit 53ad39b
Show file tree

Hide file tree

Showing 14 changed files with 182 additions and 349 deletions.
diff --git a/src/Paprika.Benchmarks/SlottedArrayBenchmarks.cs b/src/Paprika.Benchmarks/SlottedArrayBenchmarks.cs
@@ -21,7 +21,7 @@ public SlottedArrayBenchmarks()
         while (true)
         {
             BinaryPrimitives.WriteInt32LittleEndian(key, _to);
-            if (map.TrySet(key, key) == false)
+            if (map.TrySet(NibblePath.FromKey(key), key) == false)
             {
                 // filled
                 break;
@@ -45,7 +45,7 @@ public int Write_whole_page_of_data()
         for (int i = 0; i < _to; i++)
         {
             BinaryPrimitives.WriteInt32LittleEndian(key, i);
-            if (map.TrySet(key, key))
+            if (map.TrySet(NibblePath.FromKey(key), key))
             {
                 count++;
             }
@@ -66,7 +66,7 @@ public int Read_existing_keys()
         for (var i = 0; i < _to; i++)
         {
             BinaryPrimitives.WriteInt32LittleEndian(key, i);
-            if (map.TryGet(key, out var data))
+            if (map.TryGet(NibblePath.FromKey(key), out var data))
             {
                 result += data.Length;
             }
@@ -87,7 +87,7 @@ public int Read_nonexistent_keys()
         for (int i = _to; i < _to * 2; i++)
         {
             BinaryPrimitives.WriteInt32LittleEndian(key, i);
-            if (map.TryGet(key, out _) == false)
+            if (map.TryGet(NibblePath.FromKey(key), out _) == false)
             {
                 result += 1;
             }

diff --git a/src/Paprika.Tests/Data/SlottedArrayTests.cs b/src/Paprika.Tests/Data/SlottedArrayTests.cs
@@ -49,15 +49,15 @@ public void Enumerate_all()
         using var e = map.EnumerateAll();
 
         e.MoveNext().Should().BeTrue();
-        e.Current.Key.SequenceEqual(key0).Should().BeTrue();
+        e.Current.Key.RawSpan.SequenceEqual(key0).Should().BeTrue();
         e.Current.RawData.SequenceEqual(Data0).Should().BeTrue();
 
         e.MoveNext().Should().BeTrue();
-        e.Current.Key.SequenceEqual(key1).Should().BeTrue();
+        e.Current.Key.RawSpan.SequenceEqual(key1).Should().BeTrue();
         e.Current.RawData.SequenceEqual(Data1).Should().BeTrue();
 
         e.MoveNext().Should().BeTrue();
-        e.Current.Key.SequenceEqual(key2).Should().BeTrue();
+        e.Current.Key.RawSpan.SequenceEqual(key2).Should().BeTrue();
         e.Current.RawData.SequenceEqual(Data2).Should().BeTrue();
 
         e.MoveNext().Should().BeFalse();
@@ -170,19 +170,19 @@ public void Hashing()
 
         // three bytes
         Unique(stackalloc byte[] { 0xA, 0xD, 0xC });
-        Unique(stackalloc byte[] { 0xA, 0xE, 0xC });
-        Unique(stackalloc byte[] { 0xA, 0xF, 0xC });
+        Unique(stackalloc byte[] { 0xAA, 0xEE, 0xCC });
+        Unique(stackalloc byte[] { 0xAA, 0xFF, 0xCC });
 
         // three bytes
-        Unique(stackalloc byte[] { 0xA, 0xD, 0xC, 0x1 });
-        Unique(stackalloc byte[] { 0xA, 0xE, 0xC, 0x1 });
-        Unique(stackalloc byte[] { 0xA, 0xF, 0xC, 0x1 });
+        Unique(stackalloc byte[] { 0xAA, 0xDD, 0xCC, 0x11 });
+        Unique(stackalloc byte[] { 0xAA, 0xEE, 0xCC, 0x11 });
+        Unique(stackalloc byte[] { 0xAA, 0xFF, 0xCC, 0x11 });
 
         return;
 
         void Unique(in ReadOnlySpan<byte> key)
         {
-            var hash = SlottedArray.GetHash(key);
+            var hash = SlottedArray.GetHash(NibblePath.FromKey(key));
             var hex = key.ToHexString(true);
 
             if (hashes.TryAdd(hash, hex) == false)
@@ -195,24 +195,25 @@ void Unique(in ReadOnlySpan<byte> key)
 
 file static class FixedMapTestExtensions
 {
-    public static void SetAssert(this SlottedArray map, in ReadOnlySpan<byte> key, ReadOnlySpan<byte> data, string? because = null)
+    public static void SetAssert(this SlottedArray map, in ReadOnlySpan<byte> key, ReadOnlySpan<byte> data,
+        string? because = null)
     {
-        map.TrySet(key, data).Should().BeTrue(because ?? "TrySet should succeed");
+        map.TrySet(NibblePath.FromKey(key), data).Should().BeTrue(because ?? "TrySet should succeed");
     }
 
     public static void DeleteAssert(this SlottedArray map, in ReadOnlySpan<byte> key)
     {
-        map.Delete(key).Should().BeTrue("Delete should succeed");
+        map.Delete(NibblePath.FromKey(key)).Should().BeTrue("Delete should succeed");
     }
 
     public static void GetAssert(this SlottedArray map, in ReadOnlySpan<byte> key, ReadOnlySpan<byte> expected)
     {
-        map.TryGet(key, out var actual).Should().BeTrue();
+        map.TryGet(NibblePath.FromKey(key), out var actual).Should().BeTrue();
         actual.SequenceEqual(expected).Should().BeTrue("Actual data should equal expected");
     }
 
     public static void GetShouldFail(this SlottedArray map, in ReadOnlySpan<byte> key)
     {
-        map.TryGet(key, out var actual).Should().BeFalse("The key should not exist");
+        map.TryGet(NibblePath.FromKey(key), out var actual).Should().BeFalse("The key should not exist");
     }
 }
diff --git a/src/Paprika.Tests/Merkle/RootHashFuzzyTests.cs b/src/Paprika.Tests/Merkle/RootHashFuzzyTests.cs
@@ -93,7 +93,7 @@ public async Task CalculateThenDelete(string test, ulong size)
 
         using var db = PagedDb.NativeMemoryDb(1024 * 1024 * 1024, 2);
         var merkle = new ComputeMerkleBehavior(1, 1);
-        await using var blockchain = new Blockchain(db, merkle, null, new CacheBudget.Options(1000, 16));
+        await using var blockchain = new Blockchain(db, merkle, null, new CacheBudget.Options(1000, 8), new CacheBudget.Options(1000, 8));
 
         // set
         generator.Run(blockchain, 513, false, true);

diff --git a/src/Paprika.Tests/Store/AbandonedTests.cs b/src/Paprika.Tests/Store/AbandonedTests.cs
@@ -59,7 +59,7 @@ public async Task Reuse_in_limited_environment()
     public async Task Work_proper_bookkeeping_when_lots_of_reads()
     {
         const int repeats = 1_000;
-        const int multiplier = 1 + 1; // data page + abandoned page per commit
+        const int multiplier = 2 + 1; // fanout page + data page + abandoned page per commit
         const int historyDepth = 2;
 
         var account = Keccak.EmptyTreeHash;
@@ -100,7 +100,7 @@ public async Task Reuse_in_grow_and_shrink()
 
         byte[] value = [13];
 
-        using var db = PagedDb.NativeMemoryDb(256 * Page.PageSize);
+        using var db = PagedDb.NativeMemoryDb(1024 * Page.PageSize);
 
         for (var i = 0; i < repeats; i++)
         {

diff --git a/src/Paprika.Tests/Store/RootPageTests.cs b/src/Paprika.Tests/Store/RootPageTests.cs
diff --git a/src/Paprika.Tests/Utils/MetricsTests.cs b/src/Paprika.Tests/Utils/MetricsTests.cs
@@ -12,8 +12,8 @@ public class MetricsTests
 {
     private const int Mb = 1024 * 1024;
 
-    [Explicit("Sometimes faulty reporting happens with HDR here")]
     [Test]
+    [Explicit("Sometimes metrics do not report as it's HDR reporting.")]
     public async Task Metrics_should_report()
     {
         using var metrics = new Metrics();

diff --git a/src/Paprika/Data/NibblePath.cs b/src/Paprika/Data/NibblePath.cs
@@ -74,6 +74,14 @@ public static NibblePath FromRawNibbles(ReadOnlySpan<byte> nibbles, Span<byte> w
         return copy;
     }
 
+    /// <summary>
+    /// Creates the nibble path from preamble and raw slice
+    /// </summary>
+    public static NibblePath FromRaw(byte preamble, ReadOnlySpan<byte> slice)
+    {
+        return new NibblePath(slice, preamble & OddBit, preamble >> LengthShift);
+    }
+
     /// <summary>
     /// </summary>
     /// <param name="key"></param>
@@ -139,24 +147,26 @@ public Span<byte> WriteTo(Span<byte> destination)
         return destination.Slice(0, length);
     }
 
+    public byte RawPreamble => (byte)((_odd & OddBit) | (Length << LengthShift));
+
     private int WriteImpl(Span<byte> destination)
     {
         var odd = _odd & OddBit;
-        var lenght = GetSpanLength(Length, _odd);
+        var length = GetSpanLength(Length, _odd);
 
         destination[0] = (byte)(odd | (Length << LengthShift));
 
-        MemoryMarshal.CreateSpan(ref _span, lenght).CopyTo(destination.Slice(PreambleLength));
+        MemoryMarshal.CreateSpan(ref _span, length).CopyTo(destination.Slice(PreambleLength));
 
         // clearing the oldest nibble, if needed
         // yes, it can be branch free
         if (((odd + Length) & 1) == 1)
         {
-            ref var oldest = ref destination[lenght];
+            ref var oldest = ref destination[length];
             oldest = (byte)(oldest & 0b1111_0000);
         }
 
-        return lenght + PreambleLength;
+        return length + PreambleLength;
     }
 
     /// <summary>
@@ -255,14 +265,9 @@ public NibblePath Append(scoped in NibblePath other, Span<byte> workingSet)
     /// <summary>
     /// Gets the raw underlying span behind the path, removing the odd encoding.
     /// </summary>
-    public ReadOnlySpan<byte> RawSpan
-    {
-        get
-        {
-            var lenght = GetSpanLength(Length, _odd);
-            return MemoryMarshal.CreateSpan(ref _span, lenght);
-        }
-    }
+    public ReadOnlySpan<byte> RawSpan => MemoryMarshal.CreateSpan(ref _span, RawSpanLength);
+
+    public int RawSpanLength => GetSpanLength(Length, _odd);
 
     public static ReadOnlySpan<byte> ReadFrom(ReadOnlySpan<byte> source, out NibblePath nibblePath)
     {