Merge branch 'main' into importer

src/Paprika.Benchmarks/PooledSpanDictionaryBenchmarks.cs

@@ -8,7 +8,7 @@ namespace Paprika.Benchmarks;
 [DisassemblyDiagnoser]
 public class PooledSpanDictionaryBenchmarks
 {
-    private readonly BufferPool _pool = new(1, false);
+    private readonly BufferPool _pool = new(1, BufferPool.PageTracking.None);
     private readonly PooledSpanDictionary _bigDict;
 
     private const int BigDictCount = 32_000;
@@ -35,7 +35,7 @@ public class PooledSpanDictionaryBenchmarks
 
     public PooledSpanDictionaryBenchmarks()
     {
-        _bigDict = new PooledSpanDictionary(new BufferPool(128, false, null));
+        _bigDict = new PooledSpanDictionary(new BufferPool(128, BufferPool.PageTracking.None, null));
 
         Span<byte> key = stackalloc byte[4];
 
@@ -45,7 +45,7 @@ public PooledSpanDictionaryBenchmarks()
             _bigDict.Set(key, i, Value32Bytes, 1);
         }
 
-        _varLengthKeys = new PooledSpanDictionary(new BufferPool(32, false, null));
+        _varLengthKeys = new PooledSpanDictionary(new BufferPool(32, BufferPool.PageTracking.None, null));
 
         for (uint i = 0; i < VarLengthKeyCollisions; i++)
         {

src/Paprika.Cli/VerifyWholeTreeSettings.cs

@@ -16,12 +16,13 @@ public override int Execute(CommandContext context, VerifyWholeTreeSettings sett
             using var read = db.BeginReadOnlyBatch();
             using var latest = Blockchain.StartReadOnlyLatestFromDb(db);
 
-            AnsiConsole.WriteLine("Checking whole tree...");
 
-            var keccak = new ComputeMerkleBehavior().CalculateStateRootHash(latest);
+            AnsiConsole.WriteLine($"The latest state root hash persisted: {latest.Hash}.");
+            AnsiConsole.WriteLine("Verification of the whole state tree in progress...");
 
-            AnsiConsole.WriteLine($"Keccak {keccak.ToString()}");
+            var keccak = new ComputeMerkleBehavior().CalculateStateRootHash(latest);
 
+            AnsiConsole.WriteLine($"The computed state root hash {keccak.ToString()}");
 
             return 0;
         }

src/Paprika.Tests/Chain/BlockchainTests.cs

@@ -63,6 +63,51 @@ public async Task Simple()
         block3A.GetAccount(Key0).Should().Be(account1A);
     }
 
+    [Test]
+    public async Task Storage_setter()
+    {
+        var account1 = new Account(1, 1);
+        var account2 = new Account(2, 2);
+
+        using var db = PagedDb.NativeMemoryDb(1 * Mb, 2);
+
+        await using var blockchain = new Blockchain(db, new ComputeMerkleBehavior());
+
+        var storageKey1 = Key1;
+        byte[] storageValue1 = [1];
+
+        var storageKey2 = Key2;
+        byte[] storageValue2 = [7];
+
+        using var block1 = blockchain.StartNew(Keccak.EmptyTreeHash);
+
+        // account Key0
+        block1.SetAccount(Key0, account1);
+        block1.SetStorage(Key0, storageKey1, storageValue1);
+        block1.SetStorage(Key0, storageKey2, storageValue2);
+
+        // account Key1 using StorageSetter
+        block1.SetAccount(Key1, account2);
+        var setter = block1.GetStorageSetter(Key1);
+        setter.SetStorage(storageKey1, storageValue1);
+        setter.SetStorage(storageKey2, storageValue2);
+
+        var keccak1 = block1.Commit(1);
+
+        // Assert that the roots are the same.
+        using var reader = blockchain.StartReadOnly(keccak1);
+
+        var read1 = reader.GetAccount(Key0);
+        read1.Balance.Should().Be(account1.Balance);
+        read1.Nonce.Should().Be(account1.Nonce);
+
+        var read2 = reader.GetAccount(Key1);
+        read2.Balance.Should().Be(account2.Balance);
+        read2.Nonce.Should().Be(account2.Nonce);
+
+        read1.StorageRootHash.Should().Be(read2.StorageRootHash);
+    }
+
     [Test]
     [Explicit("Non parallel")]
     public async Task Delays_reporting_metrics()
@@ -664,7 +709,7 @@ private static Keccak BuildKey(int i)
 
     private class PreCommit : IPreCommitBehavior
     {
-        public Keccak BeforeCommit(ICommit commit, CacheBudget budget)
+        public Keccak BeforeCommit(ICommitWithStats commit, CacheBudget budget)
         {
             var hashCode = RuntimeHelpers.GetHashCode(commit);
             Keccak hash = default;

src/Paprika.Tests/Chain/PagePoolTests.cs → src/Paprika.Tests/Chain/BufferPoolTests.cs

@@ -7,13 +7,13 @@
 
 namespace Paprika.Tests.Chain;
 
-public class PagePoolTests
+public class BufferPoolTests
 {
     [Test]
     public void Simple_reuse()
     {
         using var meter = new Meter(nameof(Simple_reuse));
-        using var pool = new BufferPool(1, true, meter);
+        using var pool = new BufferPool(1, BufferPool.PageTracking.AssertCount, meter);
 
         // lease and return
         var initial = pool.Rent();
@@ -32,6 +32,22 @@ public void Simple_reuse()
         pool.AllocatedMB.Should().Be(0, "No megabytes should be reported, it's one page only");
     }
 
+    [Test]
+    public void Stack_trace_tracking()
+    {
+        var pool = new BufferPool(1, BufferPool.PageTracking.StackTrace);
+
+        // Rent and not return
+        pool.Rent();
+
+        var exception = Assert.Throws<Exception>(() => pool.Dispose());
+
+        exception.StackTrace
+            .Should()
+            .Contain(nameof(BufferPoolTests)).And
+            .Contain(nameof(Stack_trace_tracking));
+    }
+
     [Test]
     public void Rented_is_clear()
     {
@@ -54,7 +70,7 @@ public void Rented_is_clear()
     public void Big_pool()
     {
         const int pageCount = 1024;
-        using var pool = new BufferPool(pageCount, assertCountOnDispose: false);
+        using var pool = new BufferPool(pageCount, BufferPool.PageTracking.None);
 
         var set = new HashSet<UIntPtr>();
 

src/Paprika.Tests/Chain/PreCommitBehaviorTests.cs

@@ -50,7 +50,7 @@ public AssertingKeysPreCommit(HashSet<Keccak> keccaks)
             _found = new HashSet<Keccak>();
         }
 
-        public Keccak BeforeCommit(ICommit commit, CacheBudget budget)
+        public Keccak BeforeCommit(ICommitWithStats commit, CacheBudget budget)
         {
             _found.Clear();
 

src/Paprika.Tests/Chain/StorageStatsTests.cs

@@ -0,0 +1,28 @@
+using FluentAssertions;
+using Paprika.Chain;
+
+namespace Paprika.Tests.Chain;
+
+public class StorageStatsTests
+{
+    [TestCase(new[] { true, true }, true, TestName = "Set & Set")]
+    [TestCase(new[] { true, false }, false, TestName = "Set & Delete")]
+    [TestCase(new[] { false, true }, true, TestName = "Delete & Set")]
+    [TestCase(new[] { false, false }, false, TestName = "Delete & Delete")]
+    public void Scenarios(bool[] operations, bool expected)
+    {
+        var stats = new StorageStats();
+        var key = Values.Key0;
+
+        foreach (var operation in operations)
+        {
+            stats.SetStorage(key, operation ? [1] : []);
+        }
+
+        (expected ? stats.Set : stats.Deleted).Should().Contain(key);
+        (expected ? stats.Deleted : stats.Set).Should().BeEmpty();
+
+        stats.Set.Intersect(stats.Deleted).Should()
+            .BeEmpty("The sets of deleted and set should always have no shared elements");
+    }
+}

src/Paprika.Tests/Merkle/Commit.cs

@@ -1,3 +1,5 @@
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
 using System.Text;
 using FluentAssertions;
 using NUnit.Framework;
@@ -12,25 +14,46 @@ namespace Paprika.Tests.Merkle;
 /// <summary>
 /// A commit mock used to provide the data and assert them when needed.
 /// </summary>
-public class Commit(bool skipMemoizedRlpCheck = false) : ICommit
+public class Commit(bool skipMemoizedRlpCheck = false) : ICommitWithStats
 {
     // history <- before <- after
     private readonly Dictionary<byte[], byte[]> _history = new(Comparer);
     private readonly Dictionary<byte[], byte[]> _before = new(Comparer);
     private readonly Dictionary<byte[], byte[]> _after = new(Comparer);
 
-    private readonly Dictionary<Keccak, int> _stats = new();
-
     private bool _asserting;
 
     private static readonly BytesEqualityComparer Comparer = new();
 
+    // stats
+    private readonly HashSet<Keccak> _touchedAccounts = new();
+    private readonly Dictionary<Keccak, IStorageStats> _storageSlots = new();
+
     public void Set(in Key key, ReadOnlySpan<byte> value)
     {
         _before[GetKey(key)] = value.ToArray();
         //to enable storage root calculation for tests
+
+        if (key.Path.Length == NibblePath.KeccakNibbleCount)
+        {
+            _touchedAccounts.Add(key.Path.UnsafeAsKeccak);
+        }
+
         if (!key.IsState)
-            _stats.RegisterSetStorageAccount(key.Path.UnsafeAsKeccak);
+        {
+            var keccak = key.Path.UnsafeAsKeccak;
+
+            if (key.StoragePath.Length == NibblePath.KeccakNibbleCount)
+            {
+                ref var stats = ref CollectionsMarshal.GetValueRefOrAddDefault(_storageSlots, keccak, out var exists);
+                if (exists == false)
+                {
+                    stats = new StorageStats();
+                }
+
+                Unsafe.As<StorageStats>(stats!).SetStorage(key.StoragePath.UnsafeAsKeccak, value);
+            }
+        }
     }
 
     public void DeleteKey(in Key key) => Set(key, ReadOnlySpan<byte>.Empty);
@@ -151,7 +174,6 @@ void ICommit.Visit(CommitAction action, TrieType type)
     }
 
     public IChildCommit GetChild() => new ChildCommit(this);
-    public IReadOnlyDictionary<Keccak, int> Stats => _stats;
 
     private static byte[] Concat(in ReadOnlySpan<byte> payload0, in ReadOnlySpan<byte> payload1)
     {
@@ -205,24 +227,17 @@ public void Commit()
 
     public KeyEnumerator GetSnapshotOfBefore() => new(_before.Keys.ToArray());
 
-    public ref struct KeyEnumerator
+    public ref struct KeyEnumerator(byte[][] keys)
     {
-        private readonly byte[][] _keys;
-        private int _index;
-
-        public KeyEnumerator(byte[][] keys)
-        {
-            _keys = keys;
-            _index = -1;
-        }
+        private int _index = -1;
 
         public bool MoveNext()
         {
-            int index = _index + 1;
-            if (index < _keys.Length)
+            var index = _index + 1;
+            if (index < keys.Length)
             {
                 _index = index;
-                Key.ReadFrom(_keys[index], out var key);
+                Key.ReadFrom(keys[index], out var key);
                 Current = key;
 
                 return true;
@@ -276,7 +291,8 @@ public Commit Squash(bool removeEmpty = false)
             }
         }
 
-        _stats.Clear();
+        _touchedAccounts.Clear();
+        _storageSlots.Clear();
 
         return commit;
     }
@@ -290,4 +306,7 @@ public void MergeAfterToBefore()
 
         _after.Clear();
     }
+
+    public IReadOnlySet<Keccak> TouchedAccounts => _touchedAccounts;
+    public IReadOnlyDictionary<Keccak, IStorageStats> TouchedStorageSlots => Unsafe.As<IReadOnlyDictionary<Keccak, IStorageStats>>(_storageSlots);
 }