Merge branch 'main' into importer

src/Paprika.Tests/Chain/PooledSpanDictionaryTests.cs

@@ -163,7 +163,7 @@ public void Copy_to()
 
         // Copy
         using var copy = new PooledSpanDictionary(pool);
-        dict.CopyTo(copy);
+        dict.CopyTo(copy, b => true);
 
         // Assert
         dict.TryGet(key, hash, out result).Should().BeTrue();
@@ -228,7 +228,7 @@ public void Large_spin()
 
         // Copy & test copy
         using var copy = new PooledSpanDictionary(pool, false);
-        dict.CopyTo(copy);
+        dict.CopyTo(copy, b => true);
 
         AssertIterate(copy);
         return;

src/Paprika.Tests/Merkle/AdditionalTests.cs

@@ -109,6 +109,66 @@ public async Task Account_creation_hint(bool newHint)
         blockchain.Finalize(parent);
 
         await blockchain.WaitTillFlush(parent);
+    }
+
+    [Explicit]
+    [Test(Description = "Testing the hint that precommit can use to remember what was destroyed")]
+    public async Task Accounts_recreation()
+    {
+        const int count = 100;
+        const int spins = 500;
+        const int seed = 17;
+
+        var random = new Random(seed);
+        var value1 = new byte[] { 13, 17, 23, 29 };
+        var value2 = new byte[] { 13, 17, 23, 17 };
+
+        var parent = Keccak.EmptyTreeHash;
+        uint parentNumber = 1;
+
+        using var db = PagedDb.NativeMemoryDb(256 * 1024 * 1024, 2);
+        var merkle = new ComputeMerkleBehavior(2, 2);
+
+        await using var blockchain = new Blockchain(db, merkle);
+
+        using var block1 = blockchain.StartNew(parent);
+        for (var i = 0; i < count; i++)
+        {
+            var keccak = random.NextKeccak();
 
+            block1.SetAccount(keccak, new Account(1, 1));
+            block1.SetStorage(keccak, keccak, value1);
+        }
+
+        parent = block1.Commit(parentNumber);
+        parentNumber++;
+
+        blockchain.Finalize(parent);
+
+        await blockchain.WaitTillFlush(parent);
+
+        // destroy all but one
+        for (uint spin = 0; spin < spins; spin++)
+        {
+            random = new Random(seed);
+            using var block = blockchain.StartNew(parent);
+
+            for (var i = 0; i < count; i++)
+            {
+                var keccak = random.NextKeccak();
+
+                // recreate
+                block.DestroyAccount(keccak);
+                block.SetAccount(keccak, new Account(spin + 2, spin + 2));
+                block.SetStorage(keccak, keccak, value2);
+            }
+
+            parent = block.Commit(parentNumber);
+            parentNumber++;
+
+            blockchain.Finalize(parent);
+        }
+
+        await blockchain.WaitTillFlush(parent);
     }
 }

src/Paprika/Chain/Blockchain.cs

@@ -1,4 +1,4 @@
-using System.CodeDom.Compiler;
+using System.CodeDom.Compiler;
 using System.Diagnostics;
 using System.Diagnostics.Metrics;
 using System.Runtime.InteropServices;
@@ -393,10 +393,12 @@ public void Finalize(Keccak keccak)
     /// </summary>
     private class BlockState : RefCountingDisposable, IWorldState, ICommit, IProvideDescription, IStateStats
     {
+        [ThreadStatic]
+        private static HashSet<ulong>? s_bloomCache;
         /// <summary>
         /// A simple bloom filter to assert whether the given key was set in a given block, used to speed up getting the keys.
         /// </summary>
-        private readonly HashSet<ulong> _bloom;
+        private HashSet<ulong> _bloom;
 
         private readonly Dictionary<Keccak, int>? _stats;
 
@@ -445,7 +447,7 @@ public BlockState(Keccak parentStateRoot, IReadOnlyBatch batch, CommittedBlockSt
 
             ParentHash = parentStateRoot;
 
-            _bloom = new HashSet<ulong>();
+            _bloom = Interlocked.Exchange(ref s_bloomCache, null) ?? new HashSet<ulong>();
             _destroyed = null;
             _stats = new Dictionary<Keccak, int>();
 
@@ -538,16 +540,21 @@ public Keccak Commit(uint blockNumber)
             var data = new PooledSpanDictionary(Pool, false);
 
             // use append for faster copies as state and storage won't overwrite each other
-            _state.CopyTo(data, true);
-            _storage.CopyTo(data, true);
-            _preCommit.CopyTo(data);
+            _state.CopyTo(data, OmitUseOnce, true);
+            _storage.CopyTo(data, OmitUseOnce, true);
+            _preCommit.CopyTo(data, OmitUseOnce);
 
             // Creation acquires the lease
             return new CommittedBlockState(xor, _destroyed, _blockchain, data, hash,
                 ParentHash,
                 blockNumber, raw);
         }
 
+        /// <summary>
+        /// Filters out entries that are of type <see cref="EntryType.UseOnce"/> as they should be used once only.
+        /// </summary>
+        private static bool OmitUseOnce(byte metadata) => metadata != (int)EntryType.UseOnce;
+
         public CommittedBlockState CommitRaw() => CommitImpl(0, true)!;
 
         public void Reset()
@@ -589,7 +596,10 @@ public void DestroyAccount(in Keccak address)
             var searched = NibblePath.FromKey(address);
 
             var account = Key.Account(address);
-            _state.Destroy(account.WriteTo(stackalloc byte[account.MaxByteLength]), GetHash(account));
+
+            // set account to empty first
+            _state.Set(account.WriteTo(stackalloc byte[account.MaxByteLength]), GetHash(account),
+                ReadOnlySpan<byte>.Empty, (byte)EntryType.Persistent);
 
             Destroy(searched, _storage);
             Destroy(searched, _preCommit);
@@ -598,6 +608,9 @@ public void DestroyAccount(in Keccak address)
 
             _destroyed ??= new HashSet<Keccak>();
             _destroyed.Add(address);
+
+            _blockchain._preCommit.OnAccountDestroyed(address, this);
+
             return;
 
             static void Destroy(NibblePath searched, PooledSpanDictionary dict)
@@ -642,7 +655,7 @@ private void TryCache(in Key key, in ReadOnlySpanOwnerWithMetadata<byte> owner,
         {
             if (_cacheBudgetStorageAndStage.ShouldCache(owner))
             {
-                SetImpl(key, owner.Span, EntryType.Transient, dict);
+                SetImpl(key, owner.Span, EntryType.Cached, dict);
             }
         }
 
@@ -758,15 +771,6 @@ void ICommit.Visit(CommitAction action, TrieType type)
                     action(key, kvp.Value);
                 }
             }
-
-            if (type == TrieType.State && _destroyed != null)
-            {
-                foreach (var destroyed in _destroyed)
-                {
-                    // clean the deletes
-                    action(Key.Account(destroyed), ReadOnlySpan<byte>.Empty);
-                }
-            }
         }
 
         IChildCommit ICommit.GetChild() => new ChildCommit(Pool, this);
@@ -816,7 +820,7 @@ public void Commit()
                     var type = (EntryType)kvp.Metadata;
 
                     // flush down only volatiles
-                    if (type != EntryType.Volatile)
+                    if (type != EntryType.UseOnce)
                     {
                         parent.Set(key, kvp.Value, type);
                     }
@@ -913,8 +917,9 @@ private ReadOnlySpanOwner<byte> TryGetLocal(scoped in Key key, scoped ReadOnlySp
                 _ => null
             };
 
-            // first always try pre-commit as it may overwrite data
-            if (_preCommit.TryGet(keyWritten, bloom, out var span))
+            // First always try pre-commit as it may overwrite data.
+            // Don't do it for the storage though! StorageCell entries are not modified by pre-commit! It can only read them!
+            if (key.Type != DataType.StorageCell && _preCommit.TryGet(keyWritten, bloom, out var span))
             {
                 // return with owned lease
                 succeeded = true;
@@ -967,32 +972,19 @@ protected override void CleanUp()
             {
                 ancestor.Dispose();
             }
-        }
-
-        public void Assert(IReadOnlyBatch batch)
-        {
-            using var squashed = new PooledSpanDictionary(Pool);
 
-            _state.CopyTo(squashed);
-            _storage.CopyTo(squashed);
-            _preCommit.CopyTo(squashed);
+            ReturnCacheToPool();
 
-            foreach (var kvp in squashed)
+            void ReturnCacheToPool()
             {
-                Key.ReadFrom(kvp.Key, out var key);
-
-                if (!batch.TryGet(key, out var value))
+                var bloom = _bloom;
+                _bloom = null!;
+                ref var cache = ref s_bloomCache;
+                if (cache is null)
                 {
-                    throw new KeyNotFoundException($"Key {key.ToString()} not found.");
-                }
-
-                if (!value.SequenceEqual(kvp.Value))
-                {
-                    var expected = kvp.Value.ToHexString(false);
-                    var actual = value.ToHexString(false);
-
-                    throw new Exception($"Values are different for {key.ToString()}. " +
-                                        $"Expected is {expected} while found is {actual}.");
+                    // Return the cache to be reused
+                    bloom.Clear();
+                    cache = bloom;
                 }
             }
         }
@@ -1448,4 +1440,4 @@ private void ThrowOnFinalized()
 
         public ReadOnlySpanOwnerWithMetadata<byte> Get(scoped in Key key) => ((IReadOnlyWorldState)_current).Get(key);
     }
-}
+}

src/Paprika/Chain/CacheBudget.cs

@@ -49,14 +49,14 @@ public bool ShouldCache(in ReadOnlySpanOwnerWithMetadata<byte> owner, out EntryT
     {
         if (ShouldCache(owner))
         {
-            type = EntryType.Transient;
+            type = EntryType.Cached;
             return true;
         }
 
         // Either budget is full or depth was not sufficient. For anything beyond 0, use volatile
         if (owner.QueryDepth > 0)
         {
-            type = EntryType.Volatile;
+            type = EntryType.UseOnce;
             return true;
         }
 

src/Paprika/Chain/EntryType.cs

@@ -11,12 +11,12 @@ public enum EntryType : byte
     Persistent = 0,
 
     /// <summary>
-    /// An entry representing data that were cached on the behalf of the decision of <see cref="CacheBudget.ShouldCache"/>.
+    /// An entry representing data that were cached on the behalf of the decision of <see cref="CacheBudget"/>.
     /// </summary>
-    Transient = 1,
+    Cached = 1,
 
     /// <summary>
     /// The entry is put only for a short period of computation and should not be considered to be stored in memory beyond this computation.
     /// </summary>
-    Volatile = 2
+    UseOnce = 2
 }

src/Paprika/Chain/IPreCommitBehavior.cs

@@ -33,6 +33,13 @@ public interface IPreCommitBehavior
     void OnNewAccountCreated(in Keccak address, ICommit commit)
     {
     }
+
+    /// <summary>
+    /// Executed after the current state & storage is cleared.
+    /// </summary>
+    void OnAccountDestroyed(in Keccak address, ICommit commit)
+    {
+    }
 }
 
 /// <summary>

src/Paprika/Chain/PooledSpanDictionary.cs

@@ -135,11 +135,14 @@ private SearchResult TryGetImpl(scoped ReadOnlySpan<byte> key, uint hash)
         return default;
     }
 
-    public void CopyTo(PooledSpanDictionary destination, bool append = false)
+    public void CopyTo(PooledSpanDictionary destination, Predicate<byte> metadataWhere, bool append = false)
     {
         foreach (var kvp in this)
         {
-            destination.SetImpl(kvp.Key, kvp.Hash, kvp.Value, ReadOnlySpan<byte>.Empty, kvp.Metadata, append);
+            if (metadataWhere(kvp.Metadata))
+            {
+                destination.SetImpl(kvp.Key, kvp.Hash, kvp.Value, ReadOnlySpan<byte>.Empty, kvp.Metadata, append);
+            }
         }
     }
 

src/Paprika/Merkle/ComputeMerkleBehavior.cs

@@ -209,9 +209,17 @@ private static void ScatterGather(ICommit commit, IEnumerable<IWorkItem> workIte
     public void OnNewAccountCreated(in Keccak address, ICommit commit)
     {
         // Set a transient empty entry for the newly created account.
+        // This simulates an empty storage tree.
+        // If this account has storage set, it won't try to query the database to get nothing, it will get nothing from here.
+        commit.Set(Key.Merkle(NibblePath.FromKey(address)), ReadOnlySpan<byte>.Empty, EntryType.UseOnce);
+    }
+
+    public void OnAccountDestroyed(in Keccak address, ICommit commit)
+    {
+        // Set an empty entry as the storage root for the destroyed account.
         // This simulates an empty storage tree. 
         // If this account has storage set, it won't try to query the database to get nothing, it will get nothing from here.
-        commit.Set(Key.Merkle(NibblePath.FromKey(address)), ReadOnlySpan<byte>.Empty, EntryType.Transient);
+        commit.Set(Key.Merkle(NibblePath.FromKey(address)), ReadOnlySpan<byte>.Empty, EntryType.UseOnce);
     }
 
     /// <summary>

src/Paprika/Paprika.csproj

@@ -8,8 +8,9 @@
 
     <ItemGroup>
       <PackageReference Include="HdrHistogram" Version="2.5.0" />
-      <PackageReference Include="Nethermind.Numerics.Int256" Version="1.1.0" />
-      <PackageReference Include="System.IO.Hashing" Version="8.0.0-preview.4.23259.5" />
+      <PackageReference Include="Nethermind.Numerics.Int256" Version="1.2.0" />
+      <PackageReference Include="NonBlocking" Version="2.1.2" />
+      <PackageReference Include="System.IO.Hashing" Version="8.0.0" />
     </ItemGroup>
 
 </Project>