Merge branch 'main' into importer

src/Paprika.Tests/Chain/BlockchainTests.cs

@@ -129,6 +129,74 @@ public async Task Account_destruction_same_block()
         before.Should().NotBe(mid);
     }
 
+    [Test]
+    [Category(Categories.LongRunning)]
+    public async Task Account_destruction_spin()
+    {
+        using var db = PagedDb.NativeMemoryDb(128 * Mb, 2);
+        await using var blockchain = new Blockchain(db, new ComputeMerkleBehavior(1, 1, Memoization.None));
+
+        var parent = Keccak.EmptyTreeHash;
+
+        var finality = new Queue<Keccak>();
+
+        byte[] value = [13];
+        byte[] read = new byte[32];
+
+        const uint spins = 2000;
+        for (uint at = 1; at < spins; at++)
+        {
+            using var block = blockchain.StartNew(parent);
+
+            // 10 deletes per block
+            for (int i = 0; i < 10; i++)
+            {
+                Keccak k = default;
+                BinaryPrimitives.WriteInt32LittleEndian(k.BytesAsSpan, i);
+                block.DestroyAccount(k);
+            }
+
+            // 10 sets of various values
+            for (int sets = 0; sets < 10; sets++)
+            {
+                Keccak k = default;
+                BinaryPrimitives.WriteInt32LittleEndian(k.BytesAsSpan, sets + 1000_000);
+                block.SetAccount(k, new Account(at, at));
+                block.SetStorage(k, Key1, value);
+            }
+
+            // destroy this one
+            block.DestroyAccount(Key0);
+
+            // set account Key2
+            block.SetAccount(Key2, new Account(at, at));
+
+            // read non-existing entries for Key2
+            const int readsPerSpin = 1000;
+            for (var readCount = 0; readCount < readsPerSpin; readCount++)
+            {
+                var unique = at * readsPerSpin + readCount;
+                // read values with unique keys, so that they are not cached
+                Keccak k = default;
+                BinaryPrimitives.WriteInt64LittleEndian(k.BytesAsSpan, unique);
+                block.GetStorage(Key2, k, read);
+            }
+
+            parent = block.Commit(at + 1);
+            finality.Enqueue(parent);
+
+            if (finality.Count > 64)
+            {
+                blockchain.Finalize(finality.Dequeue());
+            }
+        }
+
+        while (finality.TryDequeue(out var finalized))
+        {
+            blockchain.Finalize(finalized);
+        }
+    }
+
     [Test]
     public async Task Account_destruction_multi_block()
     {

src/Paprika/Chain/Blockchain.cs

@@ -97,7 +97,8 @@ public Blockchain(IDb db, IPreCommitBehavior preCommit, TimeSpan? minFlushDelay
         _bloomMissedReads = _meter.CreateCounter<long>("Bloom missed reads", "Reads",
             "Number of reads that passed bloom but missed in dictionary");
         _transientCacheUsage =
-            _meter.CreateHistogram<int>("Transient cache usage per commit", "%", "How much used was the transient cache");
+            _meter.CreateHistogram<int>("Transient cache usage per commit", "%",
+                "How much used was the transient cache");
 
         using var batch = _db.BeginReadOnlyBatch();
         _lastFinalized = batch.Metadata.BlockNumber;
@@ -581,7 +582,9 @@ public void DestroyAccount(in Keccak address)
 
             var searched = NibblePath.FromKey(address);
 
-            Destroy(searched, _state);
+            var account = Key.Account(address);
+            _state.Destroy(account.WriteTo(stackalloc byte[account.MaxByteLength]), GetHash(account));
+
             Destroy(searched, _storage);
             Destroy(searched, _preCommit);
 
@@ -598,7 +601,7 @@ static void Destroy(NibblePath searched, PooledSpanDictionary dict)
                     Key.ReadFrom(kvp.Key, out var key);
                     if (key.Path.Equals(searched))
                     {
-                        dict.Destroy(kvp.Key, GetHash(key));
+                        kvp.Destroy();
                     }
                 }
             }
@@ -844,10 +847,12 @@ private ReadOnlySpanOwnerWithMetadata<byte> TryGet(scoped in Key key, scoped Rea
 
             ushort depth = 1;
 
+            var destroyedHash = CommittedBlockState.GetDestroyedHash(key);
+
             // walk all the blocks locally
             foreach (var ancestor in _ancestors)
             {
-                owner = ancestor.TryGetLocal(key, keyWritten, bloom, out succeeded);
+                owner = ancestor.TryGetLocal(key, keyWritten, bloom, destroyedHash, out succeeded);
                 if (succeeded)
                     return owner.WithDepth(depth);
 
@@ -1024,6 +1029,11 @@ private class CommittedBlockState : RefCountingDisposable
         /// </summary>
         private readonly HashSet<Keccak>? _destroyed;
 
+        /// <summary>
+        /// Stores the xor filter of <see cref="_destroyed"/> if any.
+        /// </summary>
+        private readonly Xor8? _destroyedXor;
+
         private readonly Blockchain _blockchain;
 
         /// <summary>
@@ -1040,6 +1050,10 @@ public CommittedBlockState(Xor8 xor, HashSet<Keccak>? destroyed, Blockchain bloc
         {
             _xor = xor;
             _destroyed = destroyed;
+            _destroyedXor = _destroyed != null
+                ? new Xor8(new HashSet<ulong>(_destroyed.Select(k => GetDestroyedHash(k))))
+                : null;
+
             _blockchain = blockchain;
             _committed = committed;
             _raw = raw;
@@ -1054,19 +1068,37 @@ public CommittedBlockState(Xor8 xor, HashSet<Keccak>? destroyed, Blockchain bloc
 
         public Keccak Hash { get; }
 
+        public const ulong NonDestroyable = 0;
+        public const ulong Destroyable = 1;
+
+        public static ulong GetDestroyedHash(in Key key)
+        {
+            var path = key.Path;
+
+            // Check if the path length qualifies.
+            // The check for destruction is performed only for Account, Storage or Merkle-of-Storage that all have full paths. 
+            if (path.Length != NibblePath.KeccakNibbleCount)
+                return NonDestroyable;
+
+            // Return ulonged hash.
+            return GetDestroyedHash(path.UnsafeAsKeccak);
+        }
+
+        private static ulong GetDestroyedHash(in Keccak keccak) => keccak.GetHashCodeUlong() | Destroyable;
+
         /// <summary>
         /// Tries to get the key only from this block, acquiring no lease as it assumes that the lease is taken.
         /// </summary>
         public ReadOnlySpanOwner<byte> TryGetLocal(scoped in Key key, scoped ReadOnlySpan<byte> keyWritten,
-            ulong bloom, out bool succeeded)
+            ulong bloom, ulong destroyedHash, out bool succeeded)
         {
-            var mayHave = _xor.MayContain(unchecked((ulong)bloom));
+            var mayHave = _xor.MayContain(bloom);
 
             // check if the change is in the block
             if (!mayHave)
             {
                 // if destroyed, return false as no previous one will contain it
-                if (IsAccountDestroyed(key))
+                if (IsAccountDestroyed(key, destroyedHash))
                 {
                     succeeded = true;
                     return default;
@@ -1088,7 +1120,7 @@ public ReadOnlySpanOwner<byte> TryGetLocal(scoped in Key key, scoped ReadOnlySpa
             _blockchain._bloomMissedReads.Add(1);
 
             // if destroyed, return false as no previous one will contain it
-            if (IsAccountDestroyed(key))
+            if (IsAccountDestroyed(key, destroyedHash))
             {
                 succeeded = true;
                 return default;
@@ -1098,16 +1130,12 @@ public ReadOnlySpanOwner<byte> TryGetLocal(scoped in Key key, scoped ReadOnlySpa
             return default;
         }
 
-        private bool IsAccountDestroyed(scoped in Key key)
+        private bool IsAccountDestroyed(scoped in Key key, ulong destroyed)
         {
-            if (_destroyed == null)
-                return false;
-
-            if (key.Path.Length != NibblePath.KeccakNibbleCount)
+            if (_destroyedXor == null || destroyed == NonDestroyable)
                 return false;
 
-            // it's either Account, Storage, or Merkle that is a storage
-            return _destroyed.Contains(key.Path.UnsafeAsKeccak);
+            return _destroyedXor.MayContain(destroyed) && _destroyed!.Contains(key.Path.UnsafeAsKeccak);
         }
 
         protected override void CleanUp()
@@ -1229,12 +1257,16 @@ private ReadOnlySpanOwnerWithMetadata<byte> TryGet(scoped in Key key, scoped Rea
         {
             ushort depth = 1;
 
+            var destroyedHash = CommittedBlockState.GetDestroyedHash(key);
+
             // walk all the blocks locally
             foreach (var ancestor in _ancestors)
             {
-                var owner = ancestor.TryGetLocal(key, keyWritten, bloom, out succeeded);
+                var owner = ancestor.TryGetLocal(key, keyWritten, bloom, destroyedHash, out succeeded);
                 if (succeeded)
-                    return owner.WithDepth(1);
+                    return owner.WithDepth(depth);
+
+                depth++;
             }
 
             if (_batch.TryGet(key, out var span))

src/Paprika/Chain/PooledSpanDictionary.cs

@@ -373,6 +373,11 @@ public KeyValue(ref byte b, uint bucket)
                 _bucket = bucket;
                 _b = ref b;
             }
+
+            public void Destroy()
+            {
+                _b |= DestroyedBit;
+            }
         }
     }
 
@@ -483,14 +488,15 @@ private readonly struct Root(Page[] pages)
 
         public const int BucketCount = PageCount * BucketsPerPage;
         private const int BucketsPerPage = Page.PageSize / sizeof(uint);
+        private const int InPageMask = BucketsPerPage - 1;
+        private static readonly int PageShift = BitOperations.Log2(BucketsPerPage);
 
         public unsafe ref uint this[int bucket]
         {
             get
             {
-                var (page, buck) = Math.DivRem(bucket, BucketsPerPage);
-                var raw = pages[page].Raw;
-                return ref Unsafe.Add(ref Unsafe.AsRef<uint>(raw.ToPointer()), buck);
+                var raw = pages[bucket >> PageShift].Raw;
+                return ref Unsafe.Add(ref Unsafe.AsRef<uint>(raw.ToPointer()), bucket & InPageMask);
             }
         }
     }

src/Paprika/Utils/Xor8.cs

@@ -32,8 +32,6 @@ public class Xor8
     public Xor8(IReadOnlyCollection<ulong> keys)
     {
         // TODO: remove all array allocations, use ArrayPool<ulong> more and/or buffer pool, potentially combine chunks of memory together
-
-
         var size = keys.Count;
         var arrayLength = GetArrayLength(size);