Merge branch 'main' into importer

docs/design.md

@@ -46,7 +46,7 @@ blockchain.Finalize(Block2A);
 
 ### PagedDb
 
-The `PagedDb` component is responsible for storing the left-fold of the blocks that are beyond the cut-off point. This database uses [memory-mapped files](https://en.wikipedia.org/wiki/Memory-mapped_file) to provide storing capabilities. To handle concurrency, [Copy on Write](https://en.wikipedia.org/wiki/Copy-on-write) is used. This allows multiple concurrent readers to cooperate in a full lock-free manner and a single writer that runs the current transaction. In that manner, it's heavily inspired by [LMBD](https://github.com/LMDB/lmdb).
+The `PagedDb` component is responsible for storing the left-fold of the blocks that are beyond the cut-off point. This database uses [memory-mapped files](https://en.wikipedia.org/wiki/Memory-mapped_file) to provide storing capabilities. To handle concurrency, [Copy on Write](https://en.wikipedia.org/wiki/Copy-on-write) is used. This allows multiple concurrent readers to cooperate in a full lock-free manner and a single writer that runs the current transaction. In that manner, it's heavily inspired by [LMDB](https://github.com/LMDB/lmdb).
 
 It's worth to mention that due to the design of the `Blockchain` component, having a single writer available is sufficient. At the same time, having multiple readers allow to create readonly transactions that are later used by blocks from `Blockchain`.
 
@@ -365,6 +365,6 @@ A few remarks:
 1. Database Storage lectures by Andy Pavlo from CMU Intro to Database Systems / Fall 2022:
    1. Database Storage, pt. 1 https://www.youtube.com/watch?v=df-l2PxUidI
    1. Database Storage, pt. 2 https://www.youtube.com/watch?v=2HtfGdsrwqA
-1. LMBD
+1. LMDB
    1. Howard Chu - LMDB [The Databaseology Lectures - CMU Fall 2015](https://www.youtube.com/watch?v=tEa5sAh-kVk)
    1. The main file of LMDB [mdb.c](https://github.com/LMDB/lmdb/blob/mdb.master/libraries/liblmdb/mdb.c)

src/Paprika/Data/BitMapFilter.cs

@@ -6,6 +6,7 @@
 using System.Runtime.Intrinsics.X86;
 using Paprika.Chain;
 using Paprika.Store;
+using Paprika.Utils;
 
 namespace Paprika.Data;
 
@@ -25,10 +26,12 @@ public static BitMapFilter<OfN<TSize>> CreateOfN<TSize>(BufferPool pool)
         var pages = new Page[TSize.Count];
         for (var i = 0; i < TSize.Count; i++)
         {
-            pages[i] = pool.Rent(true);
+            pages[i] = pool.Rent(false);
         }
 
-        return new BitMapFilter<OfN<TSize>>(new OfN<TSize>(pages));
+        var accessor = new OfN<TSize>(pages);
+        accessor.Clear();
+        return new BitMapFilter<OfN<TSize>>(accessor);
     }
 
     public interface IAccessor<TAccessor>
@@ -150,10 +153,11 @@ public unsafe ref int GetSlot(uint hash)
 
         public void Clear()
         {
-            foreach (var page in _pages)
+            ParallelUnbalancedWork.For(0, PageCount, _pages, static (i, state) =>
             {
-                page.Clear();
-            }
+                state[i].Clear();
+                return state;
+            });
         }
 
         public void Return(BufferPool pool)
@@ -182,12 +186,12 @@ public void OrWith(in OfN<TSize> other)
         [Pure]
         public void OrWith(OfN<TSize>[] others)
         {
-            var pages = _pages;
+            State state = new State(others, _pages);
 
-            Parallel.For(0, PageCount, i =>
+            ParallelUnbalancedWork.For(0, PageCount, state, static (i, s) =>
             {
-                var page = pages[i];
-                var length = others.Length;
+                var page = s.Pages[i];
+                var length = s.Others.Length;
 
                 for (var j = 0; j < length - 1; j++)
                 {
@@ -196,17 +200,21 @@ public void OrWith(OfN<TSize>[] others)
                         // prefetch next
                         unsafe
                         {
-                            Sse.Prefetch2(others[j + 1]._pages[i].Payload);
+                            Sse.Prefetch2(s.Others[j + 1]._pages[i].Payload);
                         }
                     }
 
-                    page.OrWith(others[j]._pages[i]);
+                    page.OrWith(s.Others[j]._pages[i]);
                 }
 
-                page.OrWith(others[length - 1]._pages[i]);
+                page.OrWith(s.Others[length - 1]._pages[i]);
+
+                return s;
             });
         }
 
+        private readonly record struct State(OfN<TSize>[] Others, Page[] Pages);
+
         public int BucketCount => Page.PageSize * BitsPerByte * PageCount;
 
         private static int PageCount => TSize.Count;

src/Paprika/Merkle/ComputeMerkleBehavior.cs

@@ -261,9 +261,9 @@ private static void ScatterGather(ICommit commit, BuildStorageTriesItem[] workIt
         }
 
         var children = new ConcurrentQueue<IChildCommit>();
-        Parallel.For(0, workItems.Length, s_parallelOptions,
+        ParallelUnbalancedWork.For(0, workItems.Length, s_parallelOptions,
             commit.GetChild,
-            (i, _, child) =>
+            (i, child) =>
             {
                 workItems[i].DoWork(child);
                 return child;
@@ -605,7 +605,7 @@ private void EncodeBranch(scoped in Key key, scoped in ComputeContext ctx, scope
             var budget = ctx.Budget;
 
             // parallel calculation
-            Parallel.For((long)0, NibbleSet.NibbleCount, s_parallelOptions, nibble =>
+            ParallelUnbalancedWork.For(0, NibbleSet.NibbleCount, s_parallelOptions, nibble =>
             {
                 var childPath = NibblePath.Single((byte)nibble, 0);