From 79880b010875e500c2b2ab766fd0b1b9a1f16a77 Mon Sep 17 00:00:00 2001 From: scooletz Date: Thu, 19 Dec 2024 16:21:47 +0100 Subject: [PATCH 1/9] PooledSpanDicitonary uses now direct raw pointer --- src/Paprika/Chain/PooledSpanDictionary.cs | 81 ++++++++++++++--------- 1 file changed, 48 insertions(+), 33 deletions(-) diff --git a/src/Paprika/Chain/PooledSpanDictionary.cs b/src/Paprika/Chain/PooledSpanDictionary.cs index 724039e8..4782b1a9 100644 --- a/src/Paprika/Chain/PooledSpanDictionary.cs +++ b/src/Paprika/Chain/PooledSpanDictionary.cs @@ -13,6 +13,14 @@ namespace Paprika.Chain; /// public class PooledSpanDictionary : IDisposable { + /// + /// Gets the size of the address to the next item. + /// + /// + /// Pointer size. Assumes 64 bits. + /// + private const int PointerSize = 8; + private const int BufferSize = BufferPool.BufferSize; private readonly BufferPool _pool; @@ -83,24 +91,21 @@ public bool TryGet(scoped ReadOnlySpan key, ulong hash, out ReadOnlySpan key, uint leftover, uint bucket) + private SearchResult TryGetImpl(scoped ReadOnlySpan key, uint leftover, uint bucket) { Debug.Assert(BitOperations.LeadingZeroCount(leftover) >= 11, "First 10 bits should be left unused"); var address = _root[(int)bucket]; - if (address == 0) goto NotFound; + if (address == UIntPtr.Zero) goto NotFound; - ref var pages = ref MemoryMarshal.GetReference(CollectionsMarshal.AsSpan(_pages)); do { - var (pageNo, atPage) = Math.DivRem(address, Page.PageSize); - - ref var at = ref Unsafe.AsRef((byte*)Unsafe.Add(ref pages, pageNo).Raw.ToPointer() + atPage); + ref var at = ref ReadAtAddress(address); var header = at & PreambleBits; if ((header & DestroyedBit) == 0) @@ -126,12 +131,14 @@ private unsafe SearchResult TryGetImpl(scoped ReadOnlySpan key, uint lefto } // Decode next entry address - address = Unsafe.ReadUnaligned(ref Unsafe.Add(ref at, PreambleLength)); + address = Unsafe.ReadUnaligned(ref Unsafe.Add(ref at, PreambleLength)); } while (address != 0); NotFound: return default; } + private static unsafe ref byte ReadAtAddress(UIntPtr address) => ref Unsafe.AsRef(address.ToPointer()); + private static (uint leftover, uint bucket) GetBucketAndLeftover(ulong hash) => Math.DivRem(Mix(hash), Root.BucketCount); @@ -267,7 +274,7 @@ private void SetImpl(scoped ReadOnlySpan key, uint mixed, ReadOnlySpan= 10, "First 10 bits should be left unused"); - var root = _root[(int)bucket]; + UIntPtr root = _root[(int)bucket]; var dataLength = data1.Length + data0.Length; @@ -315,10 +322,10 @@ public void Destroy(scoped ReadOnlySpan key, ulong hash) /// Enumerator walks through all the values beside the ones that were destroyed in this dictionary /// with . /// - public ref struct Enumerator(PooledSpanDictionary dictionary) + public unsafe ref struct Enumerator(PooledSpanDictionary dictionary) { private int _bucket = -1; - private uint _address = 0; + private UIntPtr _address = 0; private ref byte _at; public bool MoveNext() @@ -326,7 +333,7 @@ public bool MoveNext() while (_bucket < Root.BucketCount) { // On empty, scan to the next bucket that is not empty - while (_address == 0) + while (_address == UIntPtr.Zero) { _bucket++; if (_bucket == Root.BucketCount) @@ -338,13 +345,13 @@ public bool MoveNext() } // Scan the bucket till it's not destroyed - while (_address != 0) + while (_address != UIntPtr.Zero) { // Capture the current, move address to next immediately - ref var at = ref dictionary.GetAt(_address); + ref var at = ref ReadAtAddress(_address); // The position is captured in ref at above, move to next - _address = Unsafe.ReadUnaligned(ref Unsafe.Add(ref at, PreambleLength)); + _address = Unsafe.ReadUnaligned(ref Unsafe.Add(ref at, PreambleLength)); if ((at & DestroyedBit) == 0) { @@ -414,14 +421,6 @@ public void Destroy() } } - private ref byte GetAt(uint address) - { - Debug.Assert(address > 0); - - var (pageNo, atPage) = Math.DivRem(address, Page.PageSize); - return ref Unsafe.Add(ref MemoryMarshal.GetReference(_pages[(int)pageNo].Span), (int)atPage); - } - private void AllocateNewPage() { var page = RentNewPage(false); @@ -440,7 +439,7 @@ private Page RentNewPage(bool clear) private static uint Mix(ulong hash) => unchecked((uint)((hash >> 32) ^ hash)); - private Span Write(int size, out uint addr) + private Span Write(int size, out UIntPtr addr) { if (BufferSize - _position < size) { @@ -450,10 +449,9 @@ private Span Write(int size, out uint addr) // allocated before the position is changed var span = _current.Span.Slice(_position, size); + addr = _current.Raw + (UIntPtr)_position; - addr = (uint)(_position + (_pages.Count - 1) * BufferSize); _position += size; - return span; } @@ -509,27 +507,44 @@ public void Describe(TextWriter text, Key.Predicate? predicate = null) static string S(in NibblePath full) => full.UnsafeAsKeccak.ToString(); } - private readonly struct Root(Page[] pages) + [StructLayout(LayoutKind.Explicit, Size = SizeOf)] + private readonly struct Root { + [FieldOffset(0)] + private readonly Page _pages; + + public Root(Page[] pages) + { + Debug.Assert(pages.Length == PageCount); + pages.CopyTo(MemoryMarshal.CreateSpan(ref _pages, PageCount)); + } + + /// + /// The size of this structure. + /// + private const int SizeOf = PageCount * PointerSize; + /// - /// 16gives 4kb * 16, 64kb allocated per dictionary. - /// This gives 16k buckets which should be sufficient to have a really low ratio of collisions for majority of the blocks. + /// 16 pages, gives 4kb * 16, 64kb of memory allocated per dictionary. + /// This gives 8k buckets ( + /// which should be sufficient to have a really low ratio of collisions for the majority of the blocks. /// public const int PageCount = 16; public static readonly int BucketCountLog2 = BitOperations.Log2(BucketCount); public const int BucketCount = PageCount * BucketsPerPage; - private const int BucketsPerPage = Page.PageSize / sizeof(uint); + private const int BucketsPerPage = Page.PageSize / PointerSize; private const int InPageMask = BucketsPerPage - 1; private static readonly int PageShift = BitOperations.Log2(BucketsPerPage); - public unsafe ref uint this[int bucket] + public unsafe ref UIntPtr this[int bucket] { get { - var raw = Unsafe.Add(ref MemoryMarshal.GetArrayDataReference(pages), bucket >> PageShift).Raw; - return ref Unsafe.Add(ref Unsafe.AsRef(raw.ToPointer()), bucket & InPageMask); + var shift = bucket >> PageShift; + var raw = Unsafe.Add(ref Unsafe.AsRef(in _pages), shift).Raw; + return ref Unsafe.Add(ref Unsafe.AsRef(raw.ToPointer()), bucket & InPageMask); } } } From 48c7b9c1e80c51da303100b2852c69d8700cafa3 Mon Sep 17 00:00:00 2001 From: scooletz Date: Thu, 19 Dec 2024 16:49:06 +0100 Subject: [PATCH 2/9] bigger fanout --- src/Paprika/Chain/PooledSpanDictionary.cs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/Paprika/Chain/PooledSpanDictionary.cs b/src/Paprika/Chain/PooledSpanDictionary.cs index 4782b1a9..b2ce9e75 100644 --- a/src/Paprika/Chain/PooledSpanDictionary.cs +++ b/src/Paprika/Chain/PooledSpanDictionary.cs @@ -525,11 +525,9 @@ public Root(Page[] pages) private const int SizeOf = PageCount * PointerSize; /// - /// 16 pages, gives 4kb * 16, 64kb of memory allocated per dictionary. - /// This gives 8k buckets ( - /// which should be sufficient to have a really low ratio of collisions for the majority of the blocks. + /// The total number of pages used by the root construct. The bigger, the bigger fanout it is. /// - public const int PageCount = 16; + public const int PageCount = 32; public static readonly int BucketCountLog2 = BitOperations.Log2(BucketCount); From d3aca04a7cdfaf9f07ca753bf18be55fef82e6a7 Mon Sep 17 00:00:00 2001 From: scooletz Date: Thu, 19 Dec 2024 16:55:17 +0100 Subject: [PATCH 3/9] comment fix --- src/Paprika/Chain/PooledSpanDictionary.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Paprika/Chain/PooledSpanDictionary.cs b/src/Paprika/Chain/PooledSpanDictionary.cs index b2ce9e75..59ba5f81 100644 --- a/src/Paprika/Chain/PooledSpanDictionary.cs +++ b/src/Paprika/Chain/PooledSpanDictionary.cs @@ -40,8 +40,6 @@ public class PooledSpanDictionary : IDisposable /// /// Set to true, if the data written once should not be overwritten. /// This allows to hold values returned by the dictionary through multiple operations. - /// - /// This dictionary uses to store keys buffers to allow concurrent readers /// public PooledSpanDictionary(BufferPool pool, bool preserveOldValues = false) { From 849d0a0750a3e695e8b03f3bd38eacdd9abb9dc6 Mon Sep 17 00:00:00 2001 From: scooletz Date: Thu, 19 Dec 2024 17:04:33 +0100 Subject: [PATCH 4/9] modified benchmark to not recreate dicitionary --- .../PooledSpanDictionaryBenchmarks.cs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/Paprika.Benchmarks/PooledSpanDictionaryBenchmarks.cs b/src/Paprika.Benchmarks/PooledSpanDictionaryBenchmarks.cs index 659139cc..5663f71c 100644 --- a/src/Paprika.Benchmarks/PooledSpanDictionaryBenchmarks.cs +++ b/src/Paprika.Benchmarks/PooledSpanDictionaryBenchmarks.cs @@ -22,6 +22,7 @@ public class PooledSpanDictionaryBenchmarks }; private readonly PooledSpanDictionary _varLengthKeys; + private readonly PooledSpanDictionary _readWrite; private const int VarLengthKeyCollisions = 8; private const ulong VarLengthKeyCollisionHash = 2348598349058394; @@ -51,6 +52,8 @@ public PooledSpanDictionaryBenchmarks() { _bigDict.Set(VarLengthKey[..VarLengthKeyCollisions], VarLengthKeyCollisionHash, Value32Bytes, 1); } + + _readWrite = new PooledSpanDictionary(new BufferPool(128, BufferPool.PageTracking.None, null)); } [Benchmark] @@ -124,16 +127,16 @@ public int Read_missing_with_no_hash_collisions() [Benchmark] public int Read_write_small() { - using var dict = new PooledSpanDictionary(_pool, false); - Span key = stackalloc byte[2]; var count = 0; for (byte i = 0; i < 255; i++) { key[0] = i; - dict.Set(key, i, key, 1); - dict.TryGet(key, i, out var result); + + _readWrite.Set(key, i, key, 1); + _readWrite.TryGet(key, i, out var result); + count += result[0]; } From 1f5a8b736f7c85ca50769bddbe0ceef3515d7300 Mon Sep 17 00:00:00 2001 From: scooletz Date: Thu, 19 Dec 2024 17:09:41 +0100 Subject: [PATCH 5/9] clearing pages made parallel --- .../PooledSpanDictionaryBenchmarks.cs | 4 ++-- src/Paprika/Chain/PooledSpanDictionary.cs | 11 ++++++++--- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/Paprika.Benchmarks/PooledSpanDictionaryBenchmarks.cs b/src/Paprika.Benchmarks/PooledSpanDictionaryBenchmarks.cs index 5663f71c..b7eafbf8 100644 --- a/src/Paprika.Benchmarks/PooledSpanDictionaryBenchmarks.cs +++ b/src/Paprika.Benchmarks/PooledSpanDictionaryBenchmarks.cs @@ -52,7 +52,7 @@ public PooledSpanDictionaryBenchmarks() { _bigDict.Set(VarLengthKey[..VarLengthKeyCollisions], VarLengthKeyCollisionHash, Value32Bytes, 1); } - + _readWrite = new PooledSpanDictionary(new BufferPool(128, BufferPool.PageTracking.None, null)); } @@ -136,7 +136,7 @@ public int Read_write_small() _readWrite.Set(key, i, key, 1); _readWrite.TryGet(key, i, out var result); - + count += result[0]; } diff --git a/src/Paprika/Chain/PooledSpanDictionary.cs b/src/Paprika/Chain/PooledSpanDictionary.cs index 59ba5f81..888e2a5c 100644 --- a/src/Paprika/Chain/PooledSpanDictionary.cs +++ b/src/Paprika/Chain/PooledSpanDictionary.cs @@ -5,6 +5,7 @@ using Paprika.Crypto; using Paprika.Data; using Paprika.Store; +using Paprika.Utils; namespace Paprika.Chain; @@ -49,9 +50,15 @@ public PooledSpanDictionary(BufferPool pool, bool preserveOldValues = false) var pages = new Page[Root.PageCount]; for (var i = 0; i < Root.PageCount; i++) { - pages[i] = RentNewPage(true); + pages[i] = RentNewPage(false); } + ParallelUnbalancedWork.For(0, Root.PageCount, pages, (i, p) => + { + p[i].Clear(); + return p; + }); + _root = new Root(pages); AllocateNewPage(); @@ -433,10 +440,8 @@ private Page RentNewPage(bool clear) return page; } - private static uint Mix(ulong hash) => unchecked((uint)((hash >> 32) ^ hash)); - private Span Write(int size, out UIntPtr addr) { if (BufferSize - _position < size) From a5c7d4197655182df2a6e0d90330f7ada457c318 Mon Sep 17 00:00:00 2001 From: scooletz Date: Thu, 19 Dec 2024 17:32:04 +0100 Subject: [PATCH 6/9] locked on write dictionary --- src/Paprika/Chain/PooledSpanDictionary.cs | 58 ++++++++++++++++------- 1 file changed, 42 insertions(+), 16 deletions(-) diff --git a/src/Paprika/Chain/PooledSpanDictionary.cs b/src/Paprika/Chain/PooledSpanDictionary.cs index 888e2a5c..953043cb 100644 --- a/src/Paprika/Chain/PooledSpanDictionary.cs +++ b/src/Paprika/Chain/PooledSpanDictionary.cs @@ -105,7 +105,10 @@ private SearchResult TryGetImpl(scoped ReadOnlySpan key, uint leftover, ui { Debug.Assert(BitOperations.LeadingZeroCount(leftover) >= 11, "First 10 bits should be left unused"); - var address = _root[(int)bucket]; + ref var location = ref _root[(int)bucket]; + + var address = Volatile.Read(ref location); + if (address == UIntPtr.Zero) goto NotFound; do @@ -279,8 +282,6 @@ private void SetImpl(scoped ReadOnlySpan key, uint mixed, ReadOnlySpan= 10, "First 10 bits should be left unused"); - UIntPtr root = _root[(int)bucket]; - var dataLength = data1.Length + data0.Length; var size = PreambleLength + AddressLength + KeyLengthLength + key.Length + ValueLengthLength + dataLength; @@ -291,9 +292,6 @@ private void SetImpl(scoped ReadOnlySpan key, uint mixed, ReadOnlySpan> 8); destination[2] = (byte)(leftover & 0xFF); - // Write next - Unsafe.WriteUnaligned(ref destination[PreambleLength], root); - // Key length const int keyStart = PreambleLength + AddressLength; destination[keyStart] = (byte)key.Length; @@ -309,7 +307,23 @@ private void SetImpl(scoped ReadOnlySpan key, uint mixed, ReadOnlySpan key, ulong hash) @@ -346,7 +360,7 @@ public bool MoveNext() return false; } - _address = dictionary._root[_bucket]; + _address = Volatile.Read(ref dictionary._root[_bucket]); } // Scan the bucket till it's not destroyed @@ -442,20 +456,32 @@ private Page RentNewPage(bool clear) private static uint Mix(ulong hash) => unchecked((uint)((hash >> 32) ^ hash)); + private readonly object _lock = new(); + private Span Write(int size, out UIntPtr addr) { - if (BufferSize - _position < size) + // Memoize to make contention as small as possible + Page current; + int position; + + lock (_lock) { - // not enough memory - AllocateNewPage(); + if (BufferSize - _position < size) + { + // not enough memory + AllocateNewPage(); + } + + position = _position; + current = _current; + + // Amend _position so that it can be used by other threads. + _position += size; } // allocated before the position is changed - var span = _current.Span.Slice(_position, size); - addr = _current.Raw + (UIntPtr)_position; - - _position += size; - return span; + addr = current.Raw + (UIntPtr)position; + return current.Span.Slice(position, size); } public void Dispose() From 85d402693eac424127f37a5f4bce07b6fed69d57 Mon Sep 17 00:00:00 2001 From: scooletz Date: Fri, 20 Dec 2024 11:49:52 +0100 Subject: [PATCH 7/9] align on alloc --- src/Paprika/Chain/PooledSpanDictionary.cs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/Paprika/Chain/PooledSpanDictionary.cs b/src/Paprika/Chain/PooledSpanDictionary.cs index 953043cb..d7181a81 100644 --- a/src/Paprika/Chain/PooledSpanDictionary.cs +++ b/src/Paprika/Chain/PooledSpanDictionary.cs @@ -307,7 +307,9 @@ private void SetImpl(scoped ReadOnlySpan key, uint mixed, ReadOnlySpan (value + (alignment - 1)) & -alignment; + private Span Write(int size, out UIntPtr addr) { // Memoize to make contention as small as possible Page current; int position; + // Align so that we don't step on each other too much. + size = Align(size, PointerSize); + lock (_lock) { if (BufferSize - _position < size) From f1512d3525871b82eefda6a061313b1f96f28f36 Mon Sep 17 00:00:00 2001 From: scooletz Date: Fri, 20 Dec 2024 14:34:15 +0100 Subject: [PATCH 8/9] no-copy child commit --- src/Paprika/Chain/Blockchain.cs | 62 +++++++-------------------------- 1 file changed, 12 insertions(+), 50 deletions(-) diff --git a/src/Paprika/Chain/Blockchain.cs b/src/Paprika/Chain/Blockchain.cs index 9cb224ee..bc343301 100644 --- a/src/Paprika/Chain/Blockchain.cs +++ b/src/Paprika/Chain/Blockchain.cs @@ -1168,73 +1168,35 @@ void ICommit.Visit(CommitAction action, TrieType type) } } - IChildCommit ICommit.GetChild() => new ChildCommit(Pool, this); + IChildCommit ICommit.GetChild() => new ChildCommit(this, this); public IReadOnlySet TouchedAccounts => _touchedAccounts; public IReadOnlyDictionary TouchedStorageSlots => _storageSlots; - class ChildCommit(BufferPool pool, ICommit parent) : RefCountingDisposable, IChildCommit + sealed class ChildCommit(ICommit parent, BlockState owner) : IChildCommit { - private readonly PooledSpanDictionary _dict = new(pool, true); + public ReadOnlySpanOwnerWithMetadata Get(scoped in Key key) => parent.Get(in key); - [SkipLocalsInit] - public ReadOnlySpanOwnerWithMetadata Get(scoped in Key key) - { - var hash = GetHash(key); - var keyWritten = key.WriteTo(stackalloc byte[key.MaxByteLength]); + public void Set(in Key key, in ReadOnlySpan payload, EntryType type = EntryType.Persistent) => + parent.Set(in key, in payload, type); - if (_dict.TryGet(keyWritten, hash, out var result)) - { - AcquireLease(); - return new ReadOnlySpanOwnerWithMetadata(new ReadOnlySpanOwner(result, this), 0); - } + public void Set(in Key key, in ReadOnlySpan payload0, in ReadOnlySpan payload1, + EntryType type = EntryType.Persistent) => parent.Set(in key, in payload0, in payload1, type); - // Don't nest, as reaching to parent should be easy. - return parent.Get(key); - } + public IChildCommit GetChild() => parent.GetChild(); - [SkipLocalsInit] - public void Set(in Key key, in ReadOnlySpan payload, EntryType type) - { - var hash = GetHash(key); - var keyWritten = key.WriteTo(stackalloc byte[key.MaxByteLength]); + public bool Owns(object? actualSpanOwner) => ReferenceEquals(actualSpanOwner, owner); - _dict.Set(keyWritten, hash, payload, (byte)type); - } - - [SkipLocalsInit] - public void Set(in Key key, in ReadOnlySpan payload0, in ReadOnlySpan payload1, EntryType type) + public void Dispose() { - var hash = GetHash(key); - var keyWritten = key.WriteTo(stackalloc byte[key.MaxByteLength]); - - _dict.Set(keyWritten, hash, payload0, payload1, (byte)type); + // NOOP, nothing to dispose } public void Commit() { - foreach (var kvp in _dict) - { - Key.ReadFrom(kvp.Key, out var key); - var type = (EntryType)kvp.Metadata; - - // flush down only volatiles - if (type != EntryType.UseOnce) - { - parent.Set(key, kvp.Value, type); - } - } - } - - public IChildCommit GetChild() => new ChildCommit(pool, this); - - protected override void CleanUp() - { - _dict.Dispose(); + // NOOP, nothing to commit } - - public override string ToString() => _dict.ToString(); } [SkipLocalsInit] From 58d2113c8a8e7eb62e9b6e7a35d4a7c340015a30 Mon Sep 17 00:00:00 2001 From: scooletz Date: Tue, 7 Jan 2025 11:49:27 +0100 Subject: [PATCH 9/9] tests --- src/Paprika.Tests/Merkle/Commit.cs | 12 +++--------- src/Paprika.Tests/Merkle/RootHashFuzzyTests.cs | 2 +- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/src/Paprika.Tests/Merkle/Commit.cs b/src/Paprika.Tests/Merkle/Commit.cs index 6707875d..285f5940 100644 --- a/src/Paprika.Tests/Merkle/Commit.cs +++ b/src/Paprika.Tests/Merkle/Commit.cs @@ -183,23 +183,17 @@ private static byte[] Concat(in ReadOnlySpan payload0, in ReadOnlySpan _data = new(Comparer); - public ChildCommit(ICommit commit) - { - _commit = commit; - } - public void Dispose() => _data.Clear(); public ReadOnlySpanOwnerWithMetadata Get(scoped in Key key) { return _data.TryGetValue(GetKey(key), out var value) ? new ReadOnlySpanOwner(value, null).WithDepth(0) - : _commit.Get(key); + : commit.Get(key); } public void Set(in Key key, in ReadOnlySpan payload, EntryType type) @@ -217,7 +211,7 @@ public void Commit() foreach (var kvp in _data) { Key.ReadFrom(kvp.Key, out var key); - _commit.Set(key, kvp.Value); + commit.Set(key, kvp.Value); } } diff --git a/src/Paprika.Tests/Merkle/RootHashFuzzyTests.cs b/src/Paprika.Tests/Merkle/RootHashFuzzyTests.cs index 9c59e9c9..4673f51f 100644 --- a/src/Paprika.Tests/Merkle/RootHashFuzzyTests.cs +++ b/src/Paprika.Tests/Merkle/RootHashFuzzyTests.cs @@ -50,7 +50,7 @@ public void Over_one_mock_commit(string test) [TestCase(nameof(Accounts_100_Storage_1), int.MaxValue, 4)] [TestCase(nameof(Accounts_1_Storage_100), 11, 8)] - [TestCase(nameof(Accounts_1000_Storage_1000), int.MaxValue, 1016, Category = Categories.LongRunning)] + [TestCase(nameof(Accounts_1000_Storage_1000), int.MaxValue, 2500, Category = Categories.LongRunning)] public async Task In_memory_run(string test, int commitEvery, int blockchainPoolSizeMB) { var generator = Build(test);