From 8bc90be859494c645a68b50223c9836a7d8ce27e Mon Sep 17 00:00:00 2001 From: scooletz Date: Wed, 25 Sep 2024 13:16:49 +0200 Subject: [PATCH 1/8] prefetch moved to span based --- src/Paprika.Tests/Store/BasePageTests.cs | 7 +++--- src/Paprika/Store/BatchContextBase.cs | 2 +- src/Paprika/Store/IBatchContext.cs | 25 +++++++------------ .../Store/PageManagers/PointerPageManager.cs | 17 +++++++------ src/Paprika/Store/PagedDb.cs | 6 ++--- 5 files changed, 27 insertions(+), 30 deletions(-) diff --git a/src/Paprika.Tests/Store/BasePageTests.cs b/src/Paprika.Tests/Store/BasePageTests.cs index 402e4190..c0a07fb4 100644 --- a/src/Paprika.Tests/Store/BasePageTests.cs +++ b/src/Paprika.Tests/Store/BasePageTests.cs @@ -27,8 +27,9 @@ internal class TestBatchContext(uint batchId, Stack? reusable = null) public override Page GetAt(DbAddress address) => _address2Page[address]; - public override void Prefetch(DbAddress address) - { } + public override void Prefetch(ReadOnlySpan addresses) + { + } public override DbAddress GetAddress(Page page) => _page2Address[page.Raw]; @@ -106,4 +107,4 @@ public TestBatchContext Next() } internal static TestBatchContext NewBatch(uint batchId) => new(batchId); -} +} \ No newline at end of file diff --git a/src/Paprika/Store/BatchContextBase.cs b/src/Paprika/Store/BatchContextBase.cs index 28968a21..f23913a3 100644 --- a/src/Paprika/Store/BatchContextBase.cs +++ b/src/Paprika/Store/BatchContextBase.cs @@ -10,7 +10,7 @@ abstract class BatchContextBase(uint batchId) : IBatchContext public uint BatchId { get; } = batchId; public abstract Page GetAt(DbAddress address); - public abstract void Prefetch(DbAddress address); + public abstract void Prefetch(ReadOnlySpan addresses); public abstract DbAddress GetAddress(Page page); diff --git a/src/Paprika/Store/IBatchContext.cs b/src/Paprika/Store/IBatchContext.cs index 0ad72127..958e0644 100644 --- a/src/Paprika/Store/IBatchContext.cs +++ b/src/Paprika/Store/IBatchContext.cs @@ -1,5 +1,6 @@ using System.Diagnostics; using System.Diagnostics.CodeAnalysis; +using System.Runtime.InteropServices; using Paprika.Crypto; namespace Paprika.Store; @@ -126,29 +127,21 @@ public interface IPageResolver /// Page GetAt(DbAddress address); - void Prefetch(DbAddress address); + void Prefetch(DbAddress address) => Prefetch(MemoryMarshal.CreateReadOnlySpan(ref address, 1)); - void Prefetch(ReadOnlySpan addresses) - { - foreach (var bucket in addresses) - { - if (!bucket.IsNull) - { - Prefetch(bucket); - } - } - } + void Prefetch(ReadOnlySpan addresses); void Prefetch(in TAddressList addresses) where TAddressList : struct, DbAddressList.IDbAddressList { + Span span = stackalloc DbAddress[TAddressList.Length]; + + // Copy all for (var i = 0; i < TAddressList.Length; i++) { - var addr = addresses[i]; - if (!addr.IsNull) - { - Prefetch(addr); - } + span[i] = addresses[i]; } + + Prefetch(span); } } diff --git a/src/Paprika/Store/PageManagers/PointerPageManager.cs b/src/Paprika/Store/PageManagers/PointerPageManager.cs index 12d82cc2..9660a2fb 100644 --- a/src/Paprika/Store/PageManagers/PointerPageManager.cs +++ b/src/Paprika/Store/PageManagers/PointerPageManager.cs @@ -11,18 +11,21 @@ public abstract unsafe class PointerPageManager(long size) : IPageManager protected abstract void* Ptr { get; } - public void Prefetch(DbAddress address) + public void Prefetch(ReadOnlySpan addresses) { if (Sse.IsSupported) { - if (address.IsNull || address.Raw > (uint)MaxPage) + foreach (var address in addresses) { - return; - } + if (address.IsNull || address.Raw > (uint)MaxPage) + { + return; + } - // Fetch to L2 cache as we don't know if will need it - // So don't pollute L1 cache - Sse.Prefetch1((byte*)Ptr + address.FileOffset); + // Fetch to L2 cache as we don't know if will need it + // So don't pollute L1 cache + Sse.Prefetch1((byte*)Ptr + address.FileOffset); + } } } diff --git a/src/Paprika/Store/PagedDb.cs b/src/Paprika/Store/PagedDb.cs index aa17306e..429b57bf 100644 --- a/src/Paprika/Store/PagedDb.cs +++ b/src/Paprika/Store/PagedDb.cs @@ -126,7 +126,7 @@ public static PagedDb MemoryMappedDb(long size, byte historyDepth, string direct new MemoryMappedPageManager(size, historyDepth, directory, flushToDisk ? PersistenceOptions.FlushFile : PersistenceOptions.MMapOnly), historyDepth); - public void Prefetch(DbAddress address) => _manager.Prefetch(address); + public void Prefetch(ReadOnlySpan addresses) => _manager.Prefetch(addresses); private void ReportReads(long number) => _reads.Add(number); @@ -506,7 +506,7 @@ public IDictionary IdCache } } - public void Prefetch(DbAddress address) => db.Prefetch(address); + public void Prefetch(ReadOnlySpan addresses) => db.Prefetch(addresses); public Page GetAt(DbAddress address) => db._manager.GetAt(address); @@ -678,7 +678,7 @@ public override Page GetAt(DbAddress address) return page; } - public override void Prefetch(DbAddress address) => _db.Prefetch(address); + public override void Prefetch(ReadOnlySpan addresses) => _db.Prefetch(addresses); public override DbAddress GetAddress(Page page) => _db.GetAddress(page); From b2139510905d296cb023b9d57d486c81b95858cb Mon Sep 17 00:00:00 2001 From: scooletz Date: Wed, 25 Sep 2024 13:24:54 +0200 Subject: [PATCH 2/8] prefetch next when reusing abandoned --- src/Paprika/Store/AbandonedList.cs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/Paprika/Store/AbandonedList.cs b/src/Paprika/Store/AbandonedList.cs index 5f345b1d..6346b6d3 100644 --- a/src/Paprika/Store/AbandonedList.cs +++ b/src/Paprika/Store/AbandonedList.cs @@ -139,6 +139,13 @@ public bool TryGet(out DbAddress reused, uint minBatchId, IBatchContext batch) if (current.TryPop(out reused)) { + // Schedule prefetching next if possible + if (current.TryPeek(out var next, out _)) + { + Debug.Assert(next.IsNull == false, "Next should not be NULL here"); + batch.Prefetch(next); + } + return true; } From a4630e034bb80af2b2a618a82af6946568cbdf3d Mon Sep 17 00:00:00 2001 From: scooletz Date: Wed, 25 Sep 2024 14:36:08 +0200 Subject: [PATCH 3/8] prefetch implemented --- src/Paprika/Platform.cs | 96 +++++++++++++++++++ .../PageManagers/MemoryMappedPageManager.cs | 65 +++++++++++++ .../Store/PageManagers/PointerPageManager.cs | 2 +- 3 files changed, 162 insertions(+), 1 deletion(-) create mode 100644 src/Paprika/Platform.cs diff --git a/src/Paprika/Platform.cs b/src/Paprika/Platform.cs new file mode 100644 index 00000000..28cef282 --- /dev/null +++ b/src/Paprika/Platform.cs @@ -0,0 +1,96 @@ +using System.ComponentModel; +using System.Diagnostics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +namespace Paprika; + +public static class Platform +{ + public static void Prefetch(ReadOnlySpan addresses, UIntPtr size) => Manager.SchedulePrefetch(addresses, size); + + private static readonly IMemoryManager Manager = + IsPosix() ? new PosixMemoryManager() : new WindowsMemoryManager(); + + private static bool IsPosix() => RuntimeInformation.IsOSPlatform(OSPlatform.Linux) || + RuntimeInformation.IsOSPlatform(OSPlatform.OSX); + + private sealed class PosixMemoryManager : IMemoryManager + { + [Flags] + private enum Advice : int + { + // ReSharper disable InconsistentNaming + /// + /// Expect access in the near future. (Hence, it might be a good idea to read some pages ahead.) + /// + MADV_WILLNEED = 0x3, + + /// + /// Do not expect access in the near future. + /// (For the time being, the application is finished with the given range, + /// so the kernel can free resources associated with it.) + /// + MADV_DONTNEED = 0x4, + // ReSharper restore InconsistentNaming + } + + [DllImport("LIBC_6", SetLastError = true)] + static extern int madvise(IntPtr addr, UIntPtr length, Advice advice); + + public void SchedulePrefetch(ReadOnlySpan addresses, UIntPtr length) + { + // TODO: + } + } + + private sealed class WindowsMemoryManager : IMemoryManager + { + [DllImport("kernel32.dll", SetLastError = true)] + private static extern unsafe bool PrefetchVirtualMemory(IntPtr hProcess, ulong numberOfEntries, + Win32MemoryRangeEntry* entries, int flags); + + [StructLayout(LayoutKind.Sequential)] + private struct Win32MemoryRangeEntry + { + /// + /// Starting address of the memory range + /// + public IntPtr VirtualAddress; + + /// + /// Size of the memory range in bytes + /// + public UIntPtr NumberOfBytes; + } + + [SkipLocalsInit] + public unsafe void SchedulePrefetch(ReadOnlySpan addresses, UIntPtr length) + { + var count = addresses.Length; + var ptr = stackalloc Win32MemoryRangeEntry[count]; + var span = new Span(ptr, count); + + for (var i = 0; i < span.Length; i++) + { + span[i].VirtualAddress = (IntPtr)addresses[i]; + span[i].NumberOfBytes = length; + } + + const int reserved = 0; + + if (PrefetchVirtualMemory(Process.GetCurrentProcess().Handle, (ulong)count, ptr, reserved) == false) + { + throw new Win32Exception(Marshal.GetLastWin32Error()); + } + } + } + + private interface IMemoryManager + { + /// + /// Schedules an OS dependent prefetch. + /// + void SchedulePrefetch(ReadOnlySpan addresses, UIntPtr length); + } +} \ No newline at end of file diff --git a/src/Paprika/Store/PageManagers/MemoryMappedPageManager.cs b/src/Paprika/Store/PageManagers/MemoryMappedPageManager.cs index 36151c92..f203a653 100644 --- a/src/Paprika/Store/PageManagers/MemoryMappedPageManager.cs +++ b/src/Paprika/Store/PageManagers/MemoryMappedPageManager.cs @@ -2,7 +2,10 @@ using System.Diagnostics; using System.Diagnostics.Metrics; using System.IO.MemoryMappedFiles; +using System.Runtime.CompilerServices; using System.Runtime.InteropServices; +using System.Threading.Channels; +using System.Xml.Serialization; using Paprika.Utils; namespace Paprika.Store.PageManagers; @@ -24,6 +27,18 @@ public sealed class MemoryMappedPageManager : PointerPageManager private readonly MemoryMappedViewAccessor _whole; private readonly unsafe byte* _ptr; + // Prefetcher + private const int PrefetcherCapacity = 1000; + private readonly Channel _prefetches = Channel.CreateBounded(new BoundedChannelOptions(PrefetcherCapacity) + { + FullMode = BoundedChannelFullMode.DropOldest, + SingleReader = true, + SingleWriter = false, + Capacity = PrefetcherCapacity, + AllowSynchronousContinuations = false + }); + private readonly Task _prefetcher; + // Flusher section private readonly Stack _owners = new(); private readonly List _ownersUsed = new(); @@ -72,6 +87,8 @@ public unsafe MemoryMappedPageManager(long size, byte historyDepth, string dir, _meter = new Meter("Paprika.Store.PageManager"); _fileWrites = _meter.CreateHistogram("File writes", "Syscall", "Actual numbers of file writes issued"); _writeTime = _meter.CreateHistogram("Write time", "ms", "Time spent in writing"); + + _prefetcher = Task.Factory.StartNew(RunPrefetcher); } public static string GetPaprikaFilePath(string dir) => System.IO.Path.Combine(dir, PaprikaFileName); @@ -80,6 +97,51 @@ public unsafe MemoryMappedPageManager(long size, byte historyDepth, string dir, protected override unsafe void* Ptr => _ptr; + public override void Prefetch(ReadOnlySpan addresses) + { + var writer = _prefetches.Writer; + + foreach (var address in addresses) + { + if (address.IsNull == false) + { + writer.TryWrite(address); + } + } + } + + private async Task RunPrefetcher() + { + var reader = _prefetches.Reader; + + while (await reader.WaitToReadAsync()) + { + PrefetchImpl(reader, this); + } + + [SkipLocalsInit] + static void PrefetchImpl(ChannelReader reader, MemoryMappedPageManager manager) + { + const int maxPrefetch = 128; + + Span span = stackalloc UIntPtr[maxPrefetch]; + var i = 0; + + for (; i < maxPrefetch; i++) + { + if (reader.TryRead(out var address) == false) + break; + + span[i] = manager.GetAt(address).Raw; + } + + if (i > 0) + { + Platform.Prefetch(span[..i], Page.PageSize); + } + } + } + public override async ValueTask FlushPages(ICollection dbAddresses, CommitOptions options) { if (_options == PersistenceOptions.MMapOnly) @@ -182,6 +244,9 @@ public override void ForceFlush() public override void Dispose() { + _prefetches.Writer.Complete(); + _prefetcher.Wait(); + _meter.Dispose(); _whole.SafeMemoryMappedViewHandle.ReleasePointer(); diff --git a/src/Paprika/Store/PageManagers/PointerPageManager.cs b/src/Paprika/Store/PageManagers/PointerPageManager.cs index 9660a2fb..0d3dd0bc 100644 --- a/src/Paprika/Store/PageManagers/PointerPageManager.cs +++ b/src/Paprika/Store/PageManagers/PointerPageManager.cs @@ -11,7 +11,7 @@ public abstract unsafe class PointerPageManager(long size) : IPageManager protected abstract void* Ptr { get; } - public void Prefetch(ReadOnlySpan addresses) + public virtual void Prefetch(ReadOnlySpan addresses) { if (Sse.IsSupported) { From 43a9879926fa8dda33b751e1fd782d636485a1c1 Mon Sep 17 00:00:00 2001 From: scooletz Date: Wed, 25 Sep 2024 14:36:27 +0200 Subject: [PATCH 4/8] visitor augmented with the new prefetch --- src/Paprika/Store/DataPage.cs | 7 +++++++ src/Paprika/Store/StorageFanOut.cs | 10 ++++++++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/Paprika/Store/DataPage.cs b/src/Paprika/Store/DataPage.cs index a4f8ff14..7bae3aaf 100644 --- a/src/Paprika/Store/DataPage.cs +++ b/src/Paprika/Store/DataPage.cs @@ -606,6 +606,13 @@ public void Accept(ref NibblePath.Builder builder, IPageVisitor visitor, IPageRe continue; } + // prefetch + var next = i + 1; + if (next < DbAddressList.Of16.Count) + { + resolver.Prefetch(Data.Buckets[next]); + } + var child = resolver.GetAt(bucket); if (IsFanOut) diff --git a/src/Paprika/Store/StorageFanOut.cs b/src/Paprika/Store/StorageFanOut.cs index 5baab52a..1b00cddc 100644 --- a/src/Paprika/Store/StorageFanOut.cs +++ b/src/Paprika/Store/StorageFanOut.cs @@ -479,13 +479,19 @@ public Page DeleteByPrefix(in NibblePath prefix, IBatchContext batch) public void Accept(ref NibblePath.Builder builder, IPageVisitor visitor, IPageResolver resolver, DbAddress addr) { - resolver.Prefetch(Data.Addresses); - using var scope = visitor.On(this, addr); for (var i = 0; i < DbAddressList.Of256.Length; i++) { var bucket = Data.Addresses[i]; + + // prefetch next + var next = i + 1; + if (next < DbAddressList.Of256.Count) + { + resolver.Prefetch(Data.Addresses[next]); + } + if (!bucket.IsNull) { builder.Push((byte)(i >> NibblePath.NibbleShift), (byte)(i & NibblePath.NibbleMask)); From 1cbb1fb1e6ef59f5df3898334b21f1983155d5f2 Mon Sep 17 00:00:00 2001 From: scooletz Date: Wed, 25 Sep 2024 14:50:47 +0200 Subject: [PATCH 5/8] minor --- src/Paprika/Store/IBatchContext.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Paprika/Store/IBatchContext.cs b/src/Paprika/Store/IBatchContext.cs index 958e0644..59891993 100644 --- a/src/Paprika/Store/IBatchContext.cs +++ b/src/Paprika/Store/IBatchContext.cs @@ -1,5 +1,6 @@ using System.Diagnostics; using System.Diagnostics.CodeAnalysis; +using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using Paprika.Crypto; @@ -131,6 +132,7 @@ public interface IPageResolver void Prefetch(ReadOnlySpan addresses); + [SkipLocalsInit] void Prefetch(in TAddressList addresses) where TAddressList : struct, DbAddressList.IDbAddressList { From 046538552cd70d85ed2b7655758554ecad8f72aa Mon Sep 17 00:00:00 2001 From: scooletz Date: Mon, 30 Sep 2024 10:23:33 +0200 Subject: [PATCH 6/8] different kinds of prefetch --- src/Paprika.Tests/Store/BasePageTests.cs | 4 +++ src/Paprika/Store/AbandonedList.cs | 5 ++- src/Paprika/Store/BatchContextBase.cs | 2 ++ src/Paprika/Store/DataPage.cs | 5 +-- src/Paprika/Store/IBatchContext.cs | 24 +++++++++++++- .../PageManagers/MemoryMappedPageManager.cs | 9 +++-- .../PageManagers/NativeMemoryPageManager.cs | 10 ++++++ .../Store/PageManagers/PointerPageManager.cs | 33 +++++++++++++------ src/Paprika/Store/PagedDb.cs | 6 ++++ 9 files changed, 81 insertions(+), 17 deletions(-) diff --git a/src/Paprika.Tests/Store/BasePageTests.cs b/src/Paprika.Tests/Store/BasePageTests.cs index c0a07fb4..a8c256f2 100644 --- a/src/Paprika.Tests/Store/BasePageTests.cs +++ b/src/Paprika.Tests/Store/BasePageTests.cs @@ -26,6 +26,10 @@ internal class TestBatchContext(uint batchId, Stack? reusable = null) private uint _pageCount = 1U; public override Page GetAt(DbAddress address) => _address2Page[address]; + public override void Prefetch(DbAddress address, PrefetchMode mode) + { + + } public override void Prefetch(ReadOnlySpan addresses) { diff --git a/src/Paprika/Store/AbandonedList.cs b/src/Paprika/Store/AbandonedList.cs index 6346b6d3..794ffcd5 100644 --- a/src/Paprika/Store/AbandonedList.cs +++ b/src/Paprika/Store/AbandonedList.cs @@ -143,7 +143,10 @@ public bool TryGet(out DbAddress reused, uint minBatchId, IBatchContext batch) if (current.TryPeek(out var next, out _)) { Debug.Assert(next.IsNull == false, "Next should not be NULL here"); - batch.Prefetch(next); + + // Expect that the page that will be prefetched here was not used for a while. + // It's ok to wait a bit to have it prefetched. + batch.Prefetch(next, PrefetchMode.Heavy); } return true; diff --git a/src/Paprika/Store/BatchContextBase.cs b/src/Paprika/Store/BatchContextBase.cs index f23913a3..3b55bd22 100644 --- a/src/Paprika/Store/BatchContextBase.cs +++ b/src/Paprika/Store/BatchContextBase.cs @@ -10,6 +10,8 @@ abstract class BatchContextBase(uint batchId) : IBatchContext public uint BatchId { get; } = batchId; public abstract Page GetAt(DbAddress address); + public abstract void Prefetch(DbAddress address, PrefetchMode mode); + public abstract void Prefetch(ReadOnlySpan addresses); public abstract DbAddress GetAddress(Page page); diff --git a/src/Paprika/Store/DataPage.cs b/src/Paprika/Store/DataPage.cs index 77f51229..183dda6e 100644 --- a/src/Paprika/Store/DataPage.cs +++ b/src/Paprika/Store/DataPage.cs @@ -557,10 +557,11 @@ private static bool TryGet(IReadOnlyBatchContext batch, scoped in NibblePath key if (!sliced.IsEmpty) { // As the CPU does not auto-prefetch across page boundaries - // Prefetch child page in case we go there next to reduce CPU stalls + // Prefetch child page in case we go there next to reduce CPU stalls. + // Do it using soft mode as quite likely the page is loaded to RAM but possibly not in CPU cache. bucket = page.Data.Buckets[GetIndex(sliced)]; if (bucket.IsNull == false) - batch.Prefetch(bucket); + batch.Prefetch(bucket, PrefetchMode.Soft); } // try regular map diff --git a/src/Paprika/Store/IBatchContext.cs b/src/Paprika/Store/IBatchContext.cs index 59891993..21887fc5 100644 --- a/src/Paprika/Store/IBatchContext.cs +++ b/src/Paprika/Store/IBatchContext.cs @@ -128,8 +128,16 @@ public interface IPageResolver /// Page GetAt(DbAddress address); - void Prefetch(DbAddress address) => Prefetch(MemoryMarshal.CreateReadOnlySpan(ref address, 1)); + /// + /// Issues a prefetch request for the page at the specific location + /// using mechanism defined by the . + /// + void Prefetch(DbAddress address, PrefetchMode mode); + /// + /// Issues a prefetch request for a set of pages residing at . + /// The prefetch mode that is used is . + /// void Prefetch(ReadOnlySpan addresses); [SkipLocalsInit] @@ -147,3 +155,17 @@ void Prefetch(in TAddressList addresses) Prefetch(span); } } + +public enum PrefetchMode +{ + /// + /// Expects that the page was not evicted and only should be brought to CPU case using SSE prefetch. + /// + Soft, + + /// + /// Expects that the page was not accessed lately or was evicted from the memory. + /// The page should be prefetched using platform specific heavy prefetch . + /// + Heavy, +} diff --git a/src/Paprika/Store/PageManagers/MemoryMappedPageManager.cs b/src/Paprika/Store/PageManagers/MemoryMappedPageManager.cs index f203a653..ae95a0d7 100644 --- a/src/Paprika/Store/PageManagers/MemoryMappedPageManager.cs +++ b/src/Paprika/Store/PageManagers/MemoryMappedPageManager.cs @@ -5,7 +5,6 @@ using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Threading.Channels; -using System.Xml.Serialization; using Paprika.Utils; namespace Paprika.Store.PageManagers; @@ -15,8 +14,10 @@ public sealed class MemoryMappedPageManager : PointerPageManager private readonly PersistenceOptions _options; /// - /// The only option is random access. As Paprika jumps over the file, any prefetching is futile. - /// Also, the file cannot be async to use some of the mmap features. So here it is, random access file. + /// As Paprika jumps to various addresses in the file, using + /// would be harmful and is used. + /// + /// The file uses to issue proper async operations. /// private const FileOptions PaprikaFileOptions = FileOptions.RandomAccess | FileOptions.Asynchronous; @@ -97,6 +98,8 @@ public unsafe MemoryMappedPageManager(long size, byte historyDepth, string dir, protected override unsafe void* Ptr => _ptr; + protected override void PrefetchHeavy(DbAddress address) => _prefetches.Writer.TryWrite(address); + public override void Prefetch(ReadOnlySpan addresses) { var writer = _prefetches.Writer; diff --git a/src/Paprika/Store/PageManagers/NativeMemoryPageManager.cs b/src/Paprika/Store/PageManagers/NativeMemoryPageManager.cs index a99f3749..a1f155d5 100644 --- a/src/Paprika/Store/PageManagers/NativeMemoryPageManager.cs +++ b/src/Paprika/Store/PageManagers/NativeMemoryPageManager.cs @@ -19,6 +19,16 @@ public NativeMemoryPageManager(long size, byte historyDepth) : base(size) protected override void* Ptr => _ptr; + protected override void PrefetchHeavy(DbAddress address) => PrefetchSoft(address); + + public override void Prefetch(ReadOnlySpan addresses) + { + foreach (var address in addresses) + { + PrefetchSoft(address); + } + } + public override void Flush() { } diff --git a/src/Paprika/Store/PageManagers/PointerPageManager.cs b/src/Paprika/Store/PageManagers/PointerPageManager.cs index 0d3dd0bc..28a2cd2b 100644 --- a/src/Paprika/Store/PageManagers/PointerPageManager.cs +++ b/src/Paprika/Store/PageManagers/PointerPageManager.cs @@ -11,21 +11,34 @@ public abstract unsafe class PointerPageManager(long size) : IPageManager protected abstract void* Ptr { get; } - public virtual void Prefetch(ReadOnlySpan addresses) + public void Prefetch(DbAddress address, PrefetchMode mode) + { + if (mode == PrefetchMode.Soft) + { + PrefetchSoft(address); + } + else + { + PrefetchHeavy(address); + } + } + + protected abstract void PrefetchHeavy(DbAddress address); + + public abstract void Prefetch(ReadOnlySpan addresses); + + protected void PrefetchSoft(DbAddress address) { if (Sse.IsSupported) { - foreach (var address in addresses) + if (address.IsNull || address.Raw > (uint)MaxPage) { - if (address.IsNull || address.Raw > (uint)MaxPage) - { - return; - } - - // Fetch to L2 cache as we don't know if will need it - // So don't pollute L1 cache - Sse.Prefetch1((byte*)Ptr + address.FileOffset); + return; } + + // Fetch to L2 cache as we don't know if will need it + // So don't pollute L1 cache + Sse.Prefetch1((byte*)Ptr + address.FileOffset); } } diff --git a/src/Paprika/Store/PagedDb.cs b/src/Paprika/Store/PagedDb.cs index 429b57bf..6dc875e3 100644 --- a/src/Paprika/Store/PagedDb.cs +++ b/src/Paprika/Store/PagedDb.cs @@ -126,6 +126,8 @@ public static PagedDb MemoryMappedDb(long size, byte historyDepth, string direct new MemoryMappedPageManager(size, historyDepth, directory, flushToDisk ? PersistenceOptions.FlushFile : PersistenceOptions.MMapOnly), historyDepth); + public void Prefetch(DbAddress address, PrefetchMode mode) => _manager.Prefetch(address, mode); + public void Prefetch(ReadOnlySpan addresses) => _manager.Prefetch(addresses); private void ReportReads(long number) => _reads.Add(number); @@ -506,6 +508,8 @@ public IDictionary IdCache } } + public void Prefetch(DbAddress address, PrefetchMode mode) => db.Prefetch(address, mode); + public void Prefetch(ReadOnlySpan addresses) => db.Prefetch(addresses); public Page GetAt(DbAddress address) => db._manager.GetAt(address); @@ -678,6 +682,8 @@ public override Page GetAt(DbAddress address) return page; } + public override void Prefetch(DbAddress address, PrefetchMode mode) => _db.Prefetch(address, mode); + public override void Prefetch(ReadOnlySpan addresses) => _db.Prefetch(addresses); public override DbAddress GetAddress(Page page) => _db.GetAddress(page); From 2d766b1d39157e6063b3fd29ee4a5cc0f531a81e Mon Sep 17 00:00:00 2001 From: scooletz Date: Thu, 3 Oct 2024 11:31:05 +0200 Subject: [PATCH 7/8] Linux and MacOS covered --- src/Paprika/Platform.cs | 50 ++++++++++++++++--- .../PageManagers/MemoryMappedPageManager.cs | 16 +++--- 2 files changed, 52 insertions(+), 14 deletions(-) diff --git a/src/Paprika/Platform.cs b/src/Paprika/Platform.cs index 28cef282..b5ecd49f 100644 --- a/src/Paprika/Platform.cs +++ b/src/Paprika/Platform.cs @@ -3,11 +3,13 @@ using System.Runtime.CompilerServices; using System.Runtime.InteropServices; +using AddressRange = (System.UIntPtr, uint); + namespace Paprika; public static class Platform { - public static void Prefetch(ReadOnlySpan addresses, UIntPtr size) => Manager.SchedulePrefetch(addresses, size); + public static void Prefetch(ReadOnlySpan ranges) => Manager.SchedulePrefetch(ranges); private static readonly IMemoryManager Manager = IsPosix() ? new PosixMemoryManager() : new WindowsMemoryManager(); @@ -38,9 +40,41 @@ private enum Advice : int [DllImport("LIBC_6", SetLastError = true)] static extern int madvise(IntPtr addr, UIntPtr length, Advice advice); - public void SchedulePrefetch(ReadOnlySpan addresses, UIntPtr length) + // For Linux + [DllImport("libc", SetLastError = true)] + private static extern IntPtr __errno_location(); + + // For macOS + [DllImport("libc", SetLastError = true)] + private static extern IntPtr __error(); + + public static int GetErrno() { - // TODO: + if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) + { + return Marshal.ReadInt32(__errno_location()); + } + + if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) + { + return Marshal.ReadInt32(__error()); + } + + throw new PlatformNotSupportedException("This platform is not supported."); + } + + public void SchedulePrefetch(ReadOnlySpan ranges) + { + const int success = 0; + + for (var i = 0; i < ranges.Length; i++) + { + var result = madvise((IntPtr)ranges[i].Item1, ranges[i].Item2, Advice.MADV_WILLNEED); + if (result != success) + { + throw new SystemException($"{nameof(madvise)} failed with the following error: {GetErrno()}"); + } + } } } @@ -65,16 +99,16 @@ private struct Win32MemoryRangeEntry } [SkipLocalsInit] - public unsafe void SchedulePrefetch(ReadOnlySpan addresses, UIntPtr length) + public unsafe void SchedulePrefetch(ReadOnlySpan ranges) { - var count = addresses.Length; + var count = ranges.Length; var ptr = stackalloc Win32MemoryRangeEntry[count]; var span = new Span(ptr, count); for (var i = 0; i < span.Length; i++) { - span[i].VirtualAddress = (IntPtr)addresses[i]; - span[i].NumberOfBytes = length; + span[i].VirtualAddress = (IntPtr)ranges[i].Item1; + span[i].NumberOfBytes = ranges[i].Item2; } const int reserved = 0; @@ -91,6 +125,6 @@ private interface IMemoryManager /// /// Schedules an OS dependent prefetch. /// - void SchedulePrefetch(ReadOnlySpan addresses, UIntPtr length); + void SchedulePrefetch(ReadOnlySpan addresses); } } \ No newline at end of file diff --git a/src/Paprika/Store/PageManagers/MemoryMappedPageManager.cs b/src/Paprika/Store/PageManagers/MemoryMappedPageManager.cs index ae95a0d7..a3fb64f6 100644 --- a/src/Paprika/Store/PageManagers/MemoryMappedPageManager.cs +++ b/src/Paprika/Store/PageManagers/MemoryMappedPageManager.cs @@ -122,12 +122,14 @@ private async Task RunPrefetcher() PrefetchImpl(reader, this); } + return; + [SkipLocalsInit] static void PrefetchImpl(ChannelReader reader, MemoryMappedPageManager manager) { const int maxPrefetch = 128; - Span span = stackalloc UIntPtr[maxPrefetch]; + Span<(UIntPtr, uint)> span = stackalloc (UIntPtr, uint)[maxPrefetch]; var i = 0; for (; i < maxPrefetch; i++) @@ -135,13 +137,15 @@ static void PrefetchImpl(ChannelReader reader, MemoryMappedPageManage if (reader.TryRead(out var address) == false) break; - span[i] = manager.GetAt(address).Raw; + span[i] = (manager.GetAt(address).Raw, Page.PageSize); } - if (i > 0) - { - Platform.Prefetch(span[..i], Page.PageSize); - } + if (i == 0) + return; + + // TODO: potentially sort and merge consecutive chunks + + Platform.Prefetch(span[..i]); } } From 8748cea5983aac30b5669d2c4f19f8171644d68b Mon Sep 17 00:00:00 2001 From: scooletz Date: Tue, 15 Oct 2024 12:07:42 +0200 Subject: [PATCH 8/8] merged --- src/Paprika/Store/StorageFanOut.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Paprika/Store/StorageFanOut.cs b/src/Paprika/Store/StorageFanOut.cs index d98d58d6..f2f5fc04 100644 --- a/src/Paprika/Store/StorageFanOut.cs +++ b/src/Paprika/Store/StorageFanOut.cs @@ -631,12 +631,12 @@ public void Accept(ref NibblePath.Builder builder, IPageVisitor visitor, IPageRe new DataPage(resolver.GetAt(Root)).Accept(ref builder, visitor, resolver, Root); } - public void Prefetch(IPageResolver resolver) + public void PrefetchForAccept(IPageResolver resolver) { if (Root.IsNull) return; - resolver.Prefetch(Root); + resolver.Prefetch(Root, PrefetchMode.Heavy); } } @@ -646,7 +646,7 @@ public void Accept(ref NibblePath.Builder builder, IPageVisitor visitor, IPageRe foreach (ref var bucket in Data.Buckets) { - bucket.Prefetch(resolver); + bucket.PrefetchForAccept(resolver); } foreach (ref var bucket in Data.Buckets)