Merge branch 'main' into page-table

.github/workflows/benchmarks.yml

@@ -17,7 +17,7 @@ on:
 
 env:
   BUILD_CONFIG: release
-  DOTNET_VERSION: 8
+  DOTNET_VERSION: 9
   DOTNET_INSTALL_DIR: '~/.dotnet'
 
 jobs:

.github/workflows/code-formatting.yml

@@ -1,13 +1,12 @@
-name: 'Code formatting'
+name: "Code formatting"
 
 on:
   pull_request:
   workflow_dispatch:
 
 env:
   BUILD_CONFIG: release
-  DOTNET_VERSION: 7
-  DOTNET_INSTALL_DIR: '~/.dotnet'
+  DOTNET_VERSION: 9
 
 jobs:
   spacing-check:
@@ -16,22 +15,16 @@ jobs:
     permissions:
       contents: read
     steps:
-    - name: Cache dotnet
-      id: cache-dotnet
-      uses: actions/cache@v3
-      with:
-          path: ${{ env.DOTNET_INSTALL_DIR }}
-          key: ${{ runner.os }}-dotnet-${{ env.DOTNET_VERSION }}
-          restore-keys: ${{ runner.os }}-dotnet-${{ env.DOTNET_VERSION }}
-    - name: Set up .NET
-      if: ${{ steps.cache-dotnet.outputs.cache-hit != 'true' }}
-      uses: actions/setup-dotnet@v3
-      with:
-        dotnet-version: ${{ env.DOTNET_VERSION }}
-    - name: Check out repository
-      uses: actions/checkout@v3
-      with:
-        submodules: true
-    - name: Format
-      run: |
-        dotnet format whitespace src/Paprika.sln --verify-no-changes
+      - name: Set up .NET
+        uses: actions/setup-dotnet@v4
+        with:
+          dotnet-version: ${{ env.DOTNET_VERSION }}
+
+      - name: Check out repository
+        uses: actions/checkout@v4
+        with:
+          submodules: true
+
+      - name: Format
+        run: |
+          dotnet format whitespace src/Paprika.sln --verify-no-changes

.github/workflows/codeql.yml

@@ -14,8 +14,7 @@ on:
 
 env:
   BUILD_CONFIG: release
-  DOTNET_VERSION: 8
-  DOTNET_INSTALL_DIR: "~/.dotnet"
+  DOTNET_VERSION: 9
 
 jobs:
   analyze:
@@ -27,25 +26,16 @@ jobs:
       security-events: write
 
     steps:
-      - name: Checkout repository
-        uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
-        with:
-          submodules: true
-
-      - name: Cache dotnet
-        id: cache-dotnet
-        uses: actions/cache@v3
-        with:
-          path: ${{ env.DOTNET_INSTALL_DIR }}
-          key: ${{ runner.os }}-dotnet-${{ env.DOTNET_VERSION }}
-          restore-keys: ${{ runner.os }}-dotnet-${{ env.DOTNET_VERSION }}
-
       - name: Set up .NET
-        if: ${{ steps.cache-dotnet.outputs.cache-hit != 'true' }}
-        uses: actions/setup-dotnet@v3
+        uses: actions/setup-dotnet@v4
         with:
           dotnet-version: ${{ env.DOTNET_VERSION }}
 
+      - name: Check out repository
+        uses: actions/checkout@v4
+        with:
+          submodules: true
+
       - name: Initialize CodeQL
         uses: github/codeql-action/init@5618c9fc1e675841ca52c1c6b1304f5255a905a0 # v2.19.0
         with:

.github/workflows/tests.yml

@@ -12,8 +12,7 @@ on:
 
 env:
   BUILD_CONFIG: release
-  DOTNET_VERSION: 8
-  DOTNET_INSTALL_DIR: "~/.dotnet"
+  DOTNET_VERSION: 9
 
 jobs:
   tests:
@@ -24,32 +23,16 @@ jobs:
       contents: read
       pull-requests: write # for sticky-pull-request-comment
     steps:
-      - name: Cache dotnet
-        id: cache-dotnet
-        uses: actions/cache@v3
-        with:
-          path: ${{ env.DOTNET_INSTALL_DIR }}
-          key: ${{ runner.os }}-dotnet-${{ env.DOTNET_VERSION }}
-          restore-keys: ${{ runner.os }}-dotnet-${{ env.DOTNET_VERSION }}
-
       - name: Set up .NET
-        if: ${{ steps.cache-dotnet.outputs.cache-hit != 'true' }}
-        uses: actions/setup-dotnet@v3
+        uses: actions/setup-dotnet@v4
         with:
           dotnet-version: ${{ env.DOTNET_VERSION }}
 
-      - uses: actions/checkout@v3
+      - name: Check out repository
+        uses: actions/checkout@v4
         with:
           submodules: true
 
-      - name: Cache nuget
-        uses: actions/cache@v3
-        with:
-          path: ~/.nuget/packages
-          key: ${{ runner.os }}-nuget-${{ hashFiles('**/*.csproj') }}
-          restore-keys: |
-            ${{ runner.os }}-nuget-${{ hashFiles('**/*.csproj') }}
-
       - name: Test
         run: dotnet test src/Paprika.Tests -c ${{ env.BUILD_CONFIG }} --filter TestCategory\!~LongRunning --collect:"XPlat Code Coverage" --results-directory ./coverage
 

src/Directory.Build.props

@@ -0,0 +1,8 @@
+<Project>
+    <PropertyGroup>
+        <TargetFramework>net9.0</TargetFramework>
+        <ImplicitUsings>enable</ImplicitUsings>
+        <Nullable>enable</Nullable>
+        <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
+    </PropertyGroup>
+</Project>

src/Paprika.Benchmarks/Paprika.Benchmarks.csproj

@@ -1,11 +1,6 @@
 <Project Sdk="Microsoft.NET.Sdk">
-
     <PropertyGroup>
         <OutputType>Exe</OutputType>
-        <TargetFramework>net8.0</TargetFramework>
-        <ImplicitUsings>enable</ImplicitUsings>
-        <Nullable>enable</Nullable>
-        <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
     </PropertyGroup>
 
     <ItemGroup>

src/Paprika.Benchmarks/SlottedArrayBenchmarks.cs

@@ -14,6 +14,7 @@ private const int
         BytesPerKey =
             3; // 3 repeated bytes allow to cut off the first nibble and still have a unique key. Also, allow storing some key leftover
 
+
     private readonly void* _keys;
     private readonly void* _map;
 
@@ -25,8 +26,17 @@ private const int
     private readonly void* _hashCollidingKeys;
     private readonly void* _hashCollidingMap;
 
+    // Defragmentation
+    private const int DefragmentationKeyCount = 86;
+    private readonly void* _defragKeys;
+    private readonly void* _defragmentMap;
+    private readonly void* _randomKeysIndex;
+    private readonly void* _defragmentMapCopy;
+
     public SlottedArrayBenchmarks()
     {
+        var rand = new Random(13);
+
         // Create keys
         _keys = Allocator.AllocAlignedPage();
 
@@ -82,6 +92,51 @@ public SlottedArrayBenchmarks()
                 throw new Exception("Not enough memory");
             }
         }
+
+        // Defragmentation
+        _defragmentMap = Allocator.AllocAlignedPage();
+        _defragmentMapCopy = Allocator.AllocAlignedPage();
+        _defragKeys = Allocator.AllocAlignedPage();
+
+        var keysSpan = new Span<byte>(_defragKeys, Page.PageSize);
+        for (byte i = 0; i < DefragmentationKeyCount; i++)
+        {
+            for (var j = 0; j < BytesPerKey; j++)
+            {
+                keysSpan[i * BytesPerKey + j] = i;
+            }
+        }
+
+        var defragmentMap = new SlottedArray(new Span<byte>(_defragmentMap, Page.PageSize));
+
+        for (byte i = 0; i < DefragmentationKeyCount; i++)
+        {
+            Span<byte> data = new byte[i]; // make them vary by length
+            rand.NextBytes(data);
+
+            if (defragmentMap.TrySet(GetDefragKey(i, false), data) == false)
+            {
+                throw new Exception("Not enough memory");
+            }
+        }
+
+        // Delete the first key.
+        defragmentMap.Delete(GetDefragKey(0, false));
+
+        // Create randomly ordered keys index
+        _randomKeysIndex = Allocator.AllocAlignedPage();
+        var randomKeysIndexSpan = new Span<byte>(_randomKeysIndex, Page.PageSize);
+
+        // Form a list of all key indices which can be deleted, excluding the first one
+        // since it is already deleted.
+        for (byte i = 1; i < DefragmentationKeyCount; i++)
+        {
+            randomKeysIndexSpan[i - 1] = i;
+        }
+
+        // Randomly shuffle the indices.
+        var randomKeysSlice = randomKeysIndexSpan.Slice(0, DefragmentationKeyCount - 1);
+        rand.Shuffle(randomKeysSlice);
     }
 
     [Benchmark(OperationsPerInvoke = 4)]
@@ -191,6 +246,59 @@ public int EnumerateNibble(byte nibble)
         return length;
     }
 
+    [Benchmark]
+    public bool Defragmentation()
+    {
+        var map = CreateMapForDefragmentation();
+
+        // Insert additional key to trigger defragmentation
+        var additionalKey = Keccak.Zero;
+        Span<byte> value = stackalloc byte[32];
+
+        for (byte i = 0; i < 32; i++)
+        {
+            value[i] = i;
+        }
+
+        return map.TrySet(NibblePath.FromKey(additionalKey), value);
+    }
+
+    [Benchmark]
+    [Arguments(1)]
+    [Arguments(25)]
+    [Arguments(50)]
+    [Arguments(DefragmentationKeyCount - 1)]
+    public bool Defragmentation_Large(int count)
+    {
+        var map = CreateMapForDefragmentation();
+
+        // Delete random keys
+        for (var i = 0; i < count; i++)
+        {
+            map.Delete(GetRandomDefragKey((byte)i, false));
+        }
+
+        // Insert additional key to trigger defragmentation
+        var additionalKey = Keccak.Zero;
+        Span<byte> value = stackalloc byte[32];
+
+        for (byte i = 0; i < 32; i++)
+        {
+            value[i] = i;
+        }
+
+        return map.TrySet(NibblePath.FromKey(additionalKey), value);
+    }
+
+    private SlottedArray CreateMapForDefragmentation()
+    {
+        var source = new Span<byte>(_defragmentMap, Page.PageSize);
+        var destination = new Span<byte>(_defragmentMapCopy, Page.PageSize);
+        source.CopyTo(destination);
+
+        return new SlottedArray(destination);
+    }
+
     private NibblePath GetKey(byte i, bool odd)
     {
         var span = new Span<byte>(_keys, BytesPerKey * KeyCount);
@@ -207,4 +315,18 @@ private NibblePath GetHashCollidingKey(byte i)
         // Use full key
         return NibblePath.FromKey(slice, 0, BytesPerKeyHashColliding * NibblePath.NibblePerByte);
     }
+
+    private NibblePath GetRandomDefragKey(byte i, bool odd)
+    {
+        var span = new Span<byte>(_randomKeysIndex, KeyCount - 1);
+        return GetDefragKey(span[i], odd);
+    }
+
+    private NibblePath GetDefragKey(byte i, bool odd)
+    {
+        var span = new Span<byte>(_defragKeys, BytesPerKey * DefragmentationKeyCount);
+        var slice = span.Slice(i * BytesPerKey, BytesPerKey);
+
+        return NibblePath.FromKey(slice, odd ? 1 : 0, 4);
+    }
 }

src/Paprika.Cli/Paprika.Cli.csproj

@@ -2,13 +2,10 @@
 
     <PropertyGroup>
         <OutputType>Exe</OutputType>
-        <TargetFramework>net8.0</TargetFramework>
-        <ImplicitUsings>enable</ImplicitUsings>
-        <Nullable>enable</Nullable>
     </PropertyGroup>
 
     <ItemGroup>
-      <PackageReference Include="Spectre.Console.Cli" Version="0.48.1-preview.0.5" />
+      <PackageReference Include="Spectre.Console.Cli" Version="0.49.1" />
     </ItemGroup>
 
     <ItemGroup>

src/Paprika.Cli/Program.cs

@@ -9,7 +9,7 @@
     cfg.AddCommand<StorageVisitSettings.Command>("storage");
     cfg.AddCommand<VerifyWholeTreeSettings.Command>("verify");
 
-    cfg.SetExceptionHandler(ex =>
+    cfg.SetExceptionHandler((ex, _) =>
     {
         AnsiConsole.WriteException(ex, ExceptionFormats.ShortenEverything);
         return -99;

src/Paprika.Importer/Paprika.Importer.csproj

@@ -2,9 +2,6 @@
 
     <PropertyGroup>
         <OutputType>Exe</OutputType>
-        <TargetFramework>net8.0</TargetFramework>
-        <ImplicitUsings>enable</ImplicitUsings>
-        <Nullable>enable</Nullable>
     </PropertyGroup>
 
     <ItemGroup>

src/Paprika.Runner.Pareto/Paprika.Runner.Pareto.csproj

@@ -2,10 +2,6 @@
 
     <PropertyGroup>
         <OutputType>Exe</OutputType>
-        <TargetFramework>net8.0</TargetFramework>
-        <ImplicitUsings>enable</ImplicitUsings>
-        <Nullable>enable</Nullable>
-        <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
     </PropertyGroup>
 
     <PropertyGroup Condition=" '$(Configuration)' == 'Debug' ">
@@ -22,7 +18,7 @@
     </ItemGroup>
 
     <ItemGroup>
-      <PackageReference Include="Spectre.Console" Version="0.48.1-preview.0.32" />
+      <PackageReference Include="Spectre.Console" Version="0.49.1" />
     </ItemGroup>
 
 </Project>