asan: instrumented LuaJIT memory allocator

This patch adds instrumentation of the internal LuaJIT memory allocator.
It now enables the detection of memory-related errors when using FFI and
within LuaJIT itself. This enhancement improves reliability and debugging
capabilities.

This patch introduces two scenarios for using ASAN with LuaJIT:
- LuaJIT using sysmalloc: `-DLUAJIT_USE_ASAN=ON`
- LuaJIT using internal memory allocator: `-DLUAJIT_USE_ASAN_HARDENING=ON`

If you want to skip tests when LuaJIT uses the internal memory allocator,
you can check the `LJ_ASAN_HARDENING` environment variable.

The test `test/tarantool-tests/lj-1034-tabov-error-frame.test.lua` has
been disabled under ASAN & LuaJIT's internal allocator due to consistently
failing with a timeout.

Part of #10231

.github/workflows/sanitizers-testing.yml

@@ -34,17 +34,23 @@ jobs:
         # XXX: Let's start with only Linux/x86_64
         BUILDTYPE: [Debug, Release]
         CC: [gcc-10, clang-11]
+        ALLOCATOR: [sysmalloc, dlmalloc]
         include:
           - BUILDTYPE: Debug
             CMAKEFLAGS: -DCMAKE_BUILD_TYPE=Debug -DLUA_USE_ASSERT=ON -DLUA_USE_APICHECK=ON
           - BUILDTYPE: Release
             CMAKEFLAGS: -DCMAKE_BUILD_TYPE=RelWithDebInfo
+          - ALLOCATOR: sysmalloc
+            ASANFLAGS: -DLUAJIT_USE_ASAN=ON -DLUAJIT_USE_SYSMALLOC=ON
+          - ALLOCATOR: dlmalloc
+            ASANFLAGS: -DLUAJIT_USE_ASAN_HARDENING=ON
     runs-on: [self-hosted, regular, Linux, x86_64]
     name: >
       LuaJIT with ASan and UBSan (Linux/x86_64)
       ${{ matrix.BUILDTYPE }}
       CC:${{ matrix.CC }}
-      GC64:ON SYSMALLOC:ON
+      GC64:ON
+      ALLOC=${{ matrix.ALLOCATOR }}
     steps:
       - uses: actions/checkout@v4
         with:
@@ -55,24 +61,18 @@ jobs:
         with:
           cc_name: ${{ matrix.CC }}
       - name: configure
-        # XXX: LuaJIT configuration requires a couple of tweaks:
-        # LUAJIT_USE_SYSMALLOC=ON: Unfortunately, internal LuaJIT
-        #   memory allocator is not instrumented yet, so to find
-        #   any memory errors it's better to build LuaJIT with
-        #   system provided memory allocator (i.e. run CMake
-        #   configuration phase with -DLUAJIT_USE_SYSMALLOC=ON).
-        #   For more info, see root CMakeLists.txt.
         # LUAJIT_ENABLE_GC64=ON: LUAJIT_USE_SYSMALLOC cannot be
         #   enabled on x64 without GC64, since realloc usually
         #   doesn't return addresses in the right address range.
+        #   Additionally, the ASAN instrumentation for LuaJIT's
+        #   internal memory allocator is only available for GC64.
         #   For more info, see root CMakeLists.txt.
         run: >
           cmake -S . -B ${{ env.BUILDDIR }}
           -G Ninja
           ${{ matrix.CMAKEFLAGS }}
           -DLUAJIT_ENABLE_GC64=ON
-          -DLUAJIT_USE_ASAN=ON
-          -DLUAJIT_USE_SYSMALLOC=ON
+          ${{ matrix.ASANFLAGS }}
           -DLUAJIT_USE_UBSAN=ON
       - name: build
         run: cmake --build . --parallel

CMakeLists.txt

@@ -284,17 +284,34 @@ if(LUA_USE_ASSERT)
   AppendFlags(TARGET_C_FLAGS -DLUA_USE_ASSERT)
 endif()
 
+option(LUAJIT_USE_ASAN "Build LuaJIT with AddressSanitizer" OFF)
+# Same as LUAJIT_USE_ASAN, but using the internal LuaJIT memory
+# allocator instrumented with ASAN.
+option(LUAJIT_USE_ASAN_HARDENING "Build LuaJIT with an internal allocator with integrated AddressSanitizer" OFF)
+if(LUAJIT_USE_ASAN_HARDENING)
+  set(LUAJIT_USE_ASAN ON)
+  if(NOT LUAJIT_ENABLE_GC64)
+    message(FATAL_ERROR
+      "ASAN only with GC64."
+    )
+  endif()
+  AppendFlags(CMAKE_C_FLAGS
+    # Enable ASAN instrumentation of internal LuaJIT memory allocator
+    # see (src/lj_alloc.c)
+    -DLUAJIT_USE_ASAN_HARDENING
+  )
+endif()
+
 # Turn on AddressSanitizer support. As a result, all artefacts
 # (i.e. buildvm, LuaJIT, testing infrastructure) are built with
 # ASan enabled.
-option(LUAJIT_USE_ASAN "Build LuaJIT with AddressSanitizer" OFF)
 if(LUAJIT_USE_ASAN)
-  if(NOT LUAJIT_USE_SYSMALLOC)
+  if(NOT (LUAJIT_USE_SYSMALLOC OR LUAJIT_USE_ASAN_HARDENING))
     message(WARNING
-      "Unfortunately, internal LuaJIT memory allocator is not instrumented yet,"
-      " so to find any memory errors it's better to build LuaJIT with system"
-      " provided memory allocator (i.e. run CMake configuration phase with"
-      " -DLUAJIT_USE_SYSMALLOC=ON)."
+      "Run CMake configuration phase with -DLUAJIT_USE_SYSMALLOC=ON "
+      "to use system memory allocator or replace -DLUAJIT_USE_ASAN=ON "
+      "to -DLUAJIT_USE_ASAN_HARDENING=ON to use internal LuaJIT memory"
+      "allocator."
     )
   endif()
   # Use all recommendations described in AddressSanitize docs: