add compilation time benchmarks (#50)

nautilus/src/nautilus/compiler/backends/mlir/MLIRCompilationBackend.cpp

@@ -55,7 +55,12 @@ std::unique_ptr<Executable> MLIRCompilationBackend::compile(const std::shared_pt
 	// 4. JIT compile LLVM IR module and return engine that provides access
 	// compiled execute function.
 	auto engine = JITCompiler::jitCompileModule(mlirModule, optPipeline, loweringProvider->getJitProxyFunctionSymbols(), loweringProvider->getJitProxyTargetAddresses());
-
+	if (options.getOptionOrDefault("mlir.eager_compilation", false)) {
+		auto result = engine->lookupPacked("execute");
+		if (!result) {
+			llvm::errs() << "Could not compile function" << result.takeError() << "\n";
+		}
+	}
 	// 5. Get execution function from engine. Create and return execution context.
 	return std::make_unique<MLIRExecutable>(std::move(engine));
 }

nautilus/test/CMakeLists.txt

@@ -4,6 +4,7 @@ include(Catch)
 
 set(TEST_DATA_FOLDER "${CMAKE_CURRENT_SOURCE_DIR}/test/data/")
 
+add_subdirectory(benchmark)
 add_subdirectory(execution-tests)
 add_subdirectory(val-tests)
 

nautilus/test/benchmark/CMakeLists.txt

@@ -0,0 +1,15 @@
+if (ENABLE_TRACING)
+    add_executable(nautilus-tracing-benchmarks
+            TracingBenchmark.cpp
+    )
+    target_include_directories(nautilus-tracing-benchmarks PRIVATE
+            $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/../common>
+            $<INSTALL_INTERFACE:src/nautilus/>)
+    target_include_directories(nautilus-tracing-benchmarks PRIVATE
+            $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/../../src>
+            $<INSTALL_INTERFACE:src/nautilus/>)
+
+    target_link_libraries(nautilus-tracing-benchmarks PUBLIC nautilus Catch2::Catch2WithMain)
+    list(APPEND CMAKE_MODULE_PATH ${catch2_SOURCE_DIR}/extras)
+    catch_discover_tests(nautilus-tracing-benchmarks EXTRA_ARGS --allow-running-no-tests)
+endif ()

nautilus/test/execution-tests/TracingBenchmark.cpp → nautilus/test/benchmark/TracingBenchmark.cpp

@@ -10,6 +10,7 @@
 #include "StaticLoopFunctions.hpp"
 #include "TracingUtil.hpp"
 #include "nautilus/Engine.hpp"
+#include "nautilus/compiler/backends/mlir/MLIRCompilationBackend.hpp"
 #include "nautilus/compiler/ir/IRGraph.hpp"
 #include "nautilus/config.hpp"
 #include "nautilus/tracing/ExecutionTrace.hpp"
@@ -77,4 +78,43 @@ TEST_CASE("IR Creation Benchmark") {
 	}
 }
 
+TEST_CASE("Backend Compilation Benchmark") {
+
+	auto registry = compiler::CompilationBackendRegistry();
+
+	std::vector<std::string> backends = {};
+#ifdef ENABLE_MLIR_BACKEND
+	backends.emplace_back("mlir");
+#endif
+#ifdef ENABLE_C_BACKEND
+	backends.emplace_back("cpp");
+#endif
+#ifdef ENABLE_BC_BACKEND
+	backends.emplace_back("bc");
+#endif
+#ifdef ENABLE_ASMJIT_BACKEND
+	backends.emplace_back("asmjit");
+#endif
+	for (auto& backend : backends) {
+		for (auto& test : tests) {
+			auto func = std::get<1>(test);
+			auto name = std::get<0>(test);
+
+			Catch::Benchmark::Benchmark("backend_compilation_" + backend + "_" + name).operator=([&func, &registry](Catch::Benchmark::Chronometer meter) {
+				std::shared_ptr<tracing::ExecutionTrace> trace = tracing::TraceContext::trace(func);
+				auto ssaCreationPhase = tracing::SSACreationPhase();
+				trace = ssaCreationPhase.apply(trace);
+				auto backend = registry.getBackend("mlir");
+				auto irConversionPhase = tracing::TraceToIRConversionPhase();
+				auto ir = irConversionPhase.apply(trace);
+				auto op = engine::Options();
+				// force compilation for the MLIR backend.
+				op.setOption("mlir.eager_compilation", true);
+				auto dh = compiler::DumpHandler(op, "");
+				meter.measure([&] { return backend->compile(ir, dh, op); });
+			});
+		}
+	}
+}
+
 } // namespace nautilus::engine

nautilus/test/execution-tests/CMakeLists.txt

@@ -6,7 +6,9 @@ add_executable(nautilus-execution-tests
 )
 
 target_link_libraries(nautilus-execution-tests PUBLIC nautilus Catch2::Catch2WithMain)
-
+target_include_directories(nautilus-execution-tests PRIVATE
+        $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/../common>
+        $<INSTALL_INTERFACE:src/nautilus/>)
 
 list(APPEND CMAKE_MODULE_PATH ${catch2_SOURCE_DIR}/extras)
 
@@ -18,6 +20,10 @@ if (ENABLE_TRACING AND (CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang"))
             TracingTest.cpp
     )
 
+    target_include_directories(nautilus-tracing-tests PRIVATE
+            $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/../common>
+            $<INSTALL_INTERFACE:src/nautilus/>)
+
     target_include_directories(nautilus-tracing-tests PRIVATE
             $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/../../src>
             $<INSTALL_INTERFACE:src/nautilus/>)
@@ -27,17 +33,4 @@ if (ENABLE_TRACING AND (CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang"))
     catch_discover_tests(nautilus-tracing-tests EXTRA_ARGS --allow-running-no-tests)
 endif ()
 
-add_subdirectory(std)
-
-if (ENABLE_TRACING)
-    add_executable(nautilus-tracing-benchmarks
-            TracingBenchmark.cpp
-    )
-    target_include_directories(nautilus-tracing-benchmarks PRIVATE
-            $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/../../src>
-            $<INSTALL_INTERFACE:src/nautilus/>)
-
-    target_link_libraries(nautilus-tracing-benchmarks PUBLIC nautilus Catch2::Catch2WithMain)
-    list(APPEND CMAKE_MODULE_PATH ${catch2_SOURCE_DIR}/extras)
-    catch_discover_tests(nautilus-tracing-benchmarks EXTRA_ARGS --allow-running-no-tests)
-endif ()
+add_subdirectory(std)