From d5af956d4a0abd0914d4156e0b71794b9955e0fa Mon Sep 17 00:00:00 2001 From: Sergey Bronnikov Date: Tue, 25 Oct 2022 22:21:12 +0300 Subject: [PATCH] cmake: replace prove with CTest The patch replaces the main test runner prove(1) with CTest [1]. The benefits are: - less external dependencies, prove(1) is gone - running tests by test title - extended test running options in comparison to prove(1) - unified test output (finally!) Note, it is not possible to attach targets to the automatically generated `test` target. It means that target `test` now is executing `LuaJIT-test`, but not `LuaJIT-lint`. Note, the testsuites in `test/LuaJIT-tests` and in `test/PUC-Rio-Lua-5.1` directories have their own test runners and currently CTest runs these testsuites as a single tests. In a future we could change these test runners to specify test from outside and after this we could run tests separately by CTest in these test suites. Note, it is not possible to add dependencies to `add_test()` in CMake, see [2]. It means that all object files and executables must be build before running ctest(1). Users in SO thread [3] recommends wraps ctest(1) in CMake by another target and use it for running or use `FIXTURES_SETUP` or `FIXTURES_REQUIRED` when CMake version >= 3.7. As a workaround one must call target `all` before running `ctest`. User-visible changes are: $ cmake -S . -B build $ cmake --build build --target test --parallel Using CTest in a build directory: $ ctest # Running all tests. $ ctest -L tarantool # Running tests in tarantool-tests dir. $ ctest -L tarantool-c # Running tests in tarantool-c-tests dir. $ ctest -L lua-Harness # Running tests in lua-Harness dir. $ ctest -L luajit # Running tests in LuaJIT dir. $ ctest -L lua # Running tests in PUC-Rio-Lua-5.1 dir. $ ctest --rerun-fail $ ctest -V $ ctest --output-on-failure 1. https://cmake.org/cmake/help/latest/manual/ctest.1.html 2. https://gitlab.kitware.com/cmake/cmake/-/issues/8774 3. https://stackoverflow.com/questions/733475/cmake-ctest-make-test-doesnt-build-tests --- CMakeLists.txt | 1 + test/CMakeLists.txt | 24 +++-------- test/LuaJIT-tests/CMakeLists.txt | 18 ++++++-- test/PUC-Rio-Lua-5.1-tests/CMakeLists.txt | 24 +++++++---- test/lua-Harness-tests/CMakeLists.txt | 43 ++++++++++--------- test/tarantool-c-tests/CMakeLists.txt | 50 +++++++++++----------- test/tarantool-tests/CMakeLists.txt | 52 +++++++++-------------- 7 files changed, 102 insertions(+), 110 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index eebf3d6f62..1943d894e3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -356,6 +356,7 @@ set(LUAJIT_TEST_INIT "${PROJECT_SOURCE_DIR}/test/luajit-test-init.lua" CACHE STR "Lua code need to be run before tests are started." ) +enable_testing() add_subdirectory(test) # --- Misc rules --------------------------------------------------------------- diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 58cba5ba6e..b6c018a67a 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -73,6 +73,11 @@ add_custom_target(${PROJECT_NAME}-lint DEPENDS set(LUAJIT_TEST_COMMAND "${LUAJIT_TEST_BINARY} -e dofile[[${LUAJIT_TEST_INIT}]]") separate_arguments(LUAJIT_TEST_COMMAND) +set(TEST_FLAGS --schedule-random --parallel ${CMAKE_BUILD_PARALLEL_LEVEL}) +if(CMAKE_VERBOSE_MAKEFILE) + list(APPEND TEST_FLAGS --verbose --output-on-failure) +endif() + add_subdirectory(LuaJIT-tests) add_subdirectory(PUC-Rio-Lua-5.1-tests) add_subdirectory(lua-Harness-tests) @@ -86,22 +91,3 @@ add_custom_target(${PROJECT_NAME}-test DEPENDS tarantool-c-tests tarantool-tests ) - -if(LUAJIT_USE_TEST) - if(POLICY CMP0037) - if(CMAKE_VERSION VERSION_LESS 3.11) - # CMake below 3.11 reserves the name 'test'. Use old policy. - # https://cmake.org/cmake/help/v3.11/release/3.11.html#other-changes - cmake_policy(SET CMP0037 OLD) - else() - # Starting from CMake 3.11 the name 'test' is reserved in - # special cases and can be used as a target name. - cmake_policy(SET CMP0037 NEW) - endif() - endif(POLICY CMP0037) - - add_custom_target(test DEPENDS - ${PROJECT_NAME}-test - ${PROJECT_NAME}-lint - ) -endif() diff --git a/test/LuaJIT-tests/CMakeLists.txt b/test/LuaJIT-tests/CMakeLists.txt index 9cd76ee9fd..b2353eb8ac 100644 --- a/test/LuaJIT-tests/CMakeLists.txt +++ b/test/LuaJIT-tests/CMakeLists.txt @@ -1,12 +1,22 @@ # See the rationale in the root CMakeLists.txt cmake_minimum_required(VERSION 3.1 FATAL_ERROR) -add_custom_target(LuaJIT-tests DEPENDS ${LUAJIT_TEST_BINARY}) - -add_custom_command(TARGET LuaJIT-tests - COMMENT "Running LuaJIT-tests" +# The test suite has its own test runner +# (test/LuaJIT-tests/test.lua), it is not possible +# to run these tests separately by CTest. +message(STATUS "Add test test/LuaJIT-tests") +add_test(NAME "test/LuaJIT-tests" COMMAND ${LUAJIT_TEST_COMMAND} ${CMAKE_CURRENT_SOURCE_DIR}/test.lua +slow +ffi +bit +jit WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} ) +set_tests_properties("test/LuaJIT-tests" PROPERTIES + LABELS luajit + TIMEOUT 10 +) + +add_custom_target(LuaJIT-tests + COMMAND ctest -L luajit ${TEST_FLAGS} + DEPENDS luajit-main +) diff --git a/test/PUC-Rio-Lua-5.1-tests/CMakeLists.txt b/test/PUC-Rio-Lua-5.1-tests/CMakeLists.txt index 98277f9ad4..7288a64bcc 100644 --- a/test/PUC-Rio-Lua-5.1-tests/CMakeLists.txt +++ b/test/PUC-Rio-Lua-5.1-tests/CMakeLists.txt @@ -34,17 +34,23 @@ add_subdirectory(libs) # But, unfortunately, depends on specific PUC-Rio # Lua 5.1 internal headers and should be adapted for LuaJIT. -add_custom_target(PUC-Rio-Lua-5.1-tests - DEPENDS ${LUAJIT_TEST_BINARY} PUC-Rio-Lua-5.1-tests-prepare +# The test suite has its own test runner +# (test/PUC-Rio-Lua-5.1-tests/all.lua), it is not possible +# to run these tests separately by CTest. +message(STATUS "Add test test/PUC-Rio-Lua-5.1") +add_test(NAME "test/PUC-Rio-Lua-5.1" + COMMAND ${LUAJIT_TEST_COMMAND} ${CMAKE_CURRENT_SOURCE_DIR}/all.lua + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} +) +set_tests_properties("test/PUC-Rio-Lua-5.1" PROPERTIES + ENVIRONMENT "LUA_PATH=${LUA_PATH}\;\;" + LABELS lua + TIMEOUT 10 ) -add_custom_command(TARGET PUC-Rio-Lua-5.1-tests - COMMENT "Running PUC-Rio Lua 5.1 tests" - COMMAND - env - LUA_PATH="${LUA_PATH}" - ${LUAJIT_TEST_COMMAND} ${CMAKE_CURRENT_SOURCE_DIR}/all.lua - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} +add_custom_target(PUC-Rio-Lua-5.1-tests + COMMAND ctest -L lua ${TEST_FLAGS} + DEPENDS luajit-main PUC-Rio-Lua-5.1-tests-prepare ) # vim: expandtab tabstop=2 shiftwidth=2 diff --git a/test/lua-Harness-tests/CMakeLists.txt b/test/lua-Harness-tests/CMakeLists.txt index f748a8fd32..9cd2319e98 100644 --- a/test/lua-Harness-tests/CMakeLists.txt +++ b/test/lua-Harness-tests/CMakeLists.txt @@ -4,12 +4,6 @@ # See the rationale in the root CMakeLists.txt cmake_minimum_required(VERSION 3.1 FATAL_ERROR) -find_program(PROVE prove) -if(NOT PROVE) - message(WARNING "`prove' is not found, so lua-Harness-tests target is not generated") - return() -endif() - # Tests create temporary files (see 303-package.t and 411-luajit.t # for example) to be required. Also, they require some files from # the original test source directory. @@ -20,23 +14,30 @@ make_lua_path(LUA_PATH ${LUAJIT_SOURCE_DIR}/?.lua ${LUAJIT_BINARY_DIR}/?.lua ) -set(LUA_TEST_FLAGS --failures --shuffle) -if(CMAKE_VERBOSE_MAKEFILE) - list(APPEND LUA_TEST_FLAGS --verbose) -endif() +set(SUITE_NAME "lua-Harness-tests") +file(GLOB tests ${CMAKE_CURRENT_SOURCE_DIR} "*.t") +foreach(test_path ${tests}) + get_filename_component(test_name ${test_path} NAME) + if (${test_name} STREQUAL ${SUITE_NAME}) + continue() + endif (${test_name} STREQUAL ${SUITE_NAME}) + set(test_title "test/${SUITE_NAME}/${test_name}") + message(STATUS "Add test ${test_title}") + add_test(NAME ${test_title} + COMMAND ${LUAJIT_TEST_COMMAND} -l profile_luajit21 ${test_path} + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + ) + set_tests_properties(${test_title} PROPERTIES + ENVIRONMENT "LUA_PATH=${LUA_PATH}\;" + LABELS lua-Harness + TIMEOUT 5 + ) +endforeach() -add_custom_target(lua-Harness-tests DEPENDS ${LUAJIT_TEST_BINARY}) -add_custom_command(TARGET lua-Harness-tests - COMMENT "Running lua-Harness tests" - COMMAND - env - LUA_PATH="${LUA_PATH}" - ${PROVE} ${CMAKE_CURRENT_SOURCE_DIR} - --exec '${LUAJIT_TEST_COMMAND} -l profile_luajit21' - --jobs ${CMAKE_BUILD_PARALLEL_LEVEL} - ${LUA_TEST_FLAGS} - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} +add_custom_target(lua-Harness-tests + COMMAND ctest -L lua-Harness ${TEST_FLAGS} + DEPENDS luajit-main ) # vim: expandtab tabstop=2 shiftwidth=2 diff --git a/test/tarantool-c-tests/CMakeLists.txt b/test/tarantool-c-tests/CMakeLists.txt index 172553451b..b51ea35aa2 100644 --- a/test/tarantool-c-tests/CMakeLists.txt +++ b/test/tarantool-c-tests/CMakeLists.txt @@ -1,15 +1,4 @@ -find_program(PROVE prove) -if(NOT PROVE) - message(WARNING "`prove' is not found, so tarantool-c-tests target is not generated") - return() -endif() - set(C_TEST_SUFFIX .c_test) -set(C_TEST_FLAGS --failures --shuffle) - -if(CMAKE_VERBOSE_MAKEFILE) - list(APPEND C_TEST_FLAGS --verbose) -endif() # Build libtest. @@ -49,21 +38,30 @@ foreach(test_source ${tests}) LIST(APPEND TESTS_COMPILED ${exe}) endforeach() +# Note, we cannot globbing generated files in CMake, see +# https://stackoverflow.com/questions/44076307/cmake-globbing-generated-files +# To workaround this we globbing source files and generated tests +# with path to executable binaries. +file(GLOB tests ${CMAKE_CURRENT_SOURCE_DIR} "*.test.c") +set(SUITE_NAME "tarantool-c-tests") +foreach(test_path ${tests}) + get_filename_component(test_name ${test_path} NAME_WE) + if (${test_name} STREQUAL ${SUITE_NAME}) + continue() + endif (${test_name} STREQUAL ${SUITE_NAME}) + set(test_title "test/${SUITE_NAME}/${test_name}${C_TEST_SUFFIX}") + message(STATUS "Add test ${test_title}") + add_test(NAME ${test_title} + COMMAND ${CMAKE_CURRENT_BINARY_DIR}/${test_name}${C_TEST_SUFFIX} + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + ) + set_tests_properties(${test_title} PROPERTIES + LABELS tarantool-c + TIMEOUT 5 + ) +endforeach() + add_custom_target(tarantool-c-tests + COMMAND ctest -L tarantool-c ${TEST_FLAGS} DEPENDS libluajit libtest ${TESTS_COMPILED} ) - -add_custom_command(TARGET tarantool-c-tests - COMMENT "Running Tarantool C tests" - COMMAND - ${PROVE} - ${CMAKE_CURRENT_BINARY_DIR} - --ext ${C_TEST_SUFFIX} - --jobs ${CMAKE_BUILD_PARALLEL_LEVEL} - # Report any TAP parse errors, if any, since test module is - # maintained by us. - --parse - ${C_TEST_FLAGS} - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} -) - diff --git a/test/tarantool-tests/CMakeLists.txt b/test/tarantool-tests/CMakeLists.txt index c15d60376d..0babf243c9 100644 --- a/test/tarantool-tests/CMakeLists.txt +++ b/test/tarantool-tests/CMakeLists.txt @@ -4,12 +4,6 @@ # See the rationale in the root CMakeLists.txt. cmake_minimum_required(VERSION 3.1 FATAL_ERROR) -find_program(PROVE prove) -if(NOT PROVE) - message(WARNING "`prove' is not found, so tarantool-tests target is not generated") - return() -endif() - macro(BuildTestCLib lib sources) add_library(${lib} SHARED EXCLUDE_FROM_ALL ${sources}) target_include_directories(${lib} PRIVATE @@ -83,22 +77,19 @@ make_lua_path(LUA_PATH ${PROJECT_SOURCE_DIR}/tools/?.lua ${LUAJIT_SOURCE_DIR}/?.lua ${LUAJIT_BINARY_DIR}/?.lua + ${PROJECT_BINARY_DIR}/src/?.lua ) + # Update LUA_CPATH with the library paths collected within # macro. make_lua_path(LUA_CPATH PATHS ${LUA_CPATHS}) set(LUA_TEST_SUFFIX .test.lua) -set(LUA_TEST_FLAGS --failures --shuffle) set(LUA_TEST_ENV "LUA_PATH=\"${LUA_PATH}\"" "LUA_CPATH=\"${LUA_CPATH}\"" ) -if(CMAKE_VERBOSE_MAKEFILE) - list(APPEND LUA_TEST_FLAGS --verbose) -endif() - # XXX: Since the auxiliary libraries are built as a dynamically # loaded modules on MacOS instead of shared libraries as it is # done on Linux and BSD, another environment variable should be @@ -133,25 +124,24 @@ else() list(APPEND LUA_TEST_ENV_MORE LD_LIBRARY_PATH=${LD_LIBRARY_PATH}) endif() -# LUA_CPATH and LD_LIBRARY_PATH variables and also TESTLIBS list -# with dependecies are set in scope of BuildTestLib macro. +file(GLOB_RECURSE tests ${CMAKE_CURRENT_SOURCE_DIR} "*${LUA_TEST_SUFFIX}") +foreach(test_path ${tests}) + get_filename_component(test_name ${test_path} NAME) + set(test_title "test/tarantool-tests/${test_name}") + message(STATUS "Add test ${test_title}") + add_test(NAME ${test_title} + COMMAND ${LUAJIT_TEST_COMMAND} ${test_path} + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + ) + set_tests_properties(${test_title} PROPERTIES + ENVIRONMENT "LUA_PATH=${LUA_PATH};\;\;;LUA_CPATH=${LUA_CPATH}\;\;;LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" + DEPENDS "luajit-main;${LUAJIT_BIN};${TESTLIBS};${LUAJIT_TEST_BINARY}" + LABELS tarantool + TIMEOUT 30 + ) +endforeach() + add_custom_target(tarantool-tests - DEPENDS ${LUAJIT_TEST_BINARY} ${TESTLIBS} + COMMAND ctest -L tarantool ${TEST_FLAGS} + DEPENDS luajit-main ${TESTLIBS} ) -add_custom_command(TARGET tarantool-tests - COMMENT "Running Tarantool tests" - COMMAND - # XXX: We can't move everything to the "inner" env, since there - # are some issues with escaping ';' for different shells. As - # a result LUA_PATH/LUA_CPATH variables are set via the "outer" - # env, since they are not stripped by SIP like LD_*/DYLD_* are. - env - ${LUA_TEST_ENV} - ${PROVE} ${CMAKE_CURRENT_SOURCE_DIR} - --exec 'env ${LUA_TEST_ENV_MORE} ${LUAJIT_TEST_COMMAND}' - --ext ${LUA_TEST_SUFFIX} - --jobs ${CMAKE_BUILD_PARALLEL_LEVEL} - ${LUA_TEST_FLAGS} - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} -) -