From: Michael Wallner Date: Tue, 15 Sep 2020 09:28:01 +0000 (+0200) Subject: testing: honor MEMCACHED_BINARY; use Catch.cmake X-Git-Tag: 1.1.0-beta1~236^2~66 X-Git-Url: https://git.m6w6.name/?a=commitdiff_plain;h=b92ed473274af2ed545aa981c378fac84e0bb700;p=m6w6%2Flibmemcached testing: honor MEMCACHED_BINARY; use Catch.cmake --- diff --git a/CMake/_Include.cmake b/CMake/_Include.cmake index 3b264670..97c07831 100644 --- a/CMake/_Include.cmake +++ b/CMake/_Include.cmake @@ -1,7 +1,8 @@ -set(THREADS_PREFER_PTHREAD_FLAG ON) # globals -include(CTest) +if(CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME) + include(CTest) +endif() include(GNUInstallDirs) include(CMakePackageConfigHelpers) diff --git a/CMakeConfig.txt b/CMakeConfig.txt index 0d584981..7f611de5 100644 --- a/CMakeConfig.txt +++ b/CMakeConfig.txt @@ -44,6 +44,8 @@ set(ENABLE_SANITIZERS "" if(BUILD_TESTING) set(MEMCACHED_BINARY "/usr/bin/memcached" CACHE FILEPATH "memcached binary") + set(CMAKE_CTEST_ARGUMENTS "--output-on-failure;-j2;--repeat;until-pass:2" + ) # available since CMake 3.17 endif() if(BUILD_DOCS) diff --git a/testing/CMake/Catch.cmake b/testing/CMake/Catch.cmake new file mode 100644 index 00000000..486e3233 --- /dev/null +++ b/testing/CMake/Catch.cmake @@ -0,0 +1,175 @@ +# Distributed under the OSI-approved BSD 3-Clause License. See accompanying +# file Copyright.txt or https://cmake.org/licensing for details. + +#[=======================================================================[.rst: +Catch +----- + +This module defines a function to help use the Catch test framework. + +The :command:`catch_discover_tests` discovers tests by asking the compiled test +executable to enumerate its tests. This does not require CMake to be re-run +when tests change. However, it may not work in a cross-compiling environment, +and setting test properties is less convenient. + +This command is intended to replace use of :command:`add_test` to register +tests, and will create a separate CTest test for each Catch test case. Note +that this is in some cases less efficient, as common set-up and tear-down logic +cannot be shared by multiple test cases executing in the same instance. +However, it provides more fine-grained pass/fail information to CTest, which is +usually considered as more beneficial. By default, the CTest test name is the +same as the Catch name; see also ``TEST_PREFIX`` and ``TEST_SUFFIX``. + +.. command:: catch_discover_tests + + Automatically add tests with CTest by querying the compiled test executable + for available tests:: + + catch_discover_tests(target + [TEST_SPEC arg1...] + [EXTRA_ARGS arg1...] + [WORKING_DIRECTORY dir] + [TEST_PREFIX prefix] + [TEST_SUFFIX suffix] + [PROPERTIES name1 value1...] + [TEST_LIST var] + ) + + ``catch_discover_tests`` sets up a post-build command on the test executable + that generates the list of tests by parsing the output from running the test + with the ``--list-test-names-only`` argument. This ensures that the full + list of tests is obtained. Since test discovery occurs at build time, it is + not necessary to re-run CMake when the list of tests changes. + However, it requires that :prop_tgt:`CROSSCOMPILING_EMULATOR` is properly set + in order to function in a cross-compiling environment. + + Additionally, setting properties on tests is somewhat less convenient, since + the tests are not available at CMake time. Additional test properties may be + assigned to the set of tests as a whole using the ``PROPERTIES`` option. If + more fine-grained test control is needed, custom content may be provided + through an external CTest script using the :prop_dir:`TEST_INCLUDE_FILES` + directory property. The set of discovered tests is made accessible to such a + script via the ``_TESTS`` variable. + + The options are: + + ``target`` + Specifies the Catch executable, which must be a known CMake executable + target. CMake will substitute the location of the built executable when + running the test. + + ``TEST_SPEC arg1...`` + Specifies test cases, wildcarded test cases, tags and tag expressions to + pass to the Catch executable with the ``--list-test-names-only`` argument. + + ``EXTRA_ARGS arg1...`` + Any extra arguments to pass on the command line to each test case. + + ``WORKING_DIRECTORY dir`` + Specifies the directory in which to run the discovered test cases. If this + option is not provided, the current binary directory is used. + + ``TEST_PREFIX prefix`` + Specifies a ``prefix`` to be prepended to the name of each discovered test + case. This can be useful when the same test executable is being used in + multiple calls to ``catch_discover_tests()`` but with different + ``TEST_SPEC`` or ``EXTRA_ARGS``. + + ``TEST_SUFFIX suffix`` + Similar to ``TEST_PREFIX`` except the ``suffix`` is appended to the name of + every discovered test case. Both ``TEST_PREFIX`` and ``TEST_SUFFIX`` may + be specified. + + ``PROPERTIES name1 value1...`` + Specifies additional properties to be set on all tests discovered by this + invocation of ``catch_discover_tests``. + + ``TEST_LIST var`` + Make the list of tests available in the variable ``var``, rather than the + default ``_TESTS``. This can be useful when the same test + executable is being used in multiple calls to ``catch_discover_tests()``. + Note that this variable is only available in CTest. + +#]=======================================================================] + +#------------------------------------------------------------------------------ +function(catch_discover_tests TARGET) + cmake_parse_arguments( + "" + "" + "TEST_PREFIX;TEST_SUFFIX;WORKING_DIRECTORY;TEST_LIST" + "TEST_SPEC;EXTRA_ARGS;PROPERTIES" + ${ARGN} + ) + + if(NOT _WORKING_DIRECTORY) + set(_WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}") + endif() + if(NOT _TEST_LIST) + set(_TEST_LIST ${TARGET}_TESTS) + endif() + + ## Generate a unique name based on the extra arguments + string(SHA1 args_hash "${_TEST_SPEC} ${_EXTRA_ARGS}") + string(SUBSTRING ${args_hash} 0 7 args_hash) + + # Define rule to generate test list for aforementioned test executable + set(ctest_include_file "${CMAKE_CURRENT_BINARY_DIR}/${TARGET}_include-${args_hash}.cmake") + set(ctest_tests_file "${CMAKE_CURRENT_BINARY_DIR}/${TARGET}_tests-${args_hash}.cmake") + get_property(crosscompiling_emulator + TARGET ${TARGET} + PROPERTY CROSSCOMPILING_EMULATOR + ) + add_custom_command( + TARGET ${TARGET} POST_BUILD + BYPRODUCTS "${ctest_tests_file}" + COMMAND "${CMAKE_COMMAND}" + -D "TEST_TARGET=${TARGET}" + -D "TEST_EXECUTABLE=$" + -D "TEST_EXECUTOR=${crosscompiling_emulator}" + -D "TEST_WORKING_DIR=${_WORKING_DIRECTORY}" + -D "TEST_SPEC=${_TEST_SPEC}" + -D "TEST_EXTRA_ARGS=${_EXTRA_ARGS}" + -D "TEST_PROPERTIES=${_PROPERTIES}" + -D "TEST_PREFIX=${_TEST_PREFIX}" + -D "TEST_SUFFIX=${_TEST_SUFFIX}" + -D "TEST_LIST=${_TEST_LIST}" + -D "CTEST_FILE=${ctest_tests_file}" + -P "${_CATCH_DISCOVER_TESTS_SCRIPT}" + VERBATIM + ) + + file(WRITE "${ctest_include_file}" + "if(EXISTS \"${ctest_tests_file}\")\n" + " include(\"${ctest_tests_file}\")\n" + "else()\n" + " add_test(${TARGET}_NOT_BUILT-${args_hash} ${TARGET}_NOT_BUILT-${args_hash})\n" + "endif()\n" + ) + + if(NOT ${CMAKE_VERSION} VERSION_LESS "3.10.0") + # Add discovered tests to directory TEST_INCLUDE_FILES + set_property(DIRECTORY + APPEND PROPERTY TEST_INCLUDE_FILES "${ctest_include_file}" + ) + else() + # Add discovered tests as directory TEST_INCLUDE_FILE if possible + get_property(test_include_file_set DIRECTORY PROPERTY TEST_INCLUDE_FILE SET) + if (NOT ${test_include_file_set}) + set_property(DIRECTORY + PROPERTY TEST_INCLUDE_FILE "${ctest_include_file}" + ) + else() + message(FATAL_ERROR + "Cannot set more than one TEST_INCLUDE_FILE" + ) + endif() + endif() + +endfunction() + +############################################################################### + +set(_CATCH_DISCOVER_TESTS_SCRIPT + ${CMAKE_CURRENT_LIST_DIR}/CatchAddTests.cmake +) diff --git a/testing/CMake/CatchAddTests.cmake b/testing/CMake/CatchAddTests.cmake new file mode 100644 index 00000000..156d55ff --- /dev/null +++ b/testing/CMake/CatchAddTests.cmake @@ -0,0 +1,81 @@ +# Distributed under the OSI-approved BSD 3-Clause License. See accompanying +# file Copyright.txt or https://cmake.org/licensing for details. + +set(prefix "${TEST_PREFIX}") +set(suffix "${TEST_SUFFIX}") +set(spec ${TEST_SPEC}) +set(extra_args ${TEST_EXTRA_ARGS}) +set(properties ${TEST_PROPERTIES}) +set(script) +set(suite) +set(tests) + +function(add_command NAME) + set(_args "") + foreach(_arg ${ARGN}) + if(_arg MATCHES "[^-./:a-zA-Z0-9_]") + set(_args "${_args} [==[${_arg}]==]") # form a bracket_argument + else() + set(_args "${_args} ${_arg}") + endif() + endforeach() + set(script "${script}${NAME}(${_args})\n" PARENT_SCOPE) +endfunction() + +# Run test executable to get list of available tests +if(NOT EXISTS "${TEST_EXECUTABLE}") + message(FATAL_ERROR + "Specified test executable '${TEST_EXECUTABLE}' does not exist" + ) +endif() +execute_process( + COMMAND ${TEST_EXECUTOR} "${TEST_EXECUTABLE}" ${spec} --list-test-names-only + OUTPUT_VARIABLE output + RESULT_VARIABLE result +) +# Catch --list-test-names-only reports the number of tests, so 0 is... surprising +if(${result} EQUAL 0) + message(WARNING + "Test executable '${TEST_EXECUTABLE}' contains no tests!\n" + ) +elseif(${result} LESS 0) + message(FATAL_ERROR + "Error running test executable '${TEST_EXECUTABLE}':\n" + " Result: ${result}\n" + " Output: ${output}\n" + ) +endif() + +string(REPLACE "\n" ";" output "${output}") + +# Parse output +foreach(line ${output}) + set(test ${line}) + # Escape characters in test case names that would be parsed by Catch2 + set(test_name ${test}) + foreach(char , [ ]) + string(REPLACE ${char} "\\${char}" test_name ${test_name}) + endforeach(char) + # ...and add to script + add_command(add_test + "${prefix}${test}${suffix}" + ${TEST_EXECUTOR} + "${TEST_EXECUTABLE}" + "${test_name}" + ${extra_args} + ) + add_command(set_tests_properties + "${prefix}${test}${suffix}" + PROPERTIES + WORKING_DIRECTORY "${TEST_WORKING_DIR}" + ${properties} + ) + list(APPEND tests "${prefix}${test}${suffix}") +endforeach() + +# Create a list of all discovered tests, which users may use to e.g. set +# properties on the tests +add_command(set ${TEST_LIST} ${tests}) + +# Write CTest script +file(WRITE "${CTEST_FILE}" "${script}") diff --git a/testing/CMakeLists.txt b/testing/CMakeLists.txt index b7de98fe..acb63695 100644 --- a/testing/CMakeLists.txt +++ b/testing/CMakeLists.txt @@ -1,7 +1,9 @@ -if(NOT BUILD_TESTING) +if(NOT BUILD_TESTING OR NOT CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME) return() endif() +include(CMake/Catch.cmake) + file(GLOB_RECURSE TESTING_SRC RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} *.cpp) set(TESTING_ROOT ${CMAKE_CURRENT_BINARY_DIR}) set_source_files_properties(main.cpp PROPERTIES SKIP_UNITY_BUILD_INCLUSION ON) @@ -16,36 +18,11 @@ target_include_directories(runtests PRIVATE target_link_libraries(runtests PRIVATE libhashkit libmemcached libmemcachedutil) add_dependencies(runtests ${CLIENTS}) -macro(add_test TEST_CASE) - _add_test(${TEST_CASE} runtests ${TEST_CASE}) -endmacro() - -add_test("lib/Server") -add_test("lib/Cluster") - -add_test("hashkit") - -add_test("memcached_append") -add_test("memcached_basic") -add_test("memcached_callbacks") -add_test("memcached_cas") -add_test("memcached_dump") -add_test("memcached_encoding_key") -add_test("memcached_exist") -add_test("memcached_generate_hash") -add_test("memcached_haldenbrand_nblock_tcp_ndelay") -add_test("memcached_inc_dec") -add_test("memcached_ketama_compat") -add_test("memcached_noreply") -add_test("memcached_prepend") -add_test("memcached_regression_binary_block_add") -add_test("memcached_servers") -add_test("memcached_simple") -add_test("memcached_util") - -add_test("memcat") -add_test("memcp") -add_test("memdump") -add_test("memerror") -add_test("memexist") -add_test("memflush") +catch_discover_tests(runtests + TEST_SPEC "lib*") +catch_discover_tests(runtests + TEST_SPEC "hashkit*") +catch_discover_tests(runtests + TEST_SPEC "memcached*") +catch_discover_tests(runtests + TEST_SPEC "bin/*") diff --git a/testing/conf.h.in b/testing/conf.h.in index e06af919..1059872a 100644 --- a/testing/conf.h.in +++ b/testing/conf.h.in @@ -1,3 +1,4 @@ #pragma once #cmakedefine TESTING_ROOT "@TESTING_ROOT@" +#cmakedefine MEMCACHED_BINARY getenv_else("MEMCACHED_BINARY", "@MEMCACHED_BINARY@") diff --git a/testing/lib/MemcachedCluster.cpp b/testing/lib/MemcachedCluster.cpp index 64736149..efb1a2de 100644 --- a/testing/lib/MemcachedCluster.cpp +++ b/testing/lib/MemcachedCluster.cpp @@ -34,7 +34,7 @@ void MemcachedCluster::flush() { MemcachedCluster::MemcachedCluster() : cluster{Server{ - getenv_else("MEMCACHED_BINARY", "memcached"), + MEMCACHED_BINARY, {random_socket_or_port_arg()} }} { @@ -53,14 +53,14 @@ MemcachedCluster MemcachedCluster::mixed() { MemcachedCluster MemcachedCluster::network() { return MemcachedCluster{Cluster{Server{ - getenv_else("MEMCACHED_BINARY", "memcached"), + MEMCACHED_BINARY, {"-p", random_socket_or_port_string} }}}; } MemcachedCluster MemcachedCluster::socket() { return MemcachedCluster{Cluster{Server{ - getenv_else("MEMCACHED_BINARY", "memcached"), + MEMCACHED_BINARY, {"-s", random_socket_or_port_string} }}}; } diff --git a/testing/tests/bin/memcat.cpp b/testing/tests/bin/memcat.cpp index 9a787f81..ab394eda 100644 --- a/testing/tests/bin/memcat.cpp +++ b/testing/tests/bin/memcat.cpp @@ -6,7 +6,7 @@ using Catch::Matchers::Contains; -TEST_CASE("memcat") { +TEST_CASE("bin/memcat") { Shell sh{string{TESTING_ROOT "/../src/bin"}}; SECTION("no servers provided") { @@ -28,7 +28,7 @@ TEST_CASE("memcat") { } SECTION("with server") { - Server server{"memcached", {"-p", random_port_string}}; + Server server{MEMCACHED_BINARY, {"-p", random_port_string}}; MemcachedPtr memc; LoneReturnMatcher test{*memc}; diff --git a/testing/tests/bin/memcp.cpp b/testing/tests/bin/memcp.cpp index 0ba1af18..2c715817 100644 --- a/testing/tests/bin/memcp.cpp +++ b/testing/tests/bin/memcp.cpp @@ -6,7 +6,7 @@ using Catch::Matchers::Contains; -TEST_CASE("memcp") { +TEST_CASE("bin/memcp") { Shell sh{string{TESTING_ROOT "/../src/bin"}}; SECTION("no servers provided") { @@ -28,7 +28,7 @@ TEST_CASE("memcp") { } SECTION("with server") { - Server server{"memcached", {"-p", random_port_string}}; + Server server{MEMCACHED_BINARY, {"-p", random_port_string}}; MemcachedPtr memc; LoneReturnMatcher test{*memc}; @@ -58,7 +58,7 @@ TEST_CASE("memcp") { SECTION("connection failure") { server.signal(SIGKILL); - server.tryWait(); + server.wait(); Tempfile temp; diff --git a/testing/tests/bin/memdump.cpp b/testing/tests/bin/memdump.cpp index da4ab9cb..0fa084d7 100644 --- a/testing/tests/bin/memdump.cpp +++ b/testing/tests/bin/memdump.cpp @@ -6,7 +6,7 @@ using Catch::Matchers::Contains; -TEST_CASE("memdump") { +TEST_CASE("bin/memdump") { Shell sh{string{TESTING_ROOT "/../src/bin"}}; SECTION("no servers provided") { @@ -28,7 +28,7 @@ TEST_CASE("memdump") { } SECTION("with server") { - Server server{"memcached", {"-p", random_port_string}}; + Server server{MEMCACHED_BINARY, {"-p", random_port_string}}; MemcachedPtr memc; LoneReturnMatcher test{*memc}; @@ -53,7 +53,7 @@ TEST_CASE("memdump") { SECTION("connection failure") { server.signal(SIGKILL); - server.tryWait(); + server.wait(); string output; REQUIRE_FALSE(sh.run(comm + "-v", output)); diff --git a/testing/tests/bin/memerror.cpp b/testing/tests/bin/memerror.cpp index 26bb6cf0..690babe6 100644 --- a/testing/tests/bin/memerror.cpp +++ b/testing/tests/bin/memerror.cpp @@ -6,7 +6,7 @@ using Catch::Matchers::Contains; -TEST_CASE("memerror") { +TEST_CASE("bin/memerror") { Shell sh{string{TESTING_ROOT "/../src/bin"}}; SECTION("--help") { diff --git a/testing/tests/bin/memexist.cpp b/testing/tests/bin/memexist.cpp index d8877909..5be420bb 100644 --- a/testing/tests/bin/memexist.cpp +++ b/testing/tests/bin/memexist.cpp @@ -6,7 +6,7 @@ using Catch::Matchers::Contains; -TEST_CASE("memexist") { +TEST_CASE("bin/memexist") { Shell sh{string{TESTING_ROOT "/../src/bin"}}; SECTION("no servers provided") { @@ -28,7 +28,7 @@ TEST_CASE("memexist") { } SECTION("with server") { - Server server{"memcached", {"-p", random_port_string}}; + Server server{MEMCACHED_BINARY, {"-p", random_port_string}}; MemcachedPtr memc; LoneReturnMatcher test{*memc}; diff --git a/testing/tests/bin/memflush.cpp b/testing/tests/bin/memflush.cpp index e6a759bb..7c2cc8c2 100644 --- a/testing/tests/bin/memflush.cpp +++ b/testing/tests/bin/memflush.cpp @@ -6,7 +6,7 @@ using Catch::Matchers::Contains; -TEST_CASE("memflush") { +TEST_CASE("bin/memflush") { Shell sh{string{TESTING_ROOT "/../src/bin"}}; SECTION("no servers provided") { @@ -28,7 +28,7 @@ TEST_CASE("memflush") { } SECTION("with server") { - Server server{"memcached", {"-p", random_port_string}}; + Server server{MEMCACHED_BINARY, {"-p", random_port_string}}; MemcachedPtr memc; LoneReturnMatcher test{*memc}; @@ -59,7 +59,7 @@ TEST_CASE("memflush") { SECTION("connection failure") { server.signal(SIGKILL); - server.tryWait(); + server.wait(); string output; REQUIRE_FALSE(sh.run(comm, output)); diff --git a/testing/tests/lib.cpp b/testing/tests/lib.cpp index 12e6665a..6dd7269f 100644 --- a/testing/tests/lib.cpp +++ b/testing/tests/lib.cpp @@ -4,7 +4,7 @@ #include "testing/lib/Server.hpp" TEST_CASE("lib/Server") { - Server server{"memcached"}; + Server server{MEMCACHED_BINARY}; SECTION("starts and listens") { @@ -33,7 +33,7 @@ TEST_CASE("lib/Server") { } TEST_CASE("lib/Cluster") { - Cluster cluster{Server{"memcached", { + Cluster cluster{Server{MEMCACHED_BINARY, { random_socket_or_port_arg(), }}};