X-Git-Url: https://git.m6w6.name/?a=blobdiff_plain;f=libtest%2Fmain.cc;h=b93f75708c1f7b12e6d45c430e0201af70420bee;hb=74d60e0cbacf53e996e9a72b644e30ed7efa3835;hp=ef10028c5800160dabb687448ec433b0bc0ba321;hpb=9069819ac2940397e96210867e0666b801b97f60;p=awesomized%2Flibmemcached diff --git a/libtest/main.cc b/libtest/main.cc index ef10028c..b93f7570 100644 --- a/libtest/main.cc +++ b/libtest/main.cc @@ -1,38 +1,58 @@ /* vim:expandtab:shiftwidth=2:tabstop=2:smarttab: - * - * libtest * - * Copyright (C) 2011 Data Differential, http://datadifferential.com/ + * Data Differential YATL (i.e. libtest) library * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 3 of the License, or (at your option) any later version. + * Copyright (C) 2012-2013 Data Differential, http://datadifferential.com/ * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * + * * The names of its contributors may not be used to endorse or + * promote products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA */ -#include +#include "libtest/yatlcon.h" #include #include #include #include +#include +#include +#include +#ifdef HAVE_STRINGS_H +# include +#endif +#include +#include +#include #include #include -#include #include #include -#include -#include -#include #include @@ -40,35 +60,31 @@ #pragma GCC diagnostic ignored "-Wold-style-cast" #endif +#if __cplusplus >= 201103L +# define UNIQUE_PTR std::unique_ptr +#else +# define UNIQUE_PTR std::auto_ptr +#endif + using namespace libtest; -static void stats_print(Stats *stats) +static void stats_print(libtest::Framework *frame) { - if (stats->collection_failed == 0 and stats->collection_success == 0) + if (frame->failed() == 0 and frame->success() == 0) { return; } - Out << "\tTotal Collections\t\t\t\t" << stats->collection_total; - Out << "\tFailed Collections\t\t\t\t" << stats->collection_failed; - Out << "\tSkipped Collections\t\t\t\t" << stats->collection_skipped; - Out << "\tSucceeded Collections\t\t\t\t" << stats->collection_success; Outn(); - Out << "Total\t\t\t\t" << stats->total; - Out << "\tFailed\t\t\t" << stats->failed; - Out << "\tSkipped\t\t\t" << stats->skipped; - Out << "\tSucceeded\t\t" << stats->success; -} - -static long int timedif(struct timeval a, struct timeval b) -{ - long us, s; - - us = (long)(a.tv_usec - b.tv_usec); - us /= 1000; - s = (long)(a.tv_sec - b.tv_sec); - s *= 1000; - return s + us; + Out << "Collections\t\t\t\t\t" << frame->total(); + Out << "\tFailed\t\t\t\t\t" << frame->failed(); + Out << "\tSkipped\t\t\t\t\t" << frame->skipped(); + Out << "\tSucceeded\t\t\t\t" << frame->success(); + Outn(); + Out << "Tests\t\t\t\t\t" << frame->sum_total(); + Out << "\tFailed\t\t\t\t" << frame->sum_failed(); + Out << "\tSkipped\t\t\t\t" << frame->sum_skipped(); + Out << "\tSucceeded\t\t\t" << frame->sum_success(); } #include @@ -80,6 +96,35 @@ int main(int argc, char *argv[]) unsigned long int opt_repeat= 1; // Run all tests once bool opt_quiet= false; std::string collection_to_run; + std::string wildcard; + std::string binary_name; + + const char *just_filename= rindex(argv[0], '/'); + if (just_filename) + { + just_filename++; + } + else + { + just_filename= argv[0]; + } + + if (just_filename[0] == 'l' and just_filename[1] == 't' and just_filename[2] == '-') + { + just_filename+= 3; + } + binary_name.append(just_filename); + + /* + Valgrind does not currently work reliably, or sometimes at all, on OSX + - Fri Jun 15 11:24:07 EDT 2012 + */ +#if defined(__APPLE__) && __APPLE__ + if (valgrind_is_caller()) + { + return EXIT_SKIP; + } +#endif // Options parsing { @@ -88,6 +133,7 @@ int main(int argc, char *argv[]) OPT_LIBYATL_MATCH_COLLECTION, OPT_LIBYATL_MASSIVE, OPT_LIBYATL_QUIET, + OPT_LIBYATL_MATCH_WILDCARD, OPT_LIBYATL_REPEAT }; @@ -95,8 +141,9 @@ int main(int argc, char *argv[]) { { "version", no_argument, NULL, OPT_LIBYATL_VERSION }, { "quiet", no_argument, NULL, OPT_LIBYATL_QUIET }, - { "repeat", no_argument, NULL, OPT_LIBYATL_REPEAT }, + { "repeat", required_argument, NULL, OPT_LIBYATL_REPEAT }, { "collection", required_argument, NULL, OPT_LIBYATL_MATCH_COLLECTION }, + { "wildcard", required_argument, NULL, OPT_LIBYATL_MATCH_WILDCARD }, { "massive", no_argument, NULL, OPT_LIBYATL_MASSIVE }, { 0, 0, 0, 0 } }; @@ -120,13 +167,23 @@ int main(int argc, char *argv[]) break; case OPT_LIBYATL_REPEAT: + errno= 0; opt_repeat= strtoul(optarg, (char **) NULL, 10); + if (errno != 0) + { + Error << "unknown value passed to --repeat: `" << optarg << "`"; + exit(EXIT_FAILURE); + } break; case OPT_LIBYATL_MATCH_COLLECTION: collection_to_run= optarg; break; + case OPT_LIBYATL_MATCH_WILDCARD: + wildcard= optarg; + break; + case OPT_LIBYATL_MASSIVE: opt_massive= true; break; @@ -144,9 +201,16 @@ int main(int argc, char *argv[]) srandom((unsigned int)time(NULL)); - if (bool(getenv("YATL_REPEAT")) and (strtoul(getenv("YATL_REPEAT"), (char **) NULL, 10) > 1)) + errno= 0; + if (bool(getenv("YATL_REPEAT"))) { + errno= 0; opt_repeat= strtoul(getenv("YATL_REPEAT"), (char **) NULL, 10); + if (errno != 0) + { + Error << "ENV YATL_REPEAT passed an invalid value: `" << getenv("YATL_REPEAT") << "`"; + exit(EXIT_FAILURE); + } } if ((bool(getenv("YATL_QUIET")) and (strcmp(getenv("YATL_QUIET"), "0") == 0)) or opt_quiet) @@ -163,27 +227,39 @@ int main(int argc, char *argv[]) } } + if ((bool(getenv("YATL_RUN_MASSIVE_TESTS"))) or opt_massive) + { + opt_massive= true; + } + if (opt_quiet) { close(STDOUT_FILENO); } - char buffer[1024]; + if (opt_massive) + { + is_massive(opt_massive); + } + + libtest::vchar_t tmp_directory; + tmp_directory.resize(1024); if (getenv("LIBTEST_TMP")) { - snprintf(buffer, sizeof(buffer), "%s", getenv("LIBTEST_TMP")); + snprintf(&tmp_directory[0], tmp_directory.size(), "%s", getenv("LIBTEST_TMP")); } else { - snprintf(buffer, sizeof(buffer), "%s", LIBTEST_TEMP); + snprintf(&tmp_directory[0], tmp_directory.size(), "%s", LIBTEST_TEMP); } - if (chdir(buffer) == -1) + if (chdir(&tmp_directory[0]) == -1) { - char getcwd_buffer[1024]; - char *dir= getcwd(getcwd_buffer, sizeof(getcwd_buffer)); + libtest::vchar_t getcwd_buffer; + getcwd_buffer.resize(1024); + char *dir= getcwd(&getcwd_buffer[0], getcwd_buffer.size()); - Error << "Unable to chdir() from " << dir << " to " << buffer << " errno:" << strerror(errno); + Error << "Unable to chdir() from " << dir << " to " << &tmp_directory[0] << " errno:" << strerror(errno); return EXIT_FAILURE; } @@ -193,13 +269,31 @@ int main(int argc, char *argv[]) return EXIT_FAILURE; } + if (getenv("YATL_COLLECTION_TO_RUN")) + { + if (strlen(getenv("YATL_COLLECTION_TO_RUN"))) + { + collection_to_run= getenv("YATL_COLLECTION_TO_RUN"); + } + } + + if (collection_to_run.compare("none") == 0) + { + return EXIT_SUCCESS; + } + + if (collection_to_run.empty() == false) + { + Out << "Only testing " << collection_to_run; + } + int exit_code; - try { - do { + try + { + do + { exit_code= EXIT_SUCCESS; - Framework world; - fatal_assert(sigignore(SIGPIPE) == 0); libtest::SignalThread signal; @@ -209,230 +303,28 @@ int main(int argc, char *argv[]) return EXIT_FAILURE; } - Stats stats; - - get_world(&world); - - test_return_t error; - void *creators_ptr= world.create(error); + UNIQUE_PTR frame(new libtest::Framework(signal, binary_name, collection_to_run, wildcard)); - switch (error) + // Run create(), bail on error. { - case TEST_SUCCESS: - break; - - case TEST_SKIPPED: - Out << "SKIP " << argv[0]; - return EXIT_SUCCESS; - - case TEST_FAILURE: - return EXIT_FAILURE; - } - - if (getenv("YATL_COLLECTION_TO_RUN")) - { - if (strlen(getenv("YATL_COLLECTION_TO_RUN"))) - { - collection_to_run= getenv("YATL_COLLECTION_TO_RUN"); - } - } - - if (collection_to_run.compare("none") == 0) - { - return EXIT_SUCCESS; - } - - if (collection_to_run.empty() == false) - { - Out << "Only testing " << collection_to_run; - } - - char *wildcard= NULL; - if (argc == 3) - { - wildcard= argv[2]; - } - - for (collection_st *next= world.collections; next and next->name and (not signal.is_shutdown()); next++) - { - bool failed= false; - bool skipped= false; - - if (collection_to_run.empty() == false and fnmatch(collection_to_run.c_str(), next->name, 0)) - { - continue; - } - - stats.collection_total++; - - test_return_t collection_rc= world.startup(creators_ptr); - - if (collection_rc == TEST_SUCCESS and next->pre) - { - collection_rc= world.runner()->pre(next->pre, creators_ptr); - } - - switch (collection_rc) + switch (frame->create()) { case TEST_SUCCESS: break; - case TEST_FAILURE: - Out << next->name << " [ failed ]"; - failed= true; - signal.set_shutdown(SHUTDOWN_GRACEFUL); - goto cleanup; - case TEST_SKIPPED: - Out << next->name << " [ skipping ]"; - skipped= true; - goto cleanup; - - default: - fatal_message("invalid return code"); - } - - Out << "Collection: " << next->name; - - for (test_st *run= next->tests; run->name; run++) - { - struct timeval start_time, end_time; - long int load_time= 0; - - if (wildcard && fnmatch(wildcard, run->name, 0)) - { - continue; - } - - test_return_t return_code; - try { - if (test_success(return_code= world.item.startup(creators_ptr))) - { - if (test_success(return_code= world.item.flush(creators_ptr, run))) - { - // @note pre will fail is SKIPPED is returned - if (test_success(return_code= world.item.pre(creators_ptr))) - { - { // Runner Code - gettimeofday(&start_time, NULL); - assert(world.runner()); - assert(run->test_fn); - try - { - return_code= world.runner()->run(run->test_fn, creators_ptr); - } - // Special case where check for the testing of the exception - // system. - catch (libtest::fatal &e) - { - if (fatal::is_disabled()) - { - fatal::increment_disabled_counter(); - return_code= TEST_SUCCESS; - } - else - { - throw; - } - } - - gettimeofday(&end_time, NULL); - load_time= timedif(end_time, start_time); - } - } - - // @todo do something if post fails - (void)world.item.post(creators_ptr); - } - else if (return_code == TEST_SKIPPED) - { } - else if (return_code == TEST_FAILURE) - { - Error << " item.flush(failure)"; - signal.set_shutdown(SHUTDOWN_GRACEFUL); - } - } - else if (return_code == TEST_SKIPPED) - { } - else if (return_code == TEST_FAILURE) - { - Error << " item.startup(failure)"; - signal.set_shutdown(SHUTDOWN_GRACEFUL); - } - } - - catch (libtest::fatal &e) - { - Error << "Fatal exception was thrown: " << e.what(); - return_code= TEST_FAILURE; - } - catch (std::exception &e) - { - Error << "Exception was thrown: " << e.what(); - return_code= TEST_FAILURE; - } - catch (...) - { - Error << "Unknown exception occurred"; - return_code= TEST_FAILURE; - } - - stats.total++; - - switch (return_code) - { - case TEST_SUCCESS: - Out << "\tTesting " << run->name << "\t\t\t\t\t" << load_time / 1000 << "." << load_time % 1000 << "[ " << test_strerror(return_code) << " ]"; - stats.success++; - break; - - case TEST_FAILURE: - stats.failed++; - failed= true; - Out << "\tTesting " << run->name << "\t\t\t\t\t" << "[ " << test_strerror(return_code) << " ]"; - break; - - case TEST_SKIPPED: - stats.skipped++; - skipped= true; - Out << "\tTesting " << run->name << "\t\t\t\t\t" << "[ " << test_strerror(return_code) << " ]"; - break; - - default: - fatal_message("invalid return code"); - } - - if (test_failed(world.on_error(return_code, creators_ptr))) - { - Error << "Failed while running on_error()"; - signal.set_shutdown(SHUTDOWN_GRACEFUL); - break; - } - } - - (void) world.runner()->post(next->post, creators_ptr); - -cleanup: - if (failed == false and skipped == false) - { - stats.collection_success++; - } - - if (failed) - { - stats.collection_failed++; - } + SKIP("SKIP was returned from framework create()"); + break; - if (skipped) - { - stats.collection_skipped++; + case TEST_FAILURE: + std::cerr << "Could not call frame->create()" << std::endl; + return EXIT_FAILURE; } - - world.shutdown(creators_ptr); - Outn(); } - if (not signal.is_shutdown()) + frame->exec(); + + if (signal.is_shutdown() == false) { signal.set_shutdown(SHUTDOWN_GRACEFUL); } @@ -443,36 +335,67 @@ cleanup: Out << "Tests were aborted."; exit_code= EXIT_FAILURE; } - else if (stats.collection_failed) + else if (frame->failed()) { Out << "Some test failed."; exit_code= EXIT_FAILURE; } - else if (stats.collection_skipped and stats.collection_failed and stats.collection_success) + else if (frame->skipped() and frame->failed() and frame->success()) { Out << "Some tests were skipped."; } - else if (stats.collection_success and stats.collection_failed == 0) + else if (frame->success() and (frame->failed() == 0)) { + Out; Out << "All tests completed successfully."; } - stats_print(&stats); + stats_print(frame.get()); + + std::ofstream xml_file; + std::string file_name; + file_name.append(&tmp_directory[0]); + file_name.append(frame->name()); + file_name.append(".xml"); + xml_file.open(file_name.c_str(), std::ios::trunc); + libtest::Formatter::xml(*frame, xml_file); Outn(); // Generate a blank to break up the messages if make check/test has been run } while (exit_code == EXIT_SUCCESS and --opt_repeat); } - catch (libtest::fatal& e) + catch (const libtest::__skipped& e) + { + return EXIT_SKIP; + } + catch (const libtest::__failure& e) + { + libtest::stream::make_cout(e.file(), e.line(), e.func()) << e.what(); + exit_code= EXIT_FAILURE; + } + catch (const libtest::fatal& e) + { + std::cerr << "FATAL:" << e.what() << std::endl; + exit_code= EXIT_FAILURE; + } + catch (const libtest::disconnected& e) + { + std::cerr << "Unhandled disconnection occurred:" << e.what() << std::endl; + exit_code= EXIT_FAILURE; + } + catch (const std::exception& e) { - std::cerr << e.what() << std::endl; + std::cerr << "std::exception:" << e.what() << std::endl; + exit_code= EXIT_FAILURE; } - catch (std::exception& e) + catch (char const* s) { - std::cerr << e.what() << std::endl; + std::cerr << "Exception:" << s << std::endl; + exit_code= EXIT_FAILURE; } catch (...) { std::cerr << "Unknown exception halted execution." << std::endl; + exit_code= EXIT_FAILURE; } return exit_code;