/* vim:expandtab:shiftwidth=2:tabstop=2:smarttab:
- *
- * libtest
*
- * Copyright (C) 2011 Data Differential, http://datadifferential.com/
+ * Data Differential YATL (i.e. libtest) library
*
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 3 of the License, or (at your option) any later version.
+ * Copyright (C) 2012-2013 Data Differential, http://datadifferential.com/
*
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Lesser General Public License for more details.
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *
+ * * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *
+ * * The names of its contributors may not be used to endorse or
+ * promote products derived from this software without specific prior
+ * written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
-#include <config.h>
+#include "libtest/yatlcon.h"
#include <libtest/common.h>
#include <cassert>
#include <ctime>
#include <fnmatch.h>
#include <iostream>
+#ifdef HAVE_STRINGS_H
+# include <strings.h>
+#endif
+#include <fstream>
#include <memory>
#include <sys/stat.h>
#include <sys/time.h>
#pragma GCC diagnostic ignored "-Wold-style-cast"
#endif
+#if __cplusplus >= 201103L
+# define UNIQUE_PTR std::unique_ptr
+#else
+# define UNIQUE_PTR std::auto_ptr
+#endif
+
using namespace libtest;
-static void stats_print(Stats *stats)
+static void stats_print(libtest::Framework *frame)
{
- if (stats->collection_failed == 0 and stats->collection_success == 0)
+ if (frame->failed() == 0 and frame->success() == 0)
{
return;
}
- Out << "\tTotal Collections\t\t\t\t" << stats->collection_total;
- Out << "\tFailed Collections\t\t\t\t" << stats->collection_failed;
- Out << "\tSkipped Collections\t\t\t\t" << stats->collection_skipped;
- Out << "\tSucceeded Collections\t\t\t\t" << stats->collection_success;
Outn();
- Out << "Total\t\t\t\t" << stats->total;
- Out << "\tFailed\t\t\t" << stats->failed;
- Out << "\tSkipped\t\t\t" << stats->skipped;
- Out << "\tSucceeded\t\t" << stats->success;
-}
-
-static long int timedif(struct timeval a, struct timeval b)
-{
- long us, s;
-
- us = (long)(a.tv_usec - b.tv_usec);
- us /= 1000;
- s = (long)(a.tv_sec - b.tv_sec);
- s *= 1000;
- return s + us;
-}
-
-static test_return_t runner_code(Framework* frame,
- test_st* run,
- void* creators_ptr,
- long int& load_time)
-{ // Runner Code
-
- struct timeval start_time, end_time;
-
- gettimeofday(&start_time, NULL);
- assert(frame->runner());
- assert(run->test_fn);
-
- test_return_t return_code;
- try
- {
- return_code= frame->runner()->run(run->test_fn, creators_ptr);
- }
- // Special case where check for the testing of the exception
- // system.
- catch (libtest::fatal &e)
- {
- if (fatal::is_disabled())
- {
- fatal::increment_disabled_counter();
- return_code= TEST_SUCCESS;
- }
- else
- {
- throw;
- }
- }
-
- gettimeofday(&end_time, NULL);
- load_time= timedif(end_time, start_time);
-
- return return_code;
+ Out << "Collections\t\t\t\t\t" << frame->total();
+ Out << "\tFailed\t\t\t\t\t" << frame->failed();
+ Out << "\tSkipped\t\t\t\t\t" << frame->skipped();
+ Out << "\tSucceeded\t\t\t\t" << frame->success();
+ Outn();
+ Out << "Tests\t\t\t\t\t" << frame->sum_total();
+ Out << "\tFailed\t\t\t\t" << frame->sum_failed();
+ Out << "\tSkipped\t\t\t\t" << frame->sum_skipped();
+ Out << "\tSucceeded\t\t\t" << frame->sum_success();
}
#include <getopt.h>
unsigned long int opt_repeat= 1; // Run all tests once
bool opt_quiet= false;
std::string collection_to_run;
+ std::string wildcard;
+ std::string binary_name;
+
+ const char *just_filename= rindex(argv[0], '/');
+ if (just_filename)
+ {
+ just_filename++;
+ }
+ else
+ {
+ just_filename= argv[0];
+ }
+
+ if (just_filename[0] == 'l' and just_filename[1] == 't' and just_filename[2] == '-')
+ {
+ just_filename+= 3;
+ }
+ binary_name.append(just_filename);
+
+ /*
+ Valgrind does not currently work reliably, or sometimes at all, on OSX
+ - Fri Jun 15 11:24:07 EDT 2012
+ */
+#if defined(__APPLE__) && __APPLE__
+ if (valgrind_is_caller())
+ {
+ return EXIT_SKIP;
+ }
+#endif
// Options parsing
{
OPT_LIBYATL_MATCH_COLLECTION,
OPT_LIBYATL_MASSIVE,
OPT_LIBYATL_QUIET,
+ OPT_LIBYATL_MATCH_WILDCARD,
OPT_LIBYATL_REPEAT
};
{
{ "version", no_argument, NULL, OPT_LIBYATL_VERSION },
{ "quiet", no_argument, NULL, OPT_LIBYATL_QUIET },
- { "repeat", no_argument, NULL, OPT_LIBYATL_REPEAT },
+ { "repeat", required_argument, NULL, OPT_LIBYATL_REPEAT },
{ "collection", required_argument, NULL, OPT_LIBYATL_MATCH_COLLECTION },
+ { "wildcard", required_argument, NULL, OPT_LIBYATL_MATCH_WILDCARD },
{ "massive", no_argument, NULL, OPT_LIBYATL_MASSIVE },
{ 0, 0, 0, 0 }
};
break;
case OPT_LIBYATL_REPEAT:
+ errno= 0;
opt_repeat= strtoul(optarg, (char **) NULL, 10);
+ if (errno != 0)
+ {
+ Error << "unknown value passed to --repeat: `" << optarg << "`";
+ exit(EXIT_FAILURE);
+ }
break;
case OPT_LIBYATL_MATCH_COLLECTION:
collection_to_run= optarg;
break;
+ case OPT_LIBYATL_MATCH_WILDCARD:
+ wildcard= optarg;
+ break;
+
case OPT_LIBYATL_MASSIVE:
opt_massive= true;
break;
srandom((unsigned int)time(NULL));
- if (bool(getenv("YATL_REPEAT")) and (strtoul(getenv("YATL_REPEAT"), (char **) NULL, 10) > 1))
+ errno= 0;
+ if (bool(getenv("YATL_REPEAT")))
{
+ errno= 0;
opt_repeat= strtoul(getenv("YATL_REPEAT"), (char **) NULL, 10);
+ if (errno != 0)
+ {
+ Error << "ENV YATL_REPEAT passed an invalid value: `" << getenv("YATL_REPEAT") << "`";
+ exit(EXIT_FAILURE);
+ }
}
if ((bool(getenv("YATL_QUIET")) and (strcmp(getenv("YATL_QUIET"), "0") == 0)) or opt_quiet)
}
}
+ if ((bool(getenv("YATL_RUN_MASSIVE_TESTS"))) or opt_massive)
+ {
+ opt_massive= true;
+ }
+
if (opt_quiet)
{
close(STDOUT_FILENO);
}
- char buffer[1024];
+ if (opt_massive)
+ {
+ is_massive(opt_massive);
+ }
+
+ libtest::vchar_t tmp_directory;
+ tmp_directory.resize(1024);
if (getenv("LIBTEST_TMP"))
{
- snprintf(buffer, sizeof(buffer), "%s", getenv("LIBTEST_TMP"));
+ snprintf(&tmp_directory[0], tmp_directory.size(), "%s", getenv("LIBTEST_TMP"));
}
else
{
- snprintf(buffer, sizeof(buffer), "%s", LIBTEST_TEMP);
+ snprintf(&tmp_directory[0], tmp_directory.size(), "%s", LIBTEST_TEMP);
}
- if (chdir(buffer) == -1)
+ if (chdir(&tmp_directory[0]) == -1)
{
- char getcwd_buffer[1024];
- char *dir= getcwd(getcwd_buffer, sizeof(getcwd_buffer));
+ libtest::vchar_t getcwd_buffer;
+ getcwd_buffer.resize(1024);
+ char *dir= getcwd(&getcwd_buffer[0], getcwd_buffer.size());
- Error << "Unable to chdir() from " << dir << " to " << buffer << " errno:" << strerror(errno);
+ Error << "Unable to chdir() from " << dir << " to " << &tmp_directory[0] << " errno:" << strerror(errno);
return EXIT_FAILURE;
}
return EXIT_FAILURE;
}
+ if (getenv("YATL_COLLECTION_TO_RUN"))
+ {
+ if (strlen(getenv("YATL_COLLECTION_TO_RUN")))
+ {
+ collection_to_run= getenv("YATL_COLLECTION_TO_RUN");
+ }
+ }
+
+ if (collection_to_run.compare("none") == 0)
+ {
+ return EXIT_SUCCESS;
+ }
+
+ if (collection_to_run.empty() == false)
+ {
+ Out << "Only testing " << collection_to_run;
+ }
+
int exit_code;
try
do
{
exit_code= EXIT_SUCCESS;
- std::auto_ptr<Framework> frame(new Framework);
-
fatal_assert(sigignore(SIGPIPE) == 0);
libtest::SignalThread signal;
return EXIT_FAILURE;
}
- Stats stats;
-
- get_world(frame.get());
-
- test_return_t error;
- void *creators_ptr= frame->create(error);
-
- switch (error)
- {
- case TEST_SUCCESS:
- break;
-
- case TEST_SKIPPED:
- Out << "SKIP " << argv[0];
- return EXIT_SUCCESS;
-
- case TEST_FAILURE:
- return EXIT_FAILURE;
- }
-
- if (getenv("YATL_COLLECTION_TO_RUN"))
- {
- if (strlen(getenv("YATL_COLLECTION_TO_RUN")))
- {
- collection_to_run= getenv("YATL_COLLECTION_TO_RUN");
- }
- }
-
- if (collection_to_run.compare("none") == 0)
- {
- return EXIT_SUCCESS;
- }
-
- if (collection_to_run.empty() == false)
- {
- Out << "Only testing " << collection_to_run;
- }
-
- char *wildcard= NULL;
- if (argc == 3)
- {
- wildcard= argv[2];
- }
+ UNIQUE_PTR<libtest::Framework> frame(new libtest::Framework(signal, binary_name, collection_to_run, wildcard));
- for (collection_st *next= frame->collections; next and next->name and (not signal.is_shutdown()); next++)
+ // Run create(), bail on error.
{
- if (collection_to_run.empty() == false and fnmatch(collection_to_run.c_str(), next->name, 0))
+ switch (frame->create())
{
- continue;
- }
+ case TEST_SUCCESS:
+ break;
- stats.collection_total++;
-
- bool failed= false;
- bool skipped= false;
- test_return_t collection_rc;
- if (test_success(collection_rc= frame->runner()->pre(next->pre, creators_ptr)))
- {
- Out << "Collection: " << next->name;
-
- for (test_st *run= next->tests; run->name; run++)
- {
- long int load_time= 0;
-
- if (wildcard && fnmatch(wildcard, run->name, 0))
- {
- continue;
- }
-
- test_return_t return_code;
- try
- {
- if (run->requires_flush)
- {
- if (test_failed(frame->runner()->flush(creators_ptr)))
- {
- Error << "frame->runner()->flush(creators_ptr)";
- continue;
- }
- }
-
- return_code= runner_code(frame.get(), run, creators_ptr, load_time);
-
- if (return_code == TEST_SKIPPED)
- { }
- else if (return_code == TEST_FAILURE)
- {
-#if 0
- Error << " frame->runner()->run(failure)";
- signal.set_shutdown(SHUTDOWN_GRACEFUL);
-#endif
- }
-
- }
- catch (libtest::fatal &e)
- {
- Error << "Fatal exception was thrown: " << e.what();
- return_code= TEST_FAILURE;
- throw;
- }
- catch (std::exception &e)
- {
- Error << "Exception was thrown: " << e.what();
- return_code= TEST_FAILURE;
- throw;
- }
- catch (...)
- {
- Error << "Unknown exception occurred";
- return_code= TEST_FAILURE;
- throw;
- }
-
- stats.total++;
-
- switch (return_code)
- {
- case TEST_SUCCESS:
- Out << "\tTesting " << run->name << "\t\t\t\t\t" << load_time / 1000 << "." << load_time % 1000 << "[ " << test_strerror(return_code) << " ]";
- stats.success++;
- break;
-
- case TEST_FAILURE:
- stats.failed++;
- failed= true;
- Out << "\tTesting " << run->name << "\t\t\t\t\t" << "[ " << test_strerror(return_code) << " ]";
- break;
-
- case TEST_SKIPPED:
- stats.skipped++;
- skipped= true;
- Out << "\tTesting " << run->name << "\t\t\t\t\t" << "[ " << test_strerror(return_code) << " ]";
- break;
-
- default:
- fatal_message("invalid return code");
- }
-#if 0
- @TODO add code here to allow for a collection to define a method to reset to allow tests to continue.
-#endif
- }
+ case TEST_SKIPPED:
+ SKIP("SKIP was returned from framework create()");
+ break;
- (void) frame->runner()->post(next->post, creators_ptr);
+ case TEST_FAILURE:
+ std::cerr << "Could not call frame->create()" << std::endl;
+ return EXIT_FAILURE;
}
- else if (collection_rc == TEST_FAILURE)
- {
- Out << next->name << " [ failed ]";
- failed= true;
-#if 0
- signal.set_shutdown(SHUTDOWN_GRACEFUL);
-#endif
- }
- else if (collection_rc == TEST_SKIPPED)
- {
- Out << next->name << " [ skipping ]";
- skipped= true;
- }
-
- if (failed == false and skipped == false)
- {
- stats.collection_success++;
- }
-
- if (failed)
- {
- stats.collection_failed++;
- }
-
- if (skipped)
- {
- stats.collection_skipped++;
- }
-
- Outn();
}
+ frame->exec();
+
if (signal.is_shutdown() == false)
{
signal.set_shutdown(SHUTDOWN_GRACEFUL);
Out << "Tests were aborted.";
exit_code= EXIT_FAILURE;
}
- else if (stats.collection_failed)
+ else if (frame->failed())
{
Out << "Some test failed.";
exit_code= EXIT_FAILURE;
}
- else if (stats.collection_skipped and stats.collection_failed and stats.collection_success)
+ else if (frame->skipped() and frame->failed() and frame->success())
{
Out << "Some tests were skipped.";
}
- else if (stats.collection_success and (stats.collection_failed == 0))
+ else if (frame->success() and (frame->failed() == 0))
{
+ Out;
Out << "All tests completed successfully.";
}
- stats_print(&stats);
+ stats_print(frame.get());
+
+ std::ofstream xml_file;
+ std::string file_name;
+ file_name.append(&tmp_directory[0]);
+ file_name.append(frame->name());
+ file_name.append(".xml");
+ xml_file.open(file_name.c_str(), std::ios::trunc);
+ libtest::Formatter::xml(*frame, xml_file);
Outn(); // Generate a blank to break up the messages if make check/test has been run
} while (exit_code == EXIT_SUCCESS and --opt_repeat);
}
- catch (libtest::fatal& e)
+ catch (const libtest::__skipped& e)
+ {
+ return EXIT_SKIP;
+ }
+ catch (const libtest::__failure& e)
{
- std::cerr << e.what() << std::endl;
+ libtest::stream::make_cout(e.file(), e.line(), e.func()) << e.what();
+ exit_code= EXIT_FAILURE;
}
- catch (libtest::disconnected& e)
+ catch (const libtest::fatal& e)
+ {
+ std::cerr << "FATAL:" << e.what() << std::endl;
+ exit_code= EXIT_FAILURE;
+ }
+ catch (const libtest::disconnected& e)
{
std::cerr << "Unhandled disconnection occurred:" << e.what() << std::endl;
+ exit_code= EXIT_FAILURE;
+ }
+ catch (const std::exception& e)
+ {
+ std::cerr << "std::exception:" << e.what() << std::endl;
+ exit_code= EXIT_FAILURE;
}
- catch (std::exception& e)
+ catch (char const* s)
{
- std::cerr << e.what() << std::endl;
+ std::cerr << "Exception:" << s << std::endl;
+ exit_code= EXIT_FAILURE;
}
catch (...)
{
std::cerr << "Unknown exception halted execution." << std::endl;
+ exit_code= EXIT_FAILURE;
}
return exit_code;