1 /* vim:expandtab:shiftwidth=2:tabstop=2:smarttab:
3 * Data Differential YATL (i.e. libtest) library
5 * Copyright (C) 2012 Data Differential, http://datadifferential.com/
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions are
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
14 * * Redistributions in binary form must reproduce the above
15 * copyright notice, this list of conditions and the following disclaimer
16 * in the documentation and/or other materials provided with the
19 * * The names of its contributors may not be used to endorse or
20 * promote products derived from this software without specific prior
23 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
24 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
25 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
26 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
27 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
28 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
29 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
30 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
31 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
32 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
33 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
38 #include <libtest/common.h>
49 #include <sys/types.h>
55 #ifndef __INTEL_COMPILER
56 #pragma GCC diagnostic ignored "-Wold-style-cast"
59 using namespace libtest
;
61 static void stats_print(Stats
*stats
)
63 if (stats
->collection_failed
== 0 and stats
->collection_success
== 0)
68 Out
<< "\tTotal Collections\t\t\t\t" << stats
->collection_total
;
69 Out
<< "\tFailed Collections\t\t\t\t" << stats
->collection_failed
;
70 Out
<< "\tSkipped Collections\t\t\t\t" << stats
->collection_skipped
;
71 Out
<< "\tSucceeded Collections\t\t\t\t" << stats
->collection_success
;
73 Out
<< "Total\t\t\t\t" << stats
->total
;
74 Out
<< "\tFailed\t\t\t" << stats
->failed
;
75 Out
<< "\tSkipped\t\t\t" << stats
->skipped
;
76 Out
<< "\tSucceeded\t\t" << stats
->success
;
79 static long int timedif(struct timeval a
, struct timeval b
)
83 us
= (long)(a
.tv_usec
- b
.tv_usec
);
85 s
= (long)(a
.tv_sec
- b
.tv_sec
);
90 static test_return_t
runner_code(Framework
* frame
,
96 struct timeval start_time
, end_time
;
98 gettimeofday(&start_time
, NULL
);
99 assert(frame
->runner());
100 assert(run
->test_fn
);
102 test_return_t return_code
;
105 return_code
= frame
->runner()->run(run
->test_fn
, creators_ptr
);
107 // Special case where check for the testing of the exception
109 catch (libtest::fatal
&e
)
111 if (fatal::is_disabled())
113 fatal::increment_disabled_counter();
114 return_code
= TEST_SUCCESS
;
122 gettimeofday(&end_time
, NULL
);
123 load_time
= timedif(end_time
, start_time
);
131 int main(int argc
, char *argv
[])
133 bool opt_massive
= false;
134 unsigned long int opt_repeat
= 1; // Run all tests once
135 bool opt_quiet
= false;
136 std::string collection_to_run
;
142 OPT_LIBYATL_MATCH_COLLECTION
,
148 static struct option long_options
[]=
150 { "version", no_argument
, NULL
, OPT_LIBYATL_VERSION
},
151 { "quiet", no_argument
, NULL
, OPT_LIBYATL_QUIET
},
152 { "repeat", no_argument
, NULL
, OPT_LIBYATL_REPEAT
},
153 { "collection", required_argument
, NULL
, OPT_LIBYATL_MATCH_COLLECTION
},
154 { "massive", no_argument
, NULL
, OPT_LIBYATL_MASSIVE
},
161 int option_rv
= getopt_long(argc
, argv
, "", long_options
, &option_index
);
169 case OPT_LIBYATL_VERSION
:
172 case OPT_LIBYATL_QUIET
:
176 case OPT_LIBYATL_REPEAT
:
177 opt_repeat
= strtoul(optarg
, (char **) NULL
, 10);
180 case OPT_LIBYATL_MATCH_COLLECTION
:
181 collection_to_run
= optarg
;
184 case OPT_LIBYATL_MASSIVE
:
189 /* getopt_long already printed an error message. */
190 Error
<< "unknown option to getopt_long()";
199 srandom((unsigned int)time(NULL
));
201 if (bool(getenv("YATL_REPEAT")) and (strtoul(getenv("YATL_REPEAT"), (char **) NULL
, 10) > 1))
203 opt_repeat
= strtoul(getenv("YATL_REPEAT"), (char **) NULL
, 10);
206 if ((bool(getenv("YATL_QUIET")) and (strcmp(getenv("YATL_QUIET"), "0") == 0)) or opt_quiet
)
210 else if (getenv("JENKINS_URL"))
212 if (bool(getenv("YATL_QUIET")) and (strcmp(getenv("YATL_QUIET"), "1") == 0))
222 close(STDOUT_FILENO
);
226 if (getenv("LIBTEST_TMP"))
228 snprintf(buffer
, sizeof(buffer
), "%s", getenv("LIBTEST_TMP"));
232 snprintf(buffer
, sizeof(buffer
), "%s", LIBTEST_TEMP
);
235 if (chdir(buffer
) == -1)
237 char getcwd_buffer
[1024];
238 char *dir
= getcwd(getcwd_buffer
, sizeof(getcwd_buffer
));
240 Error
<< "Unable to chdir() from " << dir
<< " to " << buffer
<< " errno:" << strerror(errno
);
244 if (libtest::libtool() == NULL
)
246 Error
<< "Failed to locate libtool";
256 exit_code
= EXIT_SUCCESS
;
257 std::auto_ptr
<Framework
> frame(new Framework
);
259 fatal_assert(sigignore(SIGPIPE
) == 0);
261 libtest::SignalThread signal
;
262 if (signal
.setup() == false)
264 Error
<< "Failed to setup signals";
270 get_world(frame
.get());
273 void *creators_ptr
= frame
->create(error
);
281 Out
<< "SKIP " << argv
[0];
288 if (getenv("YATL_COLLECTION_TO_RUN"))
290 if (strlen(getenv("YATL_COLLECTION_TO_RUN")))
292 collection_to_run
= getenv("YATL_COLLECTION_TO_RUN");
296 if (collection_to_run
.compare("none") == 0)
301 if (collection_to_run
.empty() == false)
303 Out
<< "Only testing " << collection_to_run
;
306 char *wildcard
= NULL
;
312 for (collection_st
*next
= frame
->collections
; next
and next
->name
and (not signal
.is_shutdown()); next
++)
314 if (collection_to_run
.empty() == false and fnmatch(collection_to_run
.c_str(), next
->name
, 0))
319 stats
.collection_total
++;
323 test_return_t collection_rc
;
324 if (test_success(collection_rc
= frame
->runner()->pre(next
->pre
, creators_ptr
)))
326 Out
<< "Collection: " << next
->name
;
328 for (test_st
*run
= next
->tests
; run
->name
; run
++)
330 long int load_time
= 0;
332 if (wildcard
&& fnmatch(wildcard
, run
->name
, 0))
337 test_return_t return_code
;
340 if (run
->requires_flush
)
342 if (test_failed(frame
->runner()->flush(creators_ptr
)))
344 Error
<< "frame->runner()->flush(creators_ptr)";
349 return_code
= runner_code(frame
.get(), run
, creators_ptr
, load_time
);
351 if (return_code
== TEST_SKIPPED
)
353 else if (return_code
== TEST_FAILURE
)
356 Error
<< " frame->runner()->run(failure)";
357 signal
.set_shutdown(SHUTDOWN_GRACEFUL
);
362 catch (libtest::fatal
&e
)
364 Error
<< "Fatal exception was thrown: " << e
.what();
365 return_code
= TEST_FAILURE
;
368 catch (std::exception
&e
)
370 Error
<< "Exception was thrown: " << e
.what();
371 return_code
= TEST_FAILURE
;
376 Error
<< "Unknown exception occurred";
377 return_code
= TEST_FAILURE
;
386 Out
<< "\tTesting " << run
->name
<< "\t\t\t\t\t" << load_time
/ 1000 << "." << load_time
% 1000 << "[ " << test_strerror(return_code
) << " ]";
393 Out
<< "\tTesting " << run
->name
<< "\t\t\t\t\t" << "[ " << test_strerror(return_code
) << " ]";
399 Out
<< "\tTesting " << run
->name
<< "\t\t\t\t\t" << "[ " << test_strerror(return_code
) << " ]";
403 fatal_message("invalid return code");
406 @TODO add code here to allow
for a collection to define a method to reset to allow tests to
continue.
410 (void) frame
->runner()->post(next
->post
, creators_ptr
);
412 else if (collection_rc
== TEST_FAILURE
)
414 Out
<< next
->name
<< " [ failed ]";
417 signal
.set_shutdown(SHUTDOWN_GRACEFUL
);
420 else if (collection_rc
== TEST_SKIPPED
)
422 Out
<< next
->name
<< " [ skipping ]";
426 if (failed
== false and skipped
== false)
428 stats
.collection_success
++;
433 stats
.collection_failed
++;
438 stats
.collection_skipped
++;
444 if (signal
.is_shutdown() == false)
446 signal
.set_shutdown(SHUTDOWN_GRACEFUL
);
449 shutdown_t status
= signal
.get_shutdown();
450 if (status
== SHUTDOWN_FORCED
)
452 Out
<< "Tests were aborted.";
453 exit_code
= EXIT_FAILURE
;
455 else if (stats
.collection_failed
)
457 Out
<< "Some test failed.";
458 exit_code
= EXIT_FAILURE
;
460 else if (stats
.collection_skipped
and stats
.collection_failed
and stats
.collection_success
)
462 Out
<< "Some tests were skipped.";
464 else if (stats
.collection_success
and (stats
.collection_failed
== 0))
466 Out
<< "All tests completed successfully.";
471 Outn(); // Generate a blank to break up the messages if make check/test has been run
472 } while (exit_code
== EXIT_SUCCESS
and --opt_repeat
);
474 catch (libtest::fatal
& e
)
476 std::cerr
<< e
.what() << std::endl
;
478 catch (libtest::disconnected
& e
)
480 std::cerr
<< "Unhandled disconnection occurred:" << e
.what() << std::endl
;
482 catch (std::exception
& e
)
484 std::cerr
<< e
.what() << std::endl
;
488 std::cerr
<< "Unknown exception halted execution." << std::endl
;