Merging bzr://gaz.tangent.org/libmemcached/build/ to Build branch
[awesomized/libmemcached] / libtest / test.cc
1 /* vim:expandtab:shiftwidth=2:tabstop=2:smarttab:
2 *
3 * libtest
4 *
5 * Copyright (C) 2011 Data Differential, http://datadifferential.com/
6 *
7 * This library is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU Lesser General Public
9 * License as published by the Free Software Foundation; either
10 * version 3 of the License, or (at your option) any later version.
11 *
12 * This library is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Lesser General Public License for more details.
16 *
17 * You should have received a copy of the GNU Lesser General Public
18 * License along with this library; if not, write to the Free Software
19 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20 */
21
22 #include <config.h>
23 #include <libtest/common.h>
24
25 #include <cassert>
26 #include <cstdlib>
27 #include <cstring>
28 #include <sys/time.h>
29 #include <sys/types.h>
30 #include <sys/stat.h>
31 #include <sys/wait.h>
32 #include <unistd.h>
33 #include <ctime>
34 #include <fnmatch.h>
35 #include <iostream>
36
37 #include <signal.h>
38
39 #ifndef __INTEL_COMPILER
40 #pragma GCC diagnostic ignored "-Wold-style-cast"
41 #endif
42
43 using namespace libtest;
44
45 static void stats_print(Stats *stats)
46 {
47 if (stats->collection_failed == 0 and stats->collection_success == 0)
48 {
49 return;
50 }
51
52 Out << "\tTotal Collections\t\t\t\t" << stats->collection_total;
53 Out << "\tFailed Collections\t\t\t\t" << stats->collection_failed;
54 Out << "\tSkipped Collections\t\t\t\t" << stats->collection_skipped;
55 Out << "\tSucceeded Collections\t\t\t\t" << stats->collection_success;
56 Outn();
57 Out << "Total\t\t\t\t" << stats->total;
58 Out << "\tFailed\t\t\t" << stats->failed;
59 Out << "\tSkipped\t\t\t" << stats->skipped;
60 Out << "\tSucceeded\t\t" << stats->success;
61 }
62
63 static long int timedif(struct timeval a, struct timeval b)
64 {
65 long us, s;
66
67 us = (long)(a.tv_usec - b.tv_usec);
68 us /= 1000;
69 s = (long)(a.tv_sec - b.tv_sec);
70 s *= 1000;
71 return s + us;
72 }
73
74 #include <getopt.h>
75 #include <unistd.h>
76
77 int main(int argc, char *argv[])
78 {
79 bool opt_massive= false;
80 unsigned long int opt_repeat= 1; // Run all tests once
81 bool opt_quiet= false;
82 std::string collection_to_run;
83
84 // Options parsing
85 {
86 enum long_option_t {
87 OPT_LIBYATL_VERSION,
88 OPT_LIBYATL_MATCH_COLLECTION,
89 OPT_LIBYATL_MASSIVE,
90 OPT_LIBYATL_QUIET,
91 OPT_LIBYATL_REPEAT
92 };
93
94 static struct option long_options[]=
95 {
96 { "version", no_argument, NULL, OPT_LIBYATL_VERSION },
97 { "quiet", no_argument, NULL, OPT_LIBYATL_QUIET },
98 { "repeat", no_argument, NULL, OPT_LIBYATL_REPEAT },
99 { "collection", required_argument, NULL, OPT_LIBYATL_MATCH_COLLECTION },
100 { "massive", no_argument, NULL, OPT_LIBYATL_MASSIVE },
101 { 0, 0, 0, 0 }
102 };
103
104 int option_index= 0;
105 while (1)
106 {
107 int option_rv= getopt_long(argc, argv, "", long_options, &option_index);
108 if (option_rv == -1)
109 {
110 break;
111 }
112
113 switch (option_rv)
114 {
115 case OPT_LIBYATL_VERSION:
116 break;
117
118 case OPT_LIBYATL_QUIET:
119 opt_quiet= true;
120 break;
121
122 case OPT_LIBYATL_REPEAT:
123 opt_repeat= strtoul(optarg, (char **) NULL, 10);
124 break;
125
126 case OPT_LIBYATL_MATCH_COLLECTION:
127 collection_to_run= optarg;
128 break;
129
130 case OPT_LIBYATL_MASSIVE:
131 opt_massive= true;
132 break;
133
134 case '?':
135 /* getopt_long already printed an error message. */
136 Error << "unknown option to getopt_long()";
137 exit(EXIT_FAILURE);
138
139 default:
140 break;
141 }
142 }
143 }
144
145 srandom((unsigned int)time(NULL));
146
147 if (bool(getenv("YATL_REPEAT")) and (strtoul(getenv("YATL_REPEAT"), (char **) NULL, 10) > 1))
148 {
149 opt_repeat= strtoul(getenv("YATL_REPEAT"), (char **) NULL, 10);
150 }
151
152 if ((bool(getenv("YATL_QUIET")) and (strcmp(getenv("YATL_QUIET"), "0") == 0)) or opt_quiet)
153 {
154 opt_quiet= true;
155 }
156 else if (getenv("JENKINS_URL"))
157 {
158 if (bool(getenv("YATL_QUIET")) and (strcmp(getenv("YATL_QUIET"), "1") == 0))
159 { }
160 else
161 {
162 opt_quiet= true;
163 }
164 }
165
166 if (opt_quiet)
167 {
168 close(STDOUT_FILENO);
169 }
170
171 char buffer[1024];
172 if (getenv("LIBTEST_TMP"))
173 {
174 snprintf(buffer, sizeof(buffer), "%s", getenv("LIBTEST_TMP"));
175 }
176 else
177 {
178 snprintf(buffer, sizeof(buffer), "%s", LIBTEST_TEMP);
179 }
180
181 if (chdir(buffer) == -1)
182 {
183 char getcwd_buffer[1024];
184 char *dir= getcwd(getcwd_buffer, sizeof(getcwd_buffer));
185
186 Error << "Unable to chdir() from " << dir << " to " << buffer << " errno:" << strerror(errno);
187 return EXIT_FAILURE;
188 }
189
190 if (libtest::libtool() == NULL)
191 {
192 Error << "Failed to locate libtool";
193 return EXIT_FAILURE;
194 }
195
196 int exit_code;
197
198 try {
199 do {
200 exit_code= EXIT_SUCCESS;
201 Framework world;
202
203 fatal_assert(sigignore(SIGPIPE) == 0);
204
205 libtest::SignalThread signal;
206 if (signal.setup() == false)
207 {
208 Error << "Failed to setup signals";
209 return EXIT_FAILURE;
210 }
211
212 Stats stats;
213
214 get_world(&world);
215
216 test_return_t error;
217 void *creators_ptr= world.create(error);
218
219 switch (error)
220 {
221 case TEST_SUCCESS:
222 break;
223
224 case TEST_SKIPPED:
225 Out << "SKIP " << argv[0];
226 return EXIT_SUCCESS;
227
228 case TEST_FAILURE:
229 return EXIT_FAILURE;
230 }
231
232 if (getenv("TEST_COLLECTION"))
233 {
234 if (strlen(getenv("TEST_COLLECTION")))
235 {
236 collection_to_run= getenv("TEST_COLLECTION");
237 }
238 }
239
240 if (collection_to_run.empty() == false)
241 {
242 Out << "Only testing " << collection_to_run;
243 }
244
245 char *wildcard= NULL;
246 if (argc == 3)
247 {
248 wildcard= argv[2];
249 }
250
251 for (collection_st *next= world.collections; next and next->name and (not signal.is_shutdown()); next++)
252 {
253 bool failed= false;
254 bool skipped= false;
255
256 if (collection_to_run.empty() == false and fnmatch(collection_to_run.c_str(), next->name, 0))
257 {
258 continue;
259 }
260
261 stats.collection_total++;
262
263 test_return_t collection_rc= world.startup(creators_ptr);
264
265 if (collection_rc == TEST_SUCCESS and next->pre)
266 {
267 collection_rc= world.runner()->pre(next->pre, creators_ptr);
268 }
269
270 switch (collection_rc)
271 {
272 case TEST_SUCCESS:
273 break;
274
275 case TEST_FAILURE:
276 Out << next->name << " [ failed ]";
277 failed= true;
278 signal.set_shutdown(SHUTDOWN_GRACEFUL);
279 goto cleanup;
280
281 case TEST_SKIPPED:
282 Out << next->name << " [ skipping ]";
283 skipped= true;
284 goto cleanup;
285
286 default:
287 fatal_message("invalid return code");
288 }
289
290 Out << "Collection: " << next->name;
291
292 for (test_st *run= next->tests; run->name; run++)
293 {
294 struct timeval start_time, end_time;
295 long int load_time= 0;
296
297 if (wildcard && fnmatch(wildcard, run->name, 0))
298 {
299 continue;
300 }
301
302 test_return_t return_code;
303 try {
304 if (test_success(return_code= world.item.startup(creators_ptr)))
305 {
306 if (test_success(return_code= world.item.flush(creators_ptr, run)))
307 {
308 // @note pre will fail is SKIPPED is returned
309 if (test_success(return_code= world.item.pre(creators_ptr)))
310 {
311 { // Runner Code
312 gettimeofday(&start_time, NULL);
313 assert(world.runner());
314 assert(run->test_fn);
315 try
316 {
317 return_code= world.runner()->run(run->test_fn, creators_ptr);
318 }
319 // Special case where check for the testing of the exception
320 // system.
321 catch (libtest::fatal &e)
322 {
323 if (fatal::is_disabled())
324 {
325 fatal::increment_disabled_counter();
326 return_code= TEST_SUCCESS;
327 }
328 else
329 {
330 throw;
331 }
332 }
333
334 gettimeofday(&end_time, NULL);
335 load_time= timedif(end_time, start_time);
336 }
337 }
338
339 // @todo do something if post fails
340 (void)world.item.post(creators_ptr);
341 }
342 else if (return_code == TEST_SKIPPED)
343 { }
344 else if (return_code == TEST_FAILURE)
345 {
346 Error << " item.flush(failure)";
347 signal.set_shutdown(SHUTDOWN_GRACEFUL);
348 }
349 }
350 else if (return_code == TEST_SKIPPED)
351 { }
352 else if (return_code == TEST_FAILURE)
353 {
354 Error << " item.startup(failure)";
355 signal.set_shutdown(SHUTDOWN_GRACEFUL);
356 }
357 }
358
359 catch (libtest::fatal &e)
360 {
361 Error << "Fatal exception was thrown: " << e.what();
362 return_code= TEST_FAILURE;
363 }
364 catch (std::exception &e)
365 {
366 Error << "Exception was thrown: " << e.what();
367 return_code= TEST_FAILURE;
368 }
369 catch (...)
370 {
371 Error << "Unknown exception occurred";
372 return_code= TEST_FAILURE;
373 }
374
375 stats.total++;
376
377 switch (return_code)
378 {
379 case TEST_SUCCESS:
380 Out << "\tTesting " << run->name << "\t\t\t\t\t" << load_time / 1000 << "." << load_time % 1000 << "[ " << test_strerror(return_code) << " ]";
381 stats.success++;
382 break;
383
384 case TEST_FAILURE:
385 stats.failed++;
386 failed= true;
387 Out << "\tTesting " << run->name << "\t\t\t\t\t" << "[ " << test_strerror(return_code) << " ]";
388 break;
389
390 case TEST_SKIPPED:
391 stats.skipped++;
392 skipped= true;
393 Out << "\tTesting " << run->name << "\t\t\t\t\t" << "[ " << test_strerror(return_code) << " ]";
394 break;
395
396 default:
397 fatal_message("invalid return code");
398 }
399
400 if (test_failed(world.on_error(return_code, creators_ptr)))
401 {
402 Error << "Failed while running on_error()";
403 signal.set_shutdown(SHUTDOWN_GRACEFUL);
404 break;
405 }
406 }
407
408 (void) world.runner()->post(next->post, creators_ptr);
409
410 cleanup:
411 if (failed == false and skipped == false)
412 {
413 stats.collection_success++;
414 }
415
416 if (failed)
417 {
418 stats.collection_failed++;
419 }
420
421 if (skipped)
422 {
423 stats.collection_skipped++;
424 }
425
426 world.shutdown(creators_ptr);
427 Outn();
428 }
429
430 if (not signal.is_shutdown())
431 {
432 signal.set_shutdown(SHUTDOWN_GRACEFUL);
433 }
434
435 shutdown_t status= signal.get_shutdown();
436 if (status == SHUTDOWN_FORCED)
437 {
438 Out << "Tests were aborted.";
439 exit_code= EXIT_FAILURE;
440 }
441 else if (stats.collection_failed)
442 {
443 Out << "Some test failed.";
444 exit_code= EXIT_FAILURE;
445 }
446 else if (stats.collection_skipped and stats.collection_failed and stats.collection_success)
447 {
448 Out << "Some tests were skipped.";
449 }
450 else if (stats.collection_success and stats.collection_failed == 0)
451 {
452 Out << "All tests completed successfully.";
453 }
454
455 stats_print(&stats);
456
457 Outn(); // Generate a blank to break up the messages if make check/test has been run
458 } while (exit_code == EXIT_SUCCESS and --opt_repeat);
459 }
460 catch (libtest::fatal& e)
461 {
462 std::cerr << e.what() << std::endl;
463 }
464 catch (std::exception& e)
465 {
466 std::cerr << e.what() << std::endl;
467 }
468 catch (...)
469 {
470 std::cerr << "Unknown exception halted execution." << std::endl;
471 }
472
473 return exit_code;
474 }