Free port logic
[awesomized/libmemcached] / libtest / test.cc
1 /* vim:expandtab:shiftwidth=2:tabstop=2:smarttab:
2 *
3 * libtest
4 *
5 * Copyright (C) 2011 Data Differential, http://datadifferential.com/
6 *
7 * This library is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU Lesser General Public
9 * License as published by the Free Software Foundation; either
10 * version 3 of the License, or (at your option) any later version.
11 *
12 * This library is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Lesser General Public License for more details.
16 *
17 * You should have received a copy of the GNU Lesser General Public
18 * License along with this library; if not, write to the Free Software
19 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20 */
21
22 #include <libtest/common.h>
23
24 #include <cassert>
25 #include <cstdlib>
26 #include <cstring>
27 #include <sys/time.h>
28 #include <sys/types.h>
29 #include <sys/stat.h>
30 #include <sys/wait.h>
31 #include <unistd.h>
32 #include <ctime>
33 #include <fnmatch.h>
34 #include <iostream>
35
36 #include <signal.h>
37
38 #if defined(HAVE_CURL_CURL_H) && HAVE_CURL_CURL_H
39 #include <curl/curl.h>
40 #endif
41
42 #ifndef __INTEL_COMPILER
43 #pragma GCC diagnostic ignored "-Wold-style-cast"
44 #endif
45
46 using namespace libtest;
47
48 static void stats_print(Stats *stats)
49 {
50 if (stats->collection_failed == 0 and stats->collection_success == 0)
51 {
52 return;
53 }
54
55 Out << "\tTotal Collections\t\t\t\t" << stats->collection_total;
56 Out << "\tFailed Collections\t\t\t\t" << stats->collection_failed;
57 Out << "\tSkipped Collections\t\t\t\t" << stats->collection_skipped;
58 Out << "\tSucceeded Collections\t\t\t\t" << stats->collection_success;
59 Outn();
60 Out << "Total\t\t\t\t" << stats->total;
61 Out << "\tFailed\t\t\t" << stats->failed;
62 Out << "\tSkipped\t\t\t" << stats->skipped;
63 Out << "\tSucceeded\t\t" << stats->success;
64 }
65
66 static long int timedif(struct timeval a, struct timeval b)
67 {
68 long us, s;
69
70 us = (long)(a.tv_usec - b.tv_usec);
71 us /= 1000;
72 s = (long)(a.tv_sec - b.tv_sec);
73 s *= 1000;
74 return s + us;
75 }
76
77 static void cleanup_curl(void)
78 {
79 #if defined(HAVE_CURL_CURL_H) && HAVE_CURL_CURL_H
80 curl_global_cleanup();
81 #endif
82 }
83
84 #include <getopt.h>
85 #include <unistd.h>
86
87 int main(int argc, char *argv[])
88 {
89 #if defined(HAVE_CURL_CURL_H) && HAVE_CURL_CURL_H
90 if (curl_global_init(CURL_GLOBAL_ALL))
91 {
92 Error << "curl_global_init(CURL_GLOBAL_ALL) failed";
93 return EXIT_FAILURE;
94 }
95 #endif
96
97 if (atexit(cleanup_curl))
98 {
99 Error << "atexit() failed";
100 return EXIT_FAILURE;
101 }
102
103 bool opt_repeat= false;
104 std::string collection_to_run;
105
106 // Options parsing
107 {
108 enum long_option_t {
109 OPT_LIBYATL_VERSION,
110 OPT_LIBYATL_MATCH_COLLECTION,
111 OPT_LIBYATL_REPEAT
112 };
113
114 static struct option long_options[]=
115 {
116 {"repeat", no_argument, NULL, OPT_LIBYATL_REPEAT},
117 {"collection", required_argument, NULL, OPT_LIBYATL_MATCH_COLLECTION},
118 {0, 0, 0, 0}
119 };
120
121 int option_index= 0;
122 while (1)
123 {
124 int option_rv= getopt_long(argc, argv, "", long_options, &option_index);
125 if (option_rv == -1)
126 {
127 break;
128 }
129
130 switch (option_rv)
131 {
132 case OPT_LIBYATL_VERSION:
133 break;
134
135 case OPT_LIBYATL_REPEAT:
136 opt_repeat= true;
137 break;
138
139 case OPT_LIBYATL_MATCH_COLLECTION:
140 collection_to_run= optarg;
141 break;
142
143 case '?':
144 /* getopt_long already printed an error message. */
145 Error << "unknown option to getopt_long()";
146 exit(EXIT_FAILURE);
147
148 default:
149 break;
150 }
151 }
152 }
153
154 srandom((unsigned int)time(NULL));
155
156 if (getenv("LIBTEST_QUIET") and strcmp(getenv("LIBTEST_QUIET"), "0") == 0)
157 {
158 close(STDOUT_FILENO);
159 }
160 else if (getenv("JENKINS_URL"))
161 {
162 close(STDOUT_FILENO);
163 }
164
165 char buffer[1024];
166 if (getenv("LIBTEST_TMP"))
167 {
168 snprintf(buffer, sizeof(buffer), "%s", getenv("LIBTEST_TMP"));
169 }
170 else
171 {
172 snprintf(buffer, sizeof(buffer), "%s", LIBTEST_TEMP);
173 }
174
175 if (chdir(buffer) == -1)
176 {
177 char getcwd_buffer[1024];
178 char *dir= getcwd(getcwd_buffer, sizeof(getcwd_buffer));
179
180 Error << "Unable to chdir() from " << dir << " to " << buffer << " errno:" << strerror(errno);
181 return EXIT_FAILURE;
182 }
183
184 if (libtest::libtool() == NULL)
185 {
186 Error << "Failed to locate libtool";
187 return EXIT_FAILURE;
188 }
189
190 int exit_code;
191
192 try {
193 do {
194 exit_code= EXIT_SUCCESS;
195 Framework *world= new Framework();
196
197 if (world == NULL)
198 {
199 Error << "Failed to create Framework()";
200 return EXIT_FAILURE;
201 }
202
203 assert(sigignore(SIGPIPE) == 0);
204
205 libtest::SignalThread signal;
206 if (not signal.setup())
207 {
208 Error << "Failed to setup signals";
209 return EXIT_FAILURE;
210 }
211
212 Stats stats;
213
214 get_world(world);
215
216 test_return_t error;
217 void *creators_ptr= world->create(error);
218
219 switch (error)
220 {
221 case TEST_SUCCESS:
222 break;
223
224 case TEST_SKIPPED:
225 Out << "SKIP " << argv[0];
226 delete world;
227 return EXIT_SUCCESS;
228
229 case TEST_FATAL:
230 case TEST_FAILURE:
231 case TEST_MEMORY_ALLOCATION_FAILURE:
232 delete world;
233 return EXIT_FAILURE;
234 }
235
236 if (getenv("TEST_COLLECTION"))
237 {
238 if (strlen(getenv("TEST_COLLECTION")))
239 {
240 collection_to_run= getenv("TEST_COLLECTION");
241 }
242 }
243
244 if (collection_to_run.empty() == false)
245 {
246 Out << "Only testing " << collection_to_run;
247 }
248
249 char *wildcard= NULL;
250 if (argc == 3)
251 {
252 wildcard= argv[2];
253 }
254
255 for (collection_st *next= world->collections; next and next->name and (not signal.is_shutdown()); next++)
256 {
257 bool failed= false;
258 bool skipped= false;
259
260 if (collection_to_run.empty() == false and fnmatch(collection_to_run.c_str(), next->name, 0))
261 {
262 continue;
263 }
264
265 stats.collection_total++;
266
267 test_return_t collection_rc= world->startup(creators_ptr);
268
269 if (collection_rc == TEST_SUCCESS and next->pre)
270 {
271 collection_rc= world->runner()->pre(next->pre, creators_ptr);
272 }
273
274 switch (collection_rc)
275 {
276 case TEST_SUCCESS:
277 break;
278
279 case TEST_FATAL:
280 case TEST_FAILURE:
281 Out << next->name << " [ failed ]";
282 failed= true;
283 signal.set_shutdown(SHUTDOWN_GRACEFUL);
284 goto cleanup;
285
286 case TEST_SKIPPED:
287 Out << next->name << " [ skipping ]";
288 skipped= true;
289 goto cleanup;
290
291 case TEST_MEMORY_ALLOCATION_FAILURE:
292 test_assert(0, "Allocation failure, or unknown return");
293 }
294
295 Out << "Collection: " << next->name;
296
297 for (test_st *run= next->tests; run->name; run++)
298 {
299 struct timeval start_time, end_time;
300 long int load_time= 0;
301
302 if (wildcard && fnmatch(wildcard, run->name, 0))
303 {
304 continue;
305 }
306
307 test_return_t return_code;
308 try {
309 if (test_success(return_code= world->item.startup(creators_ptr)))
310 {
311 if (test_success(return_code= world->item.flush(creators_ptr, run)))
312 {
313 // @note pre will fail is SKIPPED is returned
314 if (test_success(return_code= world->item.pre(creators_ptr)))
315 {
316 { // Runner Code
317 gettimeofday(&start_time, NULL);
318 assert(world->runner());
319 assert(run->test_fn);
320 try
321 {
322 return_code= world->runner()->run(run->test_fn, creators_ptr);
323 }
324 // Special case where check for the testing of the exception
325 // system.
326 catch (libtest::fatal &e)
327 {
328 if (fatal::is_disabled())
329 {
330 fatal::increment_disabled_counter();
331 return_code= TEST_SUCCESS;
332 }
333 else
334 {
335 throw;
336 }
337 }
338
339 gettimeofday(&end_time, NULL);
340 load_time= timedif(end_time, start_time);
341 }
342 }
343
344 // @todo do something if post fails
345 (void)world->item.post(creators_ptr);
346 }
347 else if (return_code == TEST_SKIPPED)
348 { }
349 else if (return_code == TEST_FAILURE)
350 {
351 Error << " item.flush(failure)";
352 signal.set_shutdown(SHUTDOWN_GRACEFUL);
353 }
354 }
355 else if (return_code == TEST_SKIPPED)
356 { }
357 else if (return_code == TEST_FAILURE)
358 {
359 Error << " item.startup(failure)";
360 signal.set_shutdown(SHUTDOWN_GRACEFUL);
361 }
362 }
363
364 catch (std::exception &e)
365 {
366 Error << "Exception was thrown: " << e.what();
367 return_code= TEST_FAILURE;
368 }
369 catch (...)
370 {
371 Error << "Unknown exception occurred";
372 return_code= TEST_FAILURE;
373 }
374
375 stats.total++;
376
377 switch (return_code)
378 {
379 case TEST_SUCCESS:
380 Out << "\tTesting " << run->name << "\t\t\t\t\t" << load_time / 1000 << "." << load_time % 1000 << "[ " << test_strerror(return_code) << " ]";
381 stats.success++;
382 break;
383
384 case TEST_FATAL:
385 case TEST_FAILURE:
386 stats.failed++;
387 failed= true;
388 Out << "\tTesting " << run->name << "\t\t\t\t\t" << "[ " << test_strerror(return_code) << " ]";
389 break;
390
391 case TEST_SKIPPED:
392 stats.skipped++;
393 skipped= true;
394 Out << "\tTesting " << run->name << "\t\t\t\t\t" << "[ " << test_strerror(return_code) << " ]";
395 break;
396
397 case TEST_MEMORY_ALLOCATION_FAILURE:
398 test_assert(0, "Memory Allocation Error");
399 }
400
401 if (test_failed(world->on_error(return_code, creators_ptr)))
402 {
403 Error << "Failed while running on_error()";
404 signal.set_shutdown(SHUTDOWN_GRACEFUL);
405 break;
406 }
407 }
408
409 (void) world->runner()->post(next->post, creators_ptr);
410
411 cleanup:
412 if (failed == false and skipped == false)
413 {
414 stats.collection_success++;
415 }
416
417 if (failed)
418 {
419 stats.collection_failed++;
420 }
421
422 if (skipped)
423 {
424 stats.collection_skipped++;
425 }
426
427 world->shutdown(creators_ptr);
428 Outn();
429 }
430
431 if (not signal.is_shutdown())
432 {
433 signal.set_shutdown(SHUTDOWN_GRACEFUL);
434 }
435
436 shutdown_t status= signal.get_shutdown();
437 if (status == SHUTDOWN_FORCED)
438 {
439 Out << "Tests were aborted.";
440 exit_code= EXIT_FAILURE;
441 }
442 else if (stats.collection_failed)
443 {
444 Out << "Some test failed.";
445 exit_code= EXIT_FAILURE;
446 }
447 else if (stats.collection_skipped and stats.collection_failed and stats.collection_success)
448 {
449 Out << "Some tests were skipped.";
450 }
451 else if (stats.collection_success and stats.collection_failed == 0)
452 {
453 Out << "All tests completed successfully.";
454 }
455
456 stats_print(&stats);
457
458 delete world;
459
460 Outn(); // Generate a blank to break up the messages if make check/test has been run
461 } while (exit_code == EXIT_SUCCESS and opt_repeat);
462 }
463 catch (libtest::fatal& e)
464 {
465 std::cerr << e.what() << std::endl;
466 }
467 catch (...)
468 {
469 std::cerr << "Unknown exception halted execution" << std::endl;
470 }
471
472 return exit_code;
473 }