Merge from build
[awesomized/libmemcached] / libtest / test.cc
1 /* vim:expandtab:shiftwidth=2:tabstop=2:smarttab:
2 *
3 * libtest
4 *
5 * Copyright (C) 2011 Data Differential, http://datadifferential.com/
6 *
7 * This library is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU Lesser General Public
9 * License as published by the Free Software Foundation; either
10 * version 3 of the License, or (at your option) any later version.
11 *
12 * This library is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Lesser General Public License for more details.
16 *
17 * You should have received a copy of the GNU Lesser General Public
18 * License along with this library; if not, write to the Free Software
19 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20 */
21
22 #include <libtest/common.h>
23
24 #include <cassert>
25 #include <cstdlib>
26 #include <cstring>
27 #include <sys/time.h>
28 #include <sys/types.h>
29 #include <sys/stat.h>
30 #include <sys/wait.h>
31 #include <unistd.h>
32 #include <ctime>
33 #include <fnmatch.h>
34 #include <iostream>
35
36 #include <signal.h>
37
38 #if defined(HAVE_CURL_CURL_H) && HAVE_CURL_CURL_H
39 #include <curl/curl.h>
40 #endif
41
42 #ifndef __INTEL_COMPILER
43 #pragma GCC diagnostic ignored "-Wold-style-cast"
44 #endif
45
46 using namespace libtest;
47
48 static void stats_print(Stats *stats)
49 {
50 if (stats->collection_failed == 0 and stats->collection_success == 0)
51 {
52 return;
53 }
54
55 Out << "\tTotal Collections\t\t\t\t" << stats->collection_total;
56 Out << "\tFailed Collections\t\t\t\t" << stats->collection_failed;
57 Out << "\tSkipped Collections\t\t\t\t" << stats->collection_skipped;
58 Out << "\tSucceeded Collections\t\t\t\t" << stats->collection_success;
59 Outn();
60 Out << "Total\t\t\t\t" << stats->total;
61 Out << "\tFailed\t\t\t" << stats->failed;
62 Out << "\tSkipped\t\t\t" << stats->skipped;
63 Out << "\tSucceeded\t\t" << stats->success;
64 }
65
66 static long int timedif(struct timeval a, struct timeval b)
67 {
68 long us, s;
69
70 us = (long)(a.tv_usec - b.tv_usec);
71 us /= 1000;
72 s = (long)(a.tv_sec - b.tv_sec);
73 s *= 1000;
74 return s + us;
75 }
76
77 static void cleanup_curl(void)
78 {
79 #if defined(HAVE_CURL_CURL_H) && HAVE_CURL_CURL_H
80 curl_global_cleanup();
81 #endif
82 }
83
84 #include <getopt.h>
85 #include <unistd.h>
86
87 int main(int argc, char *argv[])
88 {
89 #if defined(HAVE_CURL_CURL_H) && HAVE_CURL_CURL_H
90 if (curl_global_init(CURL_GLOBAL_ALL))
91 {
92 Error << "curl_global_init(CURL_GLOBAL_ALL) failed";
93 return EXIT_FAILURE;
94 }
95 #endif
96
97 if (atexit(cleanup_curl))
98 {
99 Error << "atexit() failed";
100 return EXIT_FAILURE;
101 }
102
103 bool opt_repeat= false;
104 std::string collection_to_run;
105
106 // Options parsing
107 {
108 enum long_option_t {
109 OPT_LIBYATL_VERSION,
110 OPT_LIBYATL_MATCH_COLLECTION,
111 OPT_LIBYATL_REPEAT
112 };
113
114 static struct option long_options[]=
115 {
116 {"repeat", no_argument, NULL, OPT_LIBYATL_REPEAT},
117 {"collection", required_argument, NULL, OPT_LIBYATL_MATCH_COLLECTION},
118 {0, 0, 0, 0}
119 };
120
121 int option_index= 0;
122 while (1)
123 {
124 int option_rv= getopt_long(argc, argv, "", long_options, &option_index);
125 if (option_rv == -1)
126 {
127 break;
128 }
129
130 switch (option_rv)
131 {
132 case OPT_LIBYATL_VERSION:
133 break;
134
135 case OPT_LIBYATL_REPEAT:
136 opt_repeat= true;
137 break;
138
139 case OPT_LIBYATL_MATCH_COLLECTION:
140 collection_to_run= optarg;
141 break;
142
143 case '?':
144 /* getopt_long already printed an error message. */
145 Error << "unknown option to getopt_long()";
146 exit(EXIT_FAILURE);
147
148 default:
149 break;
150 }
151 }
152 }
153
154 srandom((unsigned int)time(NULL));
155
156 if (getenv("LIBTEST_QUIET") and strcmp(getenv("LIBTEST_QUIET"), "0") == 0)
157 {
158 close(STDOUT_FILENO);
159 }
160 else if (getenv("JENKINS_URL"))
161 {
162 close(STDOUT_FILENO);
163 }
164
165 char buffer[1024];
166 if (getenv("LIBTEST_TMP"))
167 {
168 snprintf(buffer, sizeof(buffer), "%s", getenv("LIBTEST_TMP"));
169 }
170 else
171 {
172 snprintf(buffer, sizeof(buffer), "%s", LIBTEST_TEMP);
173 }
174
175 if (chdir(buffer) == -1)
176 {
177 char getcwd_buffer[1024];
178 char *dir= getcwd(getcwd_buffer, sizeof(getcwd_buffer));
179
180 Error << "Unable to chdir() from " << dir << " to " << buffer << " errno:" << strerror(errno);
181 return EXIT_FAILURE;
182 }
183
184 if (libtest::libtool() == NULL)
185 {
186 Error << "Failed to locate libtool";
187 return EXIT_FAILURE;
188 }
189
190 int exit_code;
191 do {
192 exit_code= EXIT_SUCCESS;
193 Framework *world= new Framework();
194
195 if (world == NULL)
196 {
197 Error << "Failed to create Framework()";
198 return EXIT_FAILURE;
199 }
200
201 assert(sigignore(SIGPIPE) == 0);
202
203 libtest::SignalThread signal;
204 if (not signal.setup())
205 {
206 Error << "Failed to setup signals";
207 return EXIT_FAILURE;
208 }
209
210 Stats stats;
211
212 get_world(world);
213
214 test_return_t error;
215 void *creators_ptr= world->create(error);
216
217 switch (error)
218 {
219 case TEST_SUCCESS:
220 break;
221
222 case TEST_SKIPPED:
223 Out << "SKIP " << argv[0];
224 delete world;
225 return EXIT_SUCCESS;
226
227 case TEST_FATAL:
228 case TEST_FAILURE:
229 case TEST_MEMORY_ALLOCATION_FAILURE:
230 delete world;
231 return EXIT_FAILURE;
232 }
233
234 if (getenv("TEST_COLLECTION"))
235 {
236 if (strlen(getenv("TEST_COLLECTION")))
237 {
238 collection_to_run= getenv("TEST_COLLECTION");
239 }
240 }
241
242 if (collection_to_run.empty() == false)
243 {
244 Out << "Only testing " << collection_to_run;
245 }
246
247 char *wildcard= NULL;
248 if (argc == 3)
249 {
250 wildcard= argv[2];
251 }
252
253 for (collection_st *next= world->collections; next and next->name and (not signal.is_shutdown()); next++)
254 {
255 bool failed= false;
256 bool skipped= false;
257
258 if (collection_to_run.empty() == false and fnmatch(collection_to_run.c_str(), next->name, 0))
259 {
260 continue;
261 }
262
263 stats.collection_total++;
264
265 test_return_t collection_rc= world->startup(creators_ptr);
266
267 if (collection_rc == TEST_SUCCESS and next->pre)
268 {
269 collection_rc= world->runner()->pre(next->pre, creators_ptr);
270 }
271
272 switch (collection_rc)
273 {
274 case TEST_SUCCESS:
275 break;
276
277 case TEST_FATAL:
278 case TEST_FAILURE:
279 Out << next->name << " [ failed ]";
280 failed= true;
281 signal.set_shutdown(SHUTDOWN_GRACEFUL);
282 goto cleanup;
283
284 case TEST_SKIPPED:
285 Out << next->name << " [ skipping ]";
286 skipped= true;
287 goto cleanup;
288
289 case TEST_MEMORY_ALLOCATION_FAILURE:
290 test_assert(0, "Allocation failure, or unknown return");
291 }
292
293 Out << "Collection: " << next->name;
294
295 for (test_st *run= next->tests; run->name; run++)
296 {
297 struct timeval start_time, end_time;
298 long int load_time= 0;
299
300 if (wildcard && fnmatch(wildcard, run->name, 0))
301 {
302 continue;
303 }
304
305 test_return_t return_code;
306 try {
307 if (test_success(return_code= world->item.startup(creators_ptr)))
308 {
309 if (test_success(return_code= world->item.flush(creators_ptr, run)))
310 {
311 // @note pre will fail is SKIPPED is returned
312 if (test_success(return_code= world->item.pre(creators_ptr)))
313 {
314 { // Runner Code
315 gettimeofday(&start_time, NULL);
316 assert(world->runner());
317 assert(run->test_fn);
318 return_code= world->runner()->run(run->test_fn, creators_ptr);
319 gettimeofday(&end_time, NULL);
320 load_time= timedif(end_time, start_time);
321 }
322 }
323
324 // @todo do something if post fails
325 (void)world->item.post(creators_ptr);
326 }
327 else if (return_code == TEST_SKIPPED)
328 { }
329 else if (return_code == TEST_FAILURE)
330 {
331 Error << " item.flush(failure)";
332 signal.set_shutdown(SHUTDOWN_GRACEFUL);
333 }
334 }
335 else if (return_code == TEST_SKIPPED)
336 { }
337 else if (return_code == TEST_FAILURE)
338 {
339 Error << " item.startup(failure)";
340 signal.set_shutdown(SHUTDOWN_GRACEFUL);
341 }
342 }
343
344 catch (std::exception &e)
345 {
346 Error << "Exception was thrown: " << e.what();
347 return_code= TEST_FAILURE;
348 }
349 catch (...)
350 {
351 Error << "Unknown exception occurred";
352 return_code= TEST_FAILURE;
353 }
354
355 stats.total++;
356
357 switch (return_code)
358 {
359 case TEST_SUCCESS:
360 Out << "\tTesting " << run->name << "\t\t\t\t\t" << load_time / 1000 << "." << load_time % 1000 << "[ " << test_strerror(return_code) << " ]";
361 stats.success++;
362 break;
363
364 case TEST_FATAL:
365 case TEST_FAILURE:
366 stats.failed++;
367 failed= true;
368 Out << "\tTesting " << run->name << "\t\t\t\t\t" << "[ " << test_strerror(return_code) << " ]";
369 break;
370
371 case TEST_SKIPPED:
372 stats.skipped++;
373 skipped= true;
374 Out << "\tTesting " << run->name << "\t\t\t\t\t" << "[ " << test_strerror(return_code) << " ]";
375 break;
376
377 case TEST_MEMORY_ALLOCATION_FAILURE:
378 test_assert(0, "Memory Allocation Error");
379 }
380
381 if (test_failed(world->on_error(return_code, creators_ptr)))
382 {
383 Error << "Failed while running on_error()";
384 signal.set_shutdown(SHUTDOWN_GRACEFUL);
385 break;
386 }
387 }
388
389 (void) world->runner()->post(next->post, creators_ptr);
390
391 cleanup:
392 if (failed == false and skipped == false)
393 {
394 stats.collection_success++;
395 }
396
397 if (failed)
398 {
399 stats.collection_failed++;
400 }
401
402 if (skipped)
403 {
404 stats.collection_skipped++;
405 }
406
407 world->shutdown(creators_ptr);
408 Outn();
409 }
410
411 if (not signal.is_shutdown())
412 {
413 signal.set_shutdown(SHUTDOWN_GRACEFUL);
414 }
415
416 shutdown_t status= signal.get_shutdown();
417 if (status == SHUTDOWN_FORCED)
418 {
419 Out << "Tests were aborted.";
420 exit_code= EXIT_FAILURE;
421 }
422 else if (stats.collection_failed)
423 {
424 Out << "Some test failed.";
425 exit_code= EXIT_FAILURE;
426 }
427 else if (stats.collection_skipped and stats.collection_failed and stats.collection_success)
428 {
429 Out << "Some tests were skipped.";
430 }
431 else if (stats.collection_success and stats.collection_failed == 0)
432 {
433 Out << "All tests completed successfully.";
434 }
435
436 stats_print(&stats);
437
438 delete world;
439
440 Outn(); // Generate a blank to break up the messages if make check/test has been run
441 } while (exit_code == EXIT_SUCCESS and opt_repeat);
442
443 return exit_code;
444 }