89e7c07112e674c6a8a2be1e9c40cbf5c2a9733b
[awesomized/libmemcached] / libtest / test.cc
1 /* vim:expandtab:shiftwidth=2:tabstop=2:smarttab:
2 *
3 * libtest
4 *
5 * Copyright (C) 2011 Data Differential, http://datadifferential.com/
6 *
7 * This library is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU Lesser General Public
9 * License as published by the Free Software Foundation; either
10 * version 3 of the License, or (at your option) any later version.
11 *
12 * This library is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Lesser General Public License for more details.
16 *
17 * You should have received a copy of the GNU Lesser General Public
18 * License along with this library; if not, write to the Free Software
19 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20 */
21
22 #include <libtest/common.h>
23
24 #include <cassert>
25 #include <cstdlib>
26 #include <cstring>
27 #include <sys/time.h>
28 #include <sys/types.h>
29 #include <sys/stat.h>
30 #include <sys/wait.h>
31 #include <unistd.h>
32 #include <ctime>
33 #include <fnmatch.h>
34 #include <iostream>
35
36 #include <signal.h>
37
38 #include <libtest/stats.h>
39 #include <libtest/signal.h>
40
41 #ifndef __INTEL_COMPILER
42 #pragma GCC diagnostic ignored "-Wold-style-cast"
43 #endif
44
45 using namespace libtest;
46
47 static char global_socket[1024];
48
49 const char *default_socket()
50 {
51 assert(global_socket[0]);
52 return global_socket;
53 }
54
55 bool test_is_local()
56 {
57 return (getenv("LIBTEST_LOCAL"));
58 }
59
60 void set_default_socket(const char *socket)
61 {
62 if (socket)
63 {
64 strncpy(global_socket, socket, strlen(socket));
65 }
66 }
67
68 static void stats_print(Stats *stats)
69 {
70 if (stats->collection_failed == 0 and stats->collection_success == 0)
71 {
72 return;
73 }
74
75 Out << "\tTotal Collections\t\t\t\t" << stats->collection_total;
76 Out << "\tFailed Collections\t\t\t\t" << stats->collection_failed;
77 Out << "\tSkipped Collections\t\t\t\t" << stats->collection_skipped;
78 Out << "\tSucceeded Collections\t\t\t\t" << stats->collection_success;
79 Outn();
80 Out << "Total\t\t\t\t" << stats->total;
81 Out << "\tFailed\t\t\t" << stats->failed;
82 Out << "\tSkipped\t\t\t" << stats->skipped;
83 Out << "\tSucceeded\t\t" << stats->success;
84 }
85
86 static long int timedif(struct timeval a, struct timeval b)
87 {
88 long us, s;
89
90 us = (long)(a.tv_usec - b.tv_usec);
91 us /= 1000;
92 s = (long)(a.tv_sec - b.tv_sec);
93 s *= 1000;
94 return s + us;
95 }
96
97 const char *test_strerror(test_return_t code)
98 {
99 switch (code) {
100 case TEST_SUCCESS:
101 return "ok";
102
103 case TEST_FAILURE:
104 return "failed";
105
106 case TEST_MEMORY_ALLOCATION_FAILURE:
107 return "memory allocation";
108
109 case TEST_SKIPPED:
110 return "skipped";
111
112 case TEST_FATAL:
113 break;
114 }
115
116 return "failed";
117 }
118
119 void create_core(void)
120 {
121 if (getenv("LIBMEMCACHED_NO_COREDUMP") == NULL)
122 {
123 pid_t pid= fork();
124
125 if (pid == 0)
126 {
127 abort();
128 }
129 else
130 {
131 while (waitpid(pid, NULL, 0) != pid) {};
132 }
133 }
134 }
135
136 static Framework *world= NULL;
137 int main(int argc, char *argv[])
138 {
139 srandom((unsigned int)time(NULL));
140
141 if (getenv("LIBTEST_QUIET"))
142 {
143 close(STDOUT_FILENO);
144 }
145
146 char buffer[1024];
147 if (getenv("LIBTEST_TMP"))
148 {
149 snprintf(buffer, sizeof(buffer), "%s", getenv("LIBTEST_TMP"));
150 }
151 else
152 {
153 snprintf(buffer, sizeof(buffer), "%s", LIBTEST_TEMP);
154 }
155
156 if (chdir(buffer) == -1)
157 {
158 char getcwd_buffer[1024];
159 char *dir= getcwd(getcwd_buffer, sizeof(getcwd_buffer));
160
161 Error << "Unable to chdir() from " << dir << " to " << buffer << " errno:" << strerror(errno);
162 return EXIT_FAILURE;
163 }
164
165 if (libtest::libtool() == NULL)
166 {
167 Error << "Failed to locate libtool";
168 return EXIT_FAILURE;
169 }
170
171 world= new Framework();
172
173 if (not world)
174 {
175 Error << "Failed to create Framework()";
176 return EXIT_FAILURE;
177 }
178
179 libtest::SignalThread signal;
180 if (not signal.setup())
181 {
182 return EXIT_FAILURE;
183 }
184
185 Stats stats;
186
187 get_world(world);
188
189 test_return_t error;
190 void *creators_ptr= world->create(error);
191
192 switch (error)
193 {
194 case TEST_SUCCESS:
195 break;
196
197 case TEST_SKIPPED:
198 Out << "SKIP " << argv[0];
199 delete world;
200 return EXIT_SUCCESS;
201
202 case TEST_FATAL:
203 case TEST_FAILURE:
204 case TEST_MEMORY_ALLOCATION_FAILURE:
205 Error << argv[0] << " failed in Framework::create()";
206 delete world;
207 return EXIT_FAILURE;
208 }
209
210 char *collection_to_run= NULL;
211 if (argc > 1)
212 {
213 collection_to_run= argv[1];
214 }
215 else if (getenv("TEST_COLLECTION"))
216 {
217 collection_to_run= getenv("TEST_COLLECTION");
218 }
219
220 if (collection_to_run)
221 {
222 Out << "Only testing " << collection_to_run;
223 }
224
225 char *wildcard= NULL;
226 if (argc == 3)
227 {
228 wildcard= argv[2];
229 }
230
231 for (collection_st *next= world->collections; next->name and (not signal.is_shutdown()); next++)
232 {
233 test_return_t collection_rc= TEST_SUCCESS;
234 bool failed= false;
235 bool skipped= false;
236
237 if (collection_to_run && fnmatch(collection_to_run, next->name, 0))
238 continue;
239
240 stats.collection_total++;
241
242 collection_rc= world->startup(creators_ptr);
243
244 if (collection_rc == TEST_SUCCESS and next->pre)
245 {
246 collection_rc= world->runner()->pre(next->pre, creators_ptr);
247 }
248
249 switch (collection_rc)
250 {
251 case TEST_SUCCESS:
252 break;
253
254 case TEST_FATAL:
255 case TEST_FAILURE:
256 Out << next->name << " [ failed ]";
257 failed= true;
258 signal.set_shutdown(SHUTDOWN_GRACEFUL);
259 goto cleanup;
260
261 case TEST_SKIPPED:
262 Out << next->name << " [ skipping ]";
263 skipped= true;
264 goto cleanup;
265
266 case TEST_MEMORY_ALLOCATION_FAILURE:
267 test_assert(0, "Allocation failure, or unknown return");
268 }
269
270 Out << "Collection: " << next->name;
271
272 for (test_st *run= next->tests; run->name; run++)
273 {
274 struct timeval start_time, end_time;
275 long int load_time= 0;
276
277 if (wildcard && fnmatch(wildcard, run->name, 0))
278 {
279 continue;
280 }
281
282 test_return_t return_code;
283 if (test_success(return_code= world->item.startup(creators_ptr)))
284 {
285 if (test_success(return_code= world->item.flush(creators_ptr, run)))
286 {
287 // @note pre will fail is SKIPPED is returned
288 if (test_success(return_code= world->item.pre(creators_ptr)))
289 {
290 { // Runner Code
291 gettimeofday(&start_time, NULL);
292 assert(world->runner());
293 assert(run->test_fn);
294 return_code= world->runner()->run(run->test_fn, creators_ptr);
295 gettimeofday(&end_time, NULL);
296 load_time= timedif(end_time, start_time);
297 }
298 }
299
300 // @todo do something if post fails
301 (void)world->item.post(creators_ptr);
302 }
303 else if (return_code == TEST_SKIPPED)
304 { }
305 else if (return_code == TEST_FAILURE)
306 {
307 Error << " item.flush(failure)";
308 signal.set_shutdown(SHUTDOWN_GRACEFUL);
309 }
310 }
311 else if (return_code == TEST_SKIPPED)
312 { }
313 else if (return_code == TEST_FAILURE)
314 {
315 Error << " item.startup(failure)";
316 signal.set_shutdown(SHUTDOWN_GRACEFUL);
317 }
318
319 stats.total++;
320
321 switch (return_code)
322 {
323 case TEST_SUCCESS:
324 Out << "\tTesting " << run->name << "\t\t\t\t\t" << load_time / 1000 << "." << load_time % 1000 << "[ " << test_strerror(return_code) << " ]";
325 stats.success++;
326 break;
327
328 case TEST_FATAL:
329 case TEST_FAILURE:
330 stats.failed++;
331 failed= true;
332 Out << "\tTesting " << run->name << "\t\t\t\t\t" << "[ " << test_strerror(return_code) << " ]";
333 break;
334
335 case TEST_SKIPPED:
336 stats.skipped++;
337 skipped= true;
338 Out << "\tTesting " << run->name << "\t\t\t\t\t" << "[ " << test_strerror(return_code) << " ]";
339 break;
340
341 case TEST_MEMORY_ALLOCATION_FAILURE:
342 test_assert(0, "Memory Allocation Error");
343 }
344
345 if (test_failed(world->on_error(return_code, creators_ptr)))
346 {
347 Error << "Failed while running on_error()";
348 signal.set_shutdown(SHUTDOWN_GRACEFUL);
349 break;
350 }
351 }
352
353 (void) world->runner()->post(next->post, creators_ptr);
354
355 cleanup:
356 if (failed == false and skipped == false)
357 {
358 stats.collection_success++;
359 }
360
361 if (failed)
362 {
363 stats.collection_failed++;
364 }
365
366 if (skipped)
367 {
368 stats.collection_skipped++;
369 }
370
371 world->shutdown(creators_ptr);
372 Outn();
373 }
374
375 if (not signal.is_shutdown())
376 {
377 signal.set_shutdown(SHUTDOWN_GRACEFUL);
378 }
379
380 int exit_code= EXIT_SUCCESS;
381 shutdown_t status= signal.get_shutdown();
382 if (status == SHUTDOWN_FORCED)
383 {
384 Out << "Tests were aborted.";
385 exit_code= EXIT_FAILURE;
386 }
387 else if (stats.collection_failed)
388 {
389 Out << "Some test failed.";
390 exit_code= EXIT_FAILURE;
391 }
392 else if (stats.collection_skipped and stats.collection_failed and stats.collection_success)
393 {
394 Out << "Some tests were skipped.";
395 }
396 else if (stats.collection_success and stats.collection_failed == 0)
397 {
398 Out << "All tests completed successfully.";
399 }
400
401 stats_print(&stats);
402
403 delete world;
404
405 Outn(); // Generate a blank to break up the messages if make check/test has been run
406
407 return exit_code;
408 }