d113740fca1b8e25e6fce2e94ba6dfa338b4e79d
[awesomized/libmemcached] / libtest / test.cc
1 /* vim:expandtab:shiftwidth=2:tabstop=2:smarttab:
2 *
3 * uTest, libtest
4 *
5 * Copyright (C) 2011 Data Differential, http://datadifferential.com/
6 * Copyright (C) 2006-2009 Brian Aker
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions are
10 * met:
11 *
12 * * Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 *
15 * * Redistributions in binary form must reproduce the above
16 * copyright notice, this list of conditions and the following disclaimer
17 * in the documentation and/or other materials provided with the
18 * distribution.
19 *
20 * * The names of its contributors may not be used to endorse or
21 * promote products derived from this software without specific prior
22 * written permission.
23 *
24 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
25 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
26 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
27 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
28 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
29 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
30 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
31 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
32 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
33 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
34 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
35 *
36 */
37
38 #include <libtest/common.h>
39
40 #include <cassert>
41 #include <cstdlib>
42 #include <cstring>
43 #include <sys/time.h>
44 #include <sys/types.h>
45 #include <sys/stat.h>
46 #include <sys/wait.h>
47 #include <unistd.h>
48 #include <ctime>
49 #include <fnmatch.h>
50 #include <iostream>
51
52 #include <signal.h>
53
54 #include <libtest/stats.h>
55 #include <libtest/signal.h>
56
57 #ifndef __INTEL_COMPILER
58 #pragma GCC diagnostic ignored "-Wold-style-cast"
59 #endif
60
61 using namespace libtest;
62
63 static in_port_t global_port= 0;
64 static char global_socket[1024];
65
66 in_port_t default_port()
67 {
68 return global_port;
69 }
70
71 void set_default_port(in_port_t port)
72 {
73 global_port= port;
74 }
75
76 const char *default_socket()
77 {
78 assert(global_socket[0]);
79 return global_socket;
80 }
81
82 bool test_is_local()
83 {
84 return (getenv("LIBTEST_LOCAL"));
85 }
86
87 void set_default_socket(const char *socket)
88 {
89 if (socket)
90 {
91 strncpy(global_socket, socket, strlen(socket));
92 }
93 }
94
95 static void stats_print(Stats *stats)
96 {
97 if (stats->collection_failed == 0 and stats->collection_success == 0)
98 {
99 return;
100 }
101
102 Out << "\tTotal Collections\t\t\t\t" << stats->collection_total;
103 Out << "\tFailed Collections\t\t\t\t" << stats->collection_failed;
104 Out << "\tSkipped Collections\t\t\t\t" << stats->collection_skipped;
105 Out << "\tSucceeded Collections\t\t\t\t" << stats->collection_success;
106 Outn();
107 Out << "Total\t\t\t\t" << stats->total;
108 Out << "\tFailed\t\t\t" << stats->failed;
109 Out << "\tSkipped\t\t\t" << stats->skipped;
110 Out << "\tSucceeded\t\t" << stats->success;
111 }
112
113 static long int timedif(struct timeval a, struct timeval b)
114 {
115 long us, s;
116
117 us = (long)(a.tv_usec - b.tv_usec);
118 us /= 1000;
119 s = (long)(a.tv_sec - b.tv_sec);
120 s *= 1000;
121 return s + us;
122 }
123
124 const char *test_strerror(test_return_t code)
125 {
126 switch (code) {
127 case TEST_SUCCESS:
128 return "ok";
129
130 case TEST_FAILURE:
131 return "failed";
132
133 case TEST_MEMORY_ALLOCATION_FAILURE:
134 return "memory allocation";
135
136 case TEST_SKIPPED:
137 return "skipped";
138
139 case TEST_FATAL:
140 break;
141 }
142
143 return "failed";
144 }
145
146 void create_core(void)
147 {
148 if (getenv("LIBMEMCACHED_NO_COREDUMP") == NULL)
149 {
150 pid_t pid= fork();
151
152 if (pid == 0)
153 {
154 abort();
155 }
156 else
157 {
158 while (waitpid(pid, NULL, 0) != pid) {};
159 }
160 }
161 }
162
163 static Framework *world= NULL;
164 int main(int argc, char *argv[])
165 {
166 srandom((unsigned int)time(NULL));
167
168 world= new Framework();
169
170 if (not world)
171 {
172 return EXIT_FAILURE;
173 }
174
175 libtest::SignalThread signal;
176 if (not signal.setup())
177 {
178 return EXIT_FAILURE;
179 }
180
181 Stats stats;
182
183 get_world(world);
184
185 test_return_t error;
186 void *creators_ptr= world->create(error);
187
188 switch (error)
189 {
190 case TEST_SUCCESS:
191 break;
192
193 case TEST_SKIPPED:
194 Out << "SKIP " << argv[0];
195 delete world;
196 return EXIT_SUCCESS;
197
198 case TEST_FATAL:
199 case TEST_FAILURE:
200 case TEST_MEMORY_ALLOCATION_FAILURE:
201 Error << argv[0] << "create() failed";
202 delete world;
203 return EXIT_FAILURE;
204 }
205
206 char *collection_to_run= NULL;
207 if (argc > 1)
208 {
209 collection_to_run= argv[1];
210 }
211 else if (getenv("TEST_COLLECTION"))
212 {
213 collection_to_run= getenv("TEST_COLLECTION");
214 }
215
216 if (collection_to_run)
217 {
218 Out << "Only testing " << collection_to_run;
219 }
220
221 char *wildcard= NULL;
222 if (argc == 3)
223 {
224 wildcard= argv[2];
225 }
226
227 for (collection_st *next= world->collections; next->name and (not signal.is_shutdown()); next++)
228 {
229 test_return_t collection_rc= TEST_SUCCESS;
230 bool failed= false;
231 bool skipped= false;
232
233 if (collection_to_run && fnmatch(collection_to_run, next->name, 0))
234 continue;
235
236 stats.collection_total++;
237
238 collection_rc= world->startup(creators_ptr);
239
240 if (collection_rc == TEST_SUCCESS and next->pre)
241 {
242 collection_rc= world->runner()->pre(next->pre, creators_ptr);
243 }
244
245 switch (collection_rc)
246 {
247 case TEST_SUCCESS:
248 break;
249
250 case TEST_FATAL:
251 case TEST_FAILURE:
252 Out << next->name << " [ failed ]";
253 failed= true;
254 signal.set_shutdown(SHUTDOWN_GRACEFUL);
255 goto cleanup;
256
257 case TEST_SKIPPED:
258 Out << next->name << " [ skipping ]";
259 skipped= true;
260 goto cleanup;
261
262 case TEST_MEMORY_ALLOCATION_FAILURE:
263 test_assert(0, "Allocation failure, or unknown return");
264 }
265
266 Out << "Collection: " << next->name;
267
268 for (test_st *run= next->tests; run->name; run++)
269 {
270 struct timeval start_time, end_time;
271 long int load_time= 0;
272
273 if (wildcard && fnmatch(wildcard, run->name, 0))
274 {
275 continue;
276 }
277
278 test_return_t return_code;
279 if (test_success(return_code= world->item.startup(creators_ptr)))
280 {
281 if (test_success(return_code= world->item.flush(creators_ptr, run)))
282 {
283 // @note pre will fail is SKIPPED is returned
284 if (test_success(return_code= world->item.pre(creators_ptr)))
285 {
286 { // Runner Code
287 gettimeofday(&start_time, NULL);
288 assert(world->runner());
289 assert(run->test_fn);
290 return_code= world->runner()->run(run->test_fn, creators_ptr);
291 gettimeofday(&end_time, NULL);
292 load_time= timedif(end_time, start_time);
293 }
294 }
295
296 // @todo do something if post fails
297 (void)world->item.post(creators_ptr);
298 }
299 else if (return_code == TEST_SKIPPED)
300 { }
301 else if (return_code == TEST_FAILURE)
302 {
303 Error << " item.flush(failure)";
304 signal.set_shutdown(SHUTDOWN_GRACEFUL);
305 }
306 }
307 else if (return_code == TEST_SKIPPED)
308 { }
309 else if (return_code == TEST_FAILURE)
310 {
311 Error << " item.startup(failure)";
312 signal.set_shutdown(SHUTDOWN_GRACEFUL);
313 }
314
315 stats.total++;
316
317 switch (return_code)
318 {
319 case TEST_SUCCESS:
320 Out << "\tTesting " << run->name << "\t\t\t\t\t" << load_time / 1000 << "." << load_time % 1000 << "[ " << test_strerror(return_code) << " ]";
321 stats.success++;
322 break;
323
324 case TEST_FATAL:
325 case TEST_FAILURE:
326 stats.failed++;
327 failed= true;
328 Out << "\tTesting " << run->name << "\t\t\t\t\t" << "[ " << test_strerror(return_code) << " ]";
329 break;
330
331 case TEST_SKIPPED:
332 stats.skipped++;
333 skipped= true;
334 Out << "\tTesting " << run->name << "\t\t\t\t\t" << "[ " << test_strerror(return_code) << " ]";
335 break;
336
337 case TEST_MEMORY_ALLOCATION_FAILURE:
338 test_assert(0, "Memory Allocation Error");
339 }
340
341 if (test_failed(world->on_error(return_code, creators_ptr)))
342 {
343 Error << "Failed while running on_error()";
344 signal.set_shutdown(SHUTDOWN_GRACEFUL);
345 break;
346 }
347 }
348
349 (void) world->runner()->post(next->post, creators_ptr);
350
351 cleanup:
352 if (failed == false and skipped == false)
353 {
354 stats.collection_success++;
355 }
356
357 if (failed)
358 {
359 stats.collection_failed++;
360 }
361
362 if (skipped)
363 {
364 stats.collection_skipped++;
365 }
366
367 world->shutdown(creators_ptr);
368 Outn();
369 }
370
371 if (not signal.is_shutdown())
372 {
373 signal.set_shutdown(SHUTDOWN_GRACEFUL);
374 }
375
376 int exit_code= EXIT_SUCCESS;
377 shutdown_t status= signal.get_shutdown();
378 if (status == SHUTDOWN_FORCED)
379 {
380 Out << "Tests were aborted.";
381 exit_code= EXIT_FAILURE;
382 }
383 else if (stats.collection_failed)
384 {
385 Out << "Some test failed.";
386 exit_code= EXIT_FAILURE;
387 }
388 else if (stats.collection_skipped and stats.collection_failed and stats.collection_success)
389 {
390 Out << "Some tests were skipped.";
391 }
392 else if (stats.collection_success and stats.collection_failed == 0)
393 {
394 Out << "All tests completed successfully.";
395 }
396
397 stats_print(&stats);
398
399 delete world;
400
401 Outn(); // Generate a blank to break up the messages if make check/test has been run
402
403 return exit_code;
404 }