63cf9cbc8a7a0256b0af14fd2e51de1187948ea3
[awesomized/libmemcached] / libtest / test.cc
1 /* vim:expandtab:shiftwidth=2:tabstop=2:smarttab:
2 *
3 * uTest, libtest
4 *
5 * Copyright (C) 2011 Data Differential, http://datadifferential.com/
6 * Copyright (C) 2006-2009 Brian Aker
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions are
10 * met:
11 *
12 * * Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 *
15 * * Redistributions in binary form must reproduce the above
16 * copyright notice, this list of conditions and the following disclaimer
17 * in the documentation and/or other materials provided with the
18 * distribution.
19 *
20 * * The names of its contributors may not be used to endorse or
21 * promote products derived from this software without specific prior
22 * written permission.
23 *
24 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
25 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
26 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
27 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
28 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
29 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
30 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
31 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
32 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
33 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
34 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
35 *
36 */
37
38 #include <libtest/common.h>
39
40 #include <cassert>
41 #include <cstdlib>
42 #include <cstring>
43 #include <sys/time.h>
44 #include <sys/types.h>
45 #include <sys/stat.h>
46 #include <sys/wait.h>
47 #include <unistd.h>
48 #include <ctime>
49 #include <fnmatch.h>
50 #include <iostream>
51
52 #include <signal.h>
53
54 #include <libtest/stats.h>
55 #include <libtest/signal.h>
56
57 #ifndef __INTEL_COMPILER
58 #pragma GCC diagnostic ignored "-Wold-style-cast"
59 #endif
60
61 using namespace libtest;
62
63 static in_port_t global_port= 0;
64 static char global_socket[1024];
65
66 in_port_t default_port()
67 {
68 return global_port;
69 }
70
71 void set_default_port(in_port_t port)
72 {
73 global_port= port;
74 }
75
76 const char *default_socket()
77 {
78 assert(global_socket[0]);
79 return global_socket;
80 }
81
82 bool test_is_local()
83 {
84 return (getenv("LIBTEST_LOCAL"));
85 }
86
87 void set_default_socket(const char *socket)
88 {
89 if (socket)
90 {
91 strncpy(global_socket, socket, strlen(socket));
92 }
93 }
94
95 static void stats_print(Stats *stats)
96 {
97 Log << "\tTotal Collections\t\t\t\t" << stats->collection_total;
98 Log << "\tFailed Collections\t\t\t\t" << stats->collection_failed;
99 Log << "\tSkipped Collections\t\t\t\t" << stats->collection_skipped;
100 Log << "\tSucceeded Collections\t\t\t\t" << stats->collection_success;
101 Logn();
102 Log << "Total\t\t\t\t" << stats->total;
103 Log << "\tFailed\t\t\t" << stats->failed;
104 Log << "\tSkipped\t\t\t" << stats->skipped;
105 Log << "\tSucceeded\t\t" << stats->success;
106 }
107
108 static long int timedif(struct timeval a, struct timeval b)
109 {
110 long us, s;
111
112 us = (long)(a.tv_usec - b.tv_usec);
113 us /= 1000;
114 s = (long)(a.tv_sec - b.tv_sec);
115 s *= 1000;
116 return s + us;
117 }
118
119 const char *test_strerror(test_return_t code)
120 {
121 switch (code) {
122 case TEST_SUCCESS:
123 return "ok";
124
125 case TEST_FAILURE:
126 return "failed";
127
128 case TEST_MEMORY_ALLOCATION_FAILURE:
129 return "memory allocation";
130
131 case TEST_SKIPPED:
132 return "skipped";
133
134 case TEST_FATAL:
135 break;
136 }
137
138 return "failed";
139 }
140
141 void create_core(void)
142 {
143 if (getenv("LIBMEMCACHED_NO_COREDUMP") == NULL)
144 {
145 pid_t pid= fork();
146
147 if (pid == 0)
148 {
149 abort();
150 }
151 else
152 {
153 while (waitpid(pid, NULL, 0) != pid) {};
154 }
155 }
156 }
157
158 static Framework *world= NULL;
159 int main(int argc, char *argv[])
160 {
161 srandom((unsigned int)time(NULL));
162
163 world= new Framework();
164
165 if (not world)
166 {
167 return EXIT_FAILURE;
168 }
169
170 setup_signals();
171
172 Stats stats;
173
174 get_world(world);
175
176 test_return_t error;
177 void *creators_ptr= world->create(error);
178 if (test_failed(error))
179 {
180 Error << "create() failed";
181 delete world;
182 return EXIT_FAILURE;
183 }
184
185 char *collection_to_run= NULL;
186 if (argc > 1)
187 {
188 collection_to_run= argv[1];
189 }
190 else if (getenv("TEST_COLLECTION"))
191 {
192 collection_to_run= getenv("TEST_COLLECTION");
193 }
194
195 if (collection_to_run)
196 {
197 Log << "Only testing " << collection_to_run;
198 }
199
200 char *wildcard= NULL;
201 if (argc == 3)
202 {
203 wildcard= argv[2];
204 }
205
206 for (collection_st *next= world->collections; next->name and (not is_shutdown()); next++)
207 {
208 test_return_t collection_rc= TEST_SUCCESS;
209 bool failed= false;
210 bool skipped= false;
211
212 if (collection_to_run && fnmatch(collection_to_run, next->name, 0))
213 continue;
214
215 stats.collection_total++;
216
217 collection_rc= world->startup(creators_ptr);
218
219 if (collection_rc == TEST_SUCCESS and next->pre)
220 {
221 collection_rc= world->runner()->pre(next->pre, creators_ptr);
222 }
223
224 switch (collection_rc)
225 {
226 case TEST_SUCCESS:
227 break;
228
229 case TEST_FATAL:
230 case TEST_FAILURE:
231 Error << next->name << " [ failed ]";
232 failed= true;
233 set_shutdown(SHUTDOWN_GRACEFUL);
234 goto cleanup;
235
236 case TEST_SKIPPED:
237 Log << next->name << " [ skipping ]";
238 skipped= true;
239 goto cleanup;
240
241 case TEST_MEMORY_ALLOCATION_FAILURE:
242 test_assert(0, "Allocation failure, or unknown return");
243 }
244
245 Log << "Collection: " << next->name;
246
247 for (test_st *run= next->tests; run->name; run++)
248 {
249 struct timeval start_time, end_time;
250 long int load_time= 0;
251
252 if (wildcard && fnmatch(wildcard, run->name, 0))
253 {
254 continue;
255 }
256
257 test_return_t return_code;
258 if (test_success(return_code= world->item.startup(creators_ptr)))
259 {
260 if (test_success(return_code= world->item.flush(creators_ptr, run)))
261 {
262 // @note pre will fail is SKIPPED is returned
263 if (test_success(return_code= world->item.pre(creators_ptr)))
264 {
265 { // Runner Code
266 gettimeofday(&start_time, NULL);
267 assert(world->runner());
268 assert(run->test_fn);
269 return_code= world->runner()->run(run->test_fn, creators_ptr);
270 gettimeofday(&end_time, NULL);
271 load_time= timedif(end_time, start_time);
272 }
273 }
274
275 // @todo do something if post fails
276 (void)world->item.post(creators_ptr);
277 }
278 else if (return_code == TEST_SKIPPED)
279 { }
280 else if (return_code == TEST_FAILURE)
281 {
282 Error << " item.flush(failure)";
283 set_shutdown(SHUTDOWN_GRACEFUL);
284 }
285 }
286 else if (return_code == TEST_SKIPPED)
287 { }
288 else if (return_code == TEST_FAILURE)
289 {
290 Error << " item.startup(failure)";
291 set_shutdown(SHUTDOWN_GRACEFUL);
292 }
293
294 stats.total++;
295
296 switch (return_code)
297 {
298 case TEST_SUCCESS:
299 Log << "\tTesting " << run->name << "\t\t\t\t\t" << load_time / 1000 << "." << load_time % 1000 << "[ " << test_strerror(return_code) << " ]";
300 stats.success++;
301 break;
302
303 case TEST_FATAL:
304 case TEST_FAILURE:
305 stats.failed++;
306 failed= true;
307 Log << "\tTesting " << run->name << "\t\t\t\t\t" << "[ " << test_strerror(return_code) << " ]";
308 break;
309
310 case TEST_SKIPPED:
311 stats.skipped++;
312 skipped= true;
313 Log << "\tTesting " << run->name << "\t\t\t\t\t" << "[ " << test_strerror(return_code) << " ]";
314 break;
315
316 case TEST_MEMORY_ALLOCATION_FAILURE:
317 test_assert(0, "Memory Allocation Error");
318 }
319
320 if (test_failed(world->on_error(return_code, creators_ptr)))
321 {
322 Error << "Failed while running on_error()";
323 set_shutdown(SHUTDOWN_GRACEFUL);
324 break;
325 }
326 }
327
328 (void) world->runner()->post(next->post, creators_ptr);
329
330 cleanup:
331 if (failed == false and skipped == false)
332 {
333 stats.collection_success++;
334 }
335
336 if (failed)
337 {
338 stats.collection_failed++;
339 }
340
341 if (skipped)
342 {
343 stats.collection_skipped++;
344 }
345
346 world->shutdown(creators_ptr);
347 Logn();
348 }
349
350 if (not is_shutdown())
351 {
352 set_shutdown(SHUTDOWN_GRACEFUL);
353 }
354
355 int exit_code= EXIT_SUCCESS;
356 shutdown_t status= get_shutdown();
357 if (status == SHUTDOWN_FORCED)
358 {
359 Log << "Tests were aborted.";
360 exit_code= EXIT_FAILURE;
361 }
362 else if (stats.collection_failed)
363 {
364 Log << "Some test failed.";
365 exit_code= EXIT_FAILURE;
366 }
367 else if (stats.collection_skipped)
368 {
369 Log << "Some tests were skipped.";
370 }
371 else
372 {
373 Log << "All tests completed successfully.";
374 }
375
376 stats_print(&stats);
377
378 delete world;
379
380 Logn(); // Generate a blank to break up the messages if make check/test has been run
381
382 return exit_code;
383 }