80cb8ddd64f30fee70ad949c335ebba24e1ae7bc
[awesomized/libmemcached] / libtest / test.cc
1 /* vim:expandtab:shiftwidth=2:tabstop=2:smarttab:
2 *
3 * libtest
4 *
5 * Copyright (C) 2011 Data Differential, http://datadifferential.com/
6 *
7 * This library is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU Lesser General Public
9 * License as published by the Free Software Foundation; either
10 * version 3 of the License, or (at your option) any later version.
11 *
12 * This library is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Lesser General Public License for more details.
16 *
17 * You should have received a copy of the GNU Lesser General Public
18 * License along with this library; if not, write to the Free Software
19 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20 */
21
22 #include <libtest/common.h>
23
24 #include <cassert>
25 #include <cstdlib>
26 #include <cstring>
27 #include <sys/time.h>
28 #include <sys/types.h>
29 #include <sys/stat.h>
30 #include <sys/wait.h>
31 #include <unistd.h>
32 #include <ctime>
33 #include <fnmatch.h>
34 #include <iostream>
35
36 #include <signal.h>
37
38 #ifndef __INTEL_COMPILER
39 #pragma GCC diagnostic ignored "-Wold-style-cast"
40 #endif
41
42 using namespace libtest;
43
44 static void stats_print(Stats *stats)
45 {
46 if (stats->collection_failed == 0 and stats->collection_success == 0)
47 {
48 return;
49 }
50
51 Out << "\tTotal Collections\t\t\t\t" << stats->collection_total;
52 Out << "\tFailed Collections\t\t\t\t" << stats->collection_failed;
53 Out << "\tSkipped Collections\t\t\t\t" << stats->collection_skipped;
54 Out << "\tSucceeded Collections\t\t\t\t" << stats->collection_success;
55 Outn();
56 Out << "Total\t\t\t\t" << stats->total;
57 Out << "\tFailed\t\t\t" << stats->failed;
58 Out << "\tSkipped\t\t\t" << stats->skipped;
59 Out << "\tSucceeded\t\t" << stats->success;
60 }
61
62 static long int timedif(struct timeval a, struct timeval b)
63 {
64 long us, s;
65
66 us = (long)(a.tv_usec - b.tv_usec);
67 us /= 1000;
68 s = (long)(a.tv_sec - b.tv_sec);
69 s *= 1000;
70 return s + us;
71 }
72
73 static Framework *world= NULL;
74 int main(int argc, char *argv[])
75 {
76 srandom((unsigned int)time(NULL));
77
78 if (getenv("LIBTEST_QUIET"))
79 {
80 close(STDOUT_FILENO);
81 }
82
83 char buffer[1024];
84 if (getenv("LIBTEST_TMP"))
85 {
86 snprintf(buffer, sizeof(buffer), "%s", getenv("LIBTEST_TMP"));
87 }
88 else
89 {
90 snprintf(buffer, sizeof(buffer), "%s", LIBTEST_TEMP);
91 }
92
93 if (chdir(buffer) == -1)
94 {
95 char getcwd_buffer[1024];
96 char *dir= getcwd(getcwd_buffer, sizeof(getcwd_buffer));
97
98 Error << "Unable to chdir() from " << dir << " to " << buffer << " errno:" << strerror(errno);
99 return EXIT_FAILURE;
100 }
101
102 if (libtest::libtool() == NULL)
103 {
104 Error << "Failed to locate libtool";
105 return EXIT_FAILURE;
106 }
107
108 world= new Framework();
109
110 if (world == NULL)
111 {
112 Error << "Failed to create Framework()";
113 return EXIT_FAILURE;
114 }
115
116 libtest::SignalThread signal;
117 if (not signal.setup())
118 {
119 return EXIT_FAILURE;
120 }
121
122 Stats stats;
123
124 get_world(world);
125
126 test_return_t error;
127 void *creators_ptr= world->create(error);
128
129 switch (error)
130 {
131 case TEST_SUCCESS:
132 break;
133
134 case TEST_SKIPPED:
135 Out << "SKIP " << argv[0];
136 delete world;
137 return EXIT_SUCCESS;
138
139 case TEST_FATAL:
140 case TEST_FAILURE:
141 case TEST_MEMORY_ALLOCATION_FAILURE:
142 delete world;
143 return EXIT_FAILURE;
144 }
145
146 char *collection_to_run= NULL;
147 if (argc > 1)
148 {
149 collection_to_run= argv[1];
150 }
151 else if (getenv("TEST_COLLECTION"))
152 {
153 if (strlen(getenv("TEST_COLLECTION")))
154 {
155 collection_to_run= getenv("TEST_COLLECTION");
156 }
157 }
158
159 if (collection_to_run)
160 {
161 Out << "Only testing " << collection_to_run;
162 }
163
164 char *wildcard= NULL;
165 if (argc == 3)
166 {
167 wildcard= argv[2];
168 }
169
170 for (collection_st *next= world->collections; next->name and (not signal.is_shutdown()); next++)
171 {
172 test_return_t collection_rc= TEST_SUCCESS;
173 bool failed= false;
174 bool skipped= false;
175
176 if (collection_to_run && fnmatch(collection_to_run, next->name, 0))
177 continue;
178
179 stats.collection_total++;
180
181 collection_rc= world->startup(creators_ptr);
182
183 if (collection_rc == TEST_SUCCESS and next->pre)
184 {
185 collection_rc= world->runner()->pre(next->pre, creators_ptr);
186 }
187
188 switch (collection_rc)
189 {
190 case TEST_SUCCESS:
191 break;
192
193 case TEST_FATAL:
194 case TEST_FAILURE:
195 Out << next->name << " [ failed ]";
196 failed= true;
197 signal.set_shutdown(SHUTDOWN_GRACEFUL);
198 goto cleanup;
199
200 case TEST_SKIPPED:
201 Out << next->name << " [ skipping ]";
202 skipped= true;
203 goto cleanup;
204
205 case TEST_MEMORY_ALLOCATION_FAILURE:
206 test_assert(0, "Allocation failure, or unknown return");
207 }
208
209 Out << "Collection: " << next->name;
210
211 for (test_st *run= next->tests; run->name; run++)
212 {
213 struct timeval start_time, end_time;
214 long int load_time= 0;
215
216 if (wildcard && fnmatch(wildcard, run->name, 0))
217 {
218 continue;
219 }
220
221 test_return_t return_code;
222 if (test_success(return_code= world->item.startup(creators_ptr)))
223 {
224 if (test_success(return_code= world->item.flush(creators_ptr, run)))
225 {
226 // @note pre will fail is SKIPPED is returned
227 if (test_success(return_code= world->item.pre(creators_ptr)))
228 {
229 { // Runner Code
230 gettimeofday(&start_time, NULL);
231 assert(world->runner());
232 assert(run->test_fn);
233 return_code= world->runner()->run(run->test_fn, creators_ptr);
234 gettimeofday(&end_time, NULL);
235 load_time= timedif(end_time, start_time);
236 }
237 }
238
239 // @todo do something if post fails
240 (void)world->item.post(creators_ptr);
241 }
242 else if (return_code == TEST_SKIPPED)
243 { }
244 else if (return_code == TEST_FAILURE)
245 {
246 Error << " item.flush(failure)";
247 signal.set_shutdown(SHUTDOWN_GRACEFUL);
248 }
249 }
250 else if (return_code == TEST_SKIPPED)
251 { }
252 else if (return_code == TEST_FAILURE)
253 {
254 Error << " item.startup(failure)";
255 signal.set_shutdown(SHUTDOWN_GRACEFUL);
256 }
257
258 stats.total++;
259
260 switch (return_code)
261 {
262 case TEST_SUCCESS:
263 Out << "\tTesting " << run->name << "\t\t\t\t\t" << load_time / 1000 << "." << load_time % 1000 << "[ " << test_strerror(return_code) << " ]";
264 stats.success++;
265 break;
266
267 case TEST_FATAL:
268 case TEST_FAILURE:
269 stats.failed++;
270 failed= true;
271 Out << "\tTesting " << run->name << "\t\t\t\t\t" << "[ " << test_strerror(return_code) << " ]";
272 break;
273
274 case TEST_SKIPPED:
275 stats.skipped++;
276 skipped= true;
277 Out << "\tTesting " << run->name << "\t\t\t\t\t" << "[ " << test_strerror(return_code) << " ]";
278 break;
279
280 case TEST_MEMORY_ALLOCATION_FAILURE:
281 test_assert(0, "Memory Allocation Error");
282 }
283
284 if (test_failed(world->on_error(return_code, creators_ptr)))
285 {
286 Error << "Failed while running on_error()";
287 signal.set_shutdown(SHUTDOWN_GRACEFUL);
288 break;
289 }
290 }
291
292 (void) world->runner()->post(next->post, creators_ptr);
293
294 cleanup:
295 if (failed == false and skipped == false)
296 {
297 stats.collection_success++;
298 }
299
300 if (failed)
301 {
302 stats.collection_failed++;
303 }
304
305 if (skipped)
306 {
307 stats.collection_skipped++;
308 }
309
310 world->shutdown(creators_ptr);
311 Outn();
312 }
313
314 if (not signal.is_shutdown())
315 {
316 signal.set_shutdown(SHUTDOWN_GRACEFUL);
317 }
318
319 int exit_code= EXIT_SUCCESS;
320 shutdown_t status= signal.get_shutdown();
321 if (status == SHUTDOWN_FORCED)
322 {
323 Out << "Tests were aborted.";
324 exit_code= EXIT_FAILURE;
325 }
326 else if (stats.collection_failed)
327 {
328 Out << "Some test failed.";
329 exit_code= EXIT_FAILURE;
330 }
331 else if (stats.collection_skipped and stats.collection_failed and stats.collection_success)
332 {
333 Out << "Some tests were skipped.";
334 }
335 else if (stats.collection_success and stats.collection_failed == 0)
336 {
337 Out << "All tests completed successfully.";
338 }
339
340 stats_print(&stats);
341
342 delete world;
343
344 Outn(); // Generate a blank to break up the messages if make check/test has been run
345
346 return exit_code;
347 }