git.m6w6.name Git - m6w6/libmemcached/blob - libtest/test.cc

   1 /*  vim:expandtab:shiftwidth=2:tabstop=2:smarttab:
   2  *
   3  *  uTest, libtest
   4  *
   5  *  Copyright (C) 2011 Data Differential, http://datadifferential.com/
   6  *  Copyright (C) 2006-2009 Brian Aker
   7  *
   8  *  Redistribution and use in source and binary forms, with or without
   9  *  modification, are permitted provided that the following conditions are
  10  *  met:
  11  *
  12  *      * Redistributions of source code must retain the above copyright
  13  *  notice, this list of conditions and the following disclaimer.
  14  *
  15  *      * Redistributions in binary form must reproduce the above
  16  *  copyright notice, this list of conditions and the following disclaimer
  17  *  in the documentation and/or other materials provided with the
  18  *  distribution.
  19  *
  20  *      * The names of its contributors may not be used to endorse or
  21  *  promote products derived from this software without specific prior
  22  *  written permission.
  23  *
  24  *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  25  *  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  26  *  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  27  *  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  28  *  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  29  *  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  30  *  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  31  *  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  32  *  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  33  *  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  34  *  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  35  *
  36  */
  37
  38 #include <libtest/common.h>
  39
  40 #include <cassert>
  41 #include <cstdlib>
  42 #include <cstring>
  43 #include <sys/time.h>
  44 #include <sys/types.h>
  45 #include <sys/stat.h>
  46 #include <sys/wait.h>
  47 #include <unistd.h>
  48 #include <ctime>
  49 #include <fnmatch.h>
  50 #include <iostream>
  51
  52 #include <signal.h>
  53
  54 #include <libtest/stats.h>
  55 #include <libtest/signal.h>
  56
  57 #ifndef __INTEL_COMPILER
  58 #pragma GCC diagnostic ignored "-Wold-style-cast"
  59 #endif
  60
  61 using namespace libtest;
  62
  63 static in_port_t global_port= 0;
  64 static char global_socket[1024];
  65
  66 in_port_t default_port()
  67 {
  68   return global_port;
  69 }
  70
  71 void set_default_port(in_port_t port)
  72 {
  73   global_port= port;
  74 }
  75
  76 const char *default_socket()
  77 {
  78   assert(global_socket[0]);
  79   return global_socket;
  80 }
  81
  82 bool test_is_local()
  83 {
  84   return (getenv("LIBTEST_LOCAL"));
  85 }
  86
  87 void set_default_socket(const char *socket)
  88 {
  89   if (socket)
  90   {
  91     strncpy(global_socket, socket, strlen(socket));
  92   }
  93 }
  94
  95 static void stats_print(Stats *stats)
  96 {
  97   Out << "\tTotal Collections\t\t\t\t" << stats->collection_total;
  98   Out << "\tFailed Collections\t\t\t\t" << stats->collection_failed;
  99   Out << "\tSkipped Collections\t\t\t\t" << stats->collection_skipped;
 100   Out << "\tSucceeded Collections\t\t\t\t" << stats->collection_success;
 101   Outn();
 102   Out << "Total\t\t\t\t" << stats->total;
 103   Out << "\tFailed\t\t\t" << stats->failed;
 104   Out << "\tSkipped\t\t\t" << stats->skipped;
 105   Out << "\tSucceeded\t\t" << stats->success;
 106 }
 107
 108 static long int timedif(struct timeval a, struct timeval b)
 109 {
 110   long us, s;
 111
 112   us = (long)(a.tv_usec - b.tv_usec);
 113   us /= 1000;
 114   s = (long)(a.tv_sec - b.tv_sec);
 115   s *= 1000;
 116   return s + us;
 117 }
 118
 119 const char *test_strerror(test_return_t code)
 120 {
 121   switch (code) {
 122   case TEST_SUCCESS:
 123     return "ok";
 124
 125   case TEST_FAILURE:
 126     return "failed";
 127
 128   case TEST_MEMORY_ALLOCATION_FAILURE:
 129     return "memory allocation";
 130
 131   case TEST_SKIPPED:
 132     return "skipped";
 133
 134   case TEST_FATAL:
 135     break;
 136   }
 137
 138   return "failed";
 139 }
 140
 141 void create_core(void)
 142 {
 143   if (getenv("LIBMEMCACHED_NO_COREDUMP") == NULL)
 144   {
 145     pid_t pid= fork();
 146
 147     if (pid == 0)
 148     {
 149       abort();
 150     }
 151     else
 152     {
 153       while (waitpid(pid, NULL, 0) != pid) {};
 154     }
 155   }
 156 }
 157
 158 static Framework *world= NULL;
 159 int main(int argc, char *argv[])
 160 {
 161   srandom((unsigned int)time(NULL));
 162
 163   world= new Framework();
 164
 165   if (not world)
 166   {
 167     return EXIT_FAILURE;
 168   }
 169
 170   libtest::SignalThread signal;
 171   if (not signal.setup())
 172   {
 173     return EXIT_FAILURE;
 174   }
 175
 176   Stats stats;
 177
 178   get_world(world);
 179
 180   test_return_t error;
 181   void *creators_ptr= world->create(error);
 182   if (test_failed(error))
 183   {
 184     Error << "create() failed";
 185     delete world;
 186     return EXIT_FAILURE;
 187   }
 188
 189   char *collection_to_run= NULL;
 190   if (argc > 1)
 191   {
 192     collection_to_run= argv[1];
 193   }
 194   else if (getenv("TEST_COLLECTION"))
 195   {
 196     collection_to_run= getenv("TEST_COLLECTION");
 197   }
 198
 199   if (collection_to_run)
 200   {
 201     Out << "Only testing " <<  collection_to_run;
 202   }
 203
 204   char *wildcard= NULL;
 205   if (argc == 3)
 206   {
 207     wildcard= argv[2];
 208   }
 209
 210   for (collection_st *next= world->collections; next->name and (not signal.is_shutdown()); next++)
 211   {
 212     test_return_t collection_rc= TEST_SUCCESS;
 213     bool failed= false;
 214     bool skipped= false;
 215
 216     if (collection_to_run && fnmatch(collection_to_run, next->name, 0))
 217       continue;
 218
 219     stats.collection_total++;
 220
 221     collection_rc= world->startup(creators_ptr);
 222
 223     if (collection_rc == TEST_SUCCESS and next->pre)
 224     {
 225       collection_rc= world->runner()->pre(next->pre, creators_ptr);
 226     }
 227
 228     switch (collection_rc)
 229     {
 230     case TEST_SUCCESS:
 231       break;
 232
 233     case TEST_FATAL:
 234     case TEST_FAILURE:
 235       Out << next->name << " [ failed ]";
 236       failed= true;
 237       signal.set_shutdown(SHUTDOWN_GRACEFUL);
 238       goto cleanup;
 239
 240     case TEST_SKIPPED:
 241       Out << next->name << " [ skipping ]";
 242       skipped= true;
 243       goto cleanup;
 244
 245     case TEST_MEMORY_ALLOCATION_FAILURE:
 246       test_assert(0, "Allocation failure, or unknown return");
 247     }
 248
 249     Out << "Collection: " << next->name;
 250
 251     for (test_st *run= next->tests; run->name; run++)
 252     {
 253       struct timeval start_time, end_time;
 254       long int load_time= 0;
 255
 256       if (wildcard && fnmatch(wildcard, run->name, 0))
 257       {
 258         continue;
 259       }
 260
 261       test_return_t return_code;
 262       if (test_success(return_code= world->item.startup(creators_ptr)))
 263       {
 264         if (test_success(return_code= world->item.flush(creators_ptr, run)))
 265         {
 266           // @note pre will fail is SKIPPED is returned
 267           if (test_success(return_code= world->item.pre(creators_ptr)))
 268           {
 269             { // Runner Code
 270               gettimeofday(&start_time, NULL);
 271               assert(world->runner());
 272               assert(run->test_fn);
 273               return_code= world->runner()->run(run->test_fn, creators_ptr);
 274               gettimeofday(&end_time, NULL);
 275               load_time= timedif(end_time, start_time);
 276             }
 277           }
 278
 279           // @todo do something if post fails
 280           (void)world->item.post(creators_ptr);
 281         }
 282         else if (return_code == TEST_SKIPPED)
 283         { }
 284         else if (return_code == TEST_FAILURE)
 285         {
 286           Error << " item.flush(failure)";
 287           signal.set_shutdown(SHUTDOWN_GRACEFUL);
 288         }
 289       }
 290       else if (return_code == TEST_SKIPPED)
 291       { }
 292       else if (return_code == TEST_FAILURE)
 293       {
 294         Error << " item.startup(failure)";
 295         signal.set_shutdown(SHUTDOWN_GRACEFUL);
 296       }
 297
 298       stats.total++;
 299
 300       switch (return_code)
 301       {
 302       case TEST_SUCCESS:
 303         Out << "\tTesting " << run->name <<  "\t\t\t\t\t" << load_time / 1000 << "." << load_time % 1000 << "[ " << test_strerror(return_code) << " ]";
 304         stats.success++;
 305         break;
 306
 307       case TEST_FATAL:
 308       case TEST_FAILURE:
 309         stats.failed++;
 310         failed= true;
 311         Out << "\tTesting " << run->name <<  "\t\t\t\t\t" << "[ " << test_strerror(return_code) << " ]";
 312         break;
 313
 314       case TEST_SKIPPED:
 315         stats.skipped++;
 316         skipped= true;
 317         Out << "\tTesting " << run->name <<  "\t\t\t\t\t" << "[ " << test_strerror(return_code) << " ]";
 318         break;
 319
 320       case TEST_MEMORY_ALLOCATION_FAILURE:
 321         test_assert(0, "Memory Allocation Error");
 322       }
 323
 324       if (test_failed(world->on_error(return_code, creators_ptr)))
 325       {
 326         Error << "Failed while running on_error()";
 327         signal.set_shutdown(SHUTDOWN_GRACEFUL);
 328         break;
 329       }
 330     }
 331
 332     (void) world->runner()->post(next->post, creators_ptr);
 333
 334 cleanup:
 335     if (failed == false and skipped == false)
 336     {
 337       stats.collection_success++;
 338     }
 339
 340     if (failed)
 341     {
 342       stats.collection_failed++;
 343     }
 344
 345     if (skipped)
 346     {
 347       stats.collection_skipped++;
 348     }
 349
 350     world->shutdown(creators_ptr);
 351     Outn();
 352   }
 353
 354   if (not signal.is_shutdown())
 355   {
 356     signal.set_shutdown(SHUTDOWN_GRACEFUL);
 357   }
 358
 359   int exit_code= EXIT_SUCCESS;
 360   shutdown_t status= signal.get_shutdown();
 361   if (status == SHUTDOWN_FORCED)
 362   {
 363     Out << "Tests were aborted.";
 364     exit_code= EXIT_FAILURE;
 365   }
 366   else if (stats.collection_failed)
 367   {
 368     Out << "Some test failed.";
 369     exit_code= EXIT_FAILURE;
 370   }
 371   else if (stats.collection_skipped)
 372   {
 373     Out << "Some tests were skipped.";
 374   }
 375   else
 376   {
 377     Out << "All tests completed successfully.";
 378   }
 379
 380   stats_print(&stats);
 381
 382   delete world;
 383
 384   Outn(); // Generate a blank to break up the messages if make check/test has been run
 385
 386   return exit_code;
 387 }