git.m6w6.name Git - m6w6/libmemcached/blob - libtest/test.cc

   1 /*  vim:expandtab:shiftwidth=2:tabstop=2:smarttab:
   2  *
   3  *  libtest
   4  *
   5  *  Copyright (C) 2011 Data Differential, http://datadifferential.com/
   6  *
   7  *  This library is free software; you can redistribute it and/or
   8  *  modify it under the terms of the GNU Lesser General Public
   9  *  License as published by the Free Software Foundation; either
  10  *  version 3 of the License, or (at your option) any later version.
  11  *
  12  *  This library is distributed in the hope that it will be useful,
  13  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15  *  Lesser General Public License for more details.
  16  *
  17  *  You should have received a copy of the GNU Lesser General Public
  18  *  License along with this library; if not, write to the Free Software
  19  *  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
  20  */
  21
  22 #include <config.h>
  23 #include <libtest/common.h>
  24
  25 #include <cassert>
  26 #include <cstdlib>
  27 #include <cstring>
  28 #include <sys/time.h>
  29 #include <sys/types.h>
  30 #include <sys/stat.h>
  31 #include <sys/wait.h>
  32 #include <unistd.h>
  33 #include <ctime>
  34 #include <fnmatch.h>
  35 #include <iostream>
  36
  37 #include <signal.h>
  38
  39 #ifndef __INTEL_COMPILER
  40 #pragma GCC diagnostic ignored "-Wold-style-cast"
  41 #endif
  42
  43 using namespace libtest;
  44
  45 static void stats_print(Stats *stats)
  46 {
  47   if (stats->collection_failed == 0 and stats->collection_success == 0)
  48   {
  49     return;
  50   }
  51
  52   Out << "\tTotal Collections\t\t\t\t" << stats->collection_total;
  53   Out << "\tFailed Collections\t\t\t\t" << stats->collection_failed;
  54   Out << "\tSkipped Collections\t\t\t\t" << stats->collection_skipped;
  55   Out << "\tSucceeded Collections\t\t\t\t" << stats->collection_success;
  56   Outn();
  57   Out << "Total\t\t\t\t" << stats->total;
  58   Out << "\tFailed\t\t\t" << stats->failed;
  59   Out << "\tSkipped\t\t\t" << stats->skipped;
  60   Out << "\tSucceeded\t\t" << stats->success;
  61 }
  62
  63 static long int timedif(struct timeval a, struct timeval b)
  64 {
  65   long us, s;
  66
  67   us = (long)(a.tv_usec - b.tv_usec);
  68   us /= 1000;
  69   s = (long)(a.tv_sec - b.tv_sec);
  70   s *= 1000;
  71   return s + us;
  72 }
  73
  74 #include <getopt.h>
  75 #include <unistd.h>
  76
  77 int main(int argc, char *argv[])
  78 {
  79   bool opt_repeat= false;
  80   bool opt_quiet= false;
  81   std::string collection_to_run;
  82
  83   // Options parsing
  84   {
  85     enum long_option_t {
  86       OPT_LIBYATL_VERSION,
  87       OPT_LIBYATL_MATCH_COLLECTION,
  88       OPT_LIBYATL_QUIET,
  89       OPT_LIBYATL_REPEAT
  90     };
  91
  92     static struct option long_options[]=
  93     {
  94       { "version", no_argument, NULL, OPT_LIBYATL_VERSION },
  95       { "quiet", no_argument, NULL, OPT_LIBYATL_QUIET },
  96       { "repeat", no_argument, NULL, OPT_LIBYATL_REPEAT },
  97       { "collection", required_argument, NULL, OPT_LIBYATL_MATCH_COLLECTION },
  98       { 0, 0, 0, 0 }
  99     };
 100
 101     int option_index= 0;
 102     while (1)
 103     {
 104       int option_rv= getopt_long(argc, argv, "", long_options, &option_index);
 105       if (option_rv == -1)
 106       {
 107         break;
 108       }
 109
 110       switch (option_rv)
 111       {
 112       case OPT_LIBYATL_VERSION:
 113         break;
 114
 115       case OPT_LIBYATL_QUIET:
 116         opt_quiet= true;
 117         break;
 118
 119       case OPT_LIBYATL_REPEAT:
 120         opt_repeat= true;
 121         break;
 122
 123       case OPT_LIBYATL_MATCH_COLLECTION:
 124         collection_to_run= optarg;
 125         break;
 126
 127       case '?':
 128         /* getopt_long already printed an error message. */
 129         Error << "unknown option to getopt_long()";
 130         exit(EXIT_FAILURE);
 131
 132       default:
 133         break;
 134       }
 135     }
 136   }
 137
 138   srandom((unsigned int)time(NULL));
 139
 140   int repeat;
 141   if (bool(getenv("YATL_REPEAT")) and (repeat= atoi(getenv("YATL_REPEAT"))))
 142   {
 143     opt_repeat= true;
 144   }
 145
 146   if ((getenv("YATL_QUIET") and strcmp(getenv("YATL_QUIET"), "0") == 0) or opt_quiet)
 147   {
 148     opt_quiet= true;
 149   }
 150   else if (getenv("JENKINS_URL"))
 151   {
 152     if (getenv("YATL_QUIET") and strcmp(getenv("YATL_QUIET"), "1") == 0)
 153     { }
 154     else
 155     {
 156       opt_quiet= true;
 157     }
 158   }
 159
 160   if (opt_quiet)
 161   {
 162     close(STDOUT_FILENO);
 163   }
 164
 165   char buffer[1024];
 166   if (getenv("LIBTEST_TMP"))
 167   {
 168     snprintf(buffer, sizeof(buffer), "%s", getenv("LIBTEST_TMP"));
 169   }
 170   else
 171   {
 172     snprintf(buffer, sizeof(buffer), "%s", LIBTEST_TEMP);
 173   }
 174
 175   if (chdir(buffer) == -1)
 176   {
 177     char getcwd_buffer[1024];
 178     char *dir= getcwd(getcwd_buffer, sizeof(getcwd_buffer));
 179
 180     Error << "Unable to chdir() from " << dir << " to " << buffer << " errno:" << strerror(errno);
 181     return EXIT_FAILURE;
 182   }
 183
 184   if (libtest::libtool() == NULL)
 185   {
 186     Error << "Failed to locate libtool";
 187     return EXIT_FAILURE;
 188   }
 189
 190   int exit_code;
 191
 192   try {
 193     do {
 194       exit_code= EXIT_SUCCESS;
 195       Framework world;
 196
 197       fatal_assert(sigignore(SIGPIPE) == 0);
 198
 199       libtest::SignalThread signal;
 200       if (signal.setup() == false)
 201       {
 202         Error << "Failed to setup signals";
 203         return EXIT_FAILURE;
 204       }
 205
 206       Stats stats;
 207
 208       get_world(&world);
 209
 210       test_return_t error;
 211       void *creators_ptr= world.create(error);
 212
 213       switch (error)
 214       {
 215       case TEST_SUCCESS:
 216         break;
 217
 218       case TEST_SKIPPED:
 219         Out << "SKIP " << argv[0];
 220         return EXIT_SUCCESS;
 221
 222       case TEST_FAILURE:
 223         return EXIT_FAILURE;
 224       }
 225
 226       if (getenv("TEST_COLLECTION"))
 227       {
 228         if (strlen(getenv("TEST_COLLECTION")))
 229         {
 230           collection_to_run= getenv("TEST_COLLECTION");
 231         }
 232       }
 233
 234       if (collection_to_run.empty() == false)
 235       {
 236         Out << "Only testing " <<  collection_to_run;
 237       }
 238
 239       char *wildcard= NULL;
 240       if (argc == 3)
 241       {
 242         wildcard= argv[2];
 243       }
 244
 245       for (collection_st *next= world.collections; next and next->name and (not signal.is_shutdown()); next++)
 246       {
 247         bool failed= false;
 248         bool skipped= false;
 249
 250         if (collection_to_run.empty() == false and fnmatch(collection_to_run.c_str(), next->name, 0))
 251         {
 252           continue;
 253         }
 254
 255         stats.collection_total++;
 256
 257         test_return_t collection_rc= world.startup(creators_ptr);
 258
 259         if (collection_rc == TEST_SUCCESS and next->pre)
 260         {
 261           collection_rc= world.runner()->pre(next->pre, creators_ptr);
 262         }
 263
 264         switch (collection_rc)
 265         {
 266         case TEST_SUCCESS:
 267           break;
 268
 269         case TEST_FAILURE:
 270           Out << next->name << " [ failed ]";
 271           failed= true;
 272           signal.set_shutdown(SHUTDOWN_GRACEFUL);
 273           goto cleanup;
 274
 275         case TEST_SKIPPED:
 276           Out << next->name << " [ skipping ]";
 277           skipped= true;
 278           goto cleanup;
 279
 280         default:
 281           throw fatal_message("invalid return code");
 282         }
 283
 284         Out << "Collection: " << next->name;
 285
 286         for (test_st *run= next->tests; run->name; run++)
 287         {
 288           struct timeval start_time, end_time;
 289           long int load_time= 0;
 290
 291           if (wildcard && fnmatch(wildcard, run->name, 0))
 292           {
 293             continue;
 294           }
 295
 296           test_return_t return_code;
 297           try {
 298             if (test_success(return_code= world.item.startup(creators_ptr)))
 299             {
 300               if (test_success(return_code= world.item.flush(creators_ptr, run)))
 301               {
 302                 // @note pre will fail is SKIPPED is returned
 303                 if (test_success(return_code= world.item.pre(creators_ptr)))
 304                 {
 305                   { // Runner Code
 306                     gettimeofday(&start_time, NULL);
 307                     assert(world.runner());
 308                     assert(run->test_fn);
 309                     try
 310                     {
 311                       return_code= world.runner()->run(run->test_fn, creators_ptr);
 312                     }
 313                     // Special case where check for the testing of the exception
 314                     // system.
 315                     catch (libtest::fatal &e)
 316                     {
 317                       if (fatal::is_disabled())
 318                       {
 319                         fatal::increment_disabled_counter();
 320                         return_code= TEST_SUCCESS;
 321                       }
 322                       else
 323                       {
 324                         throw;
 325                       }
 326                     }
 327
 328                     gettimeofday(&end_time, NULL);
 329                     load_time= timedif(end_time, start_time);
 330                   }
 331                 }
 332
 333                 // @todo do something if post fails
 334                 (void)world.item.post(creators_ptr);
 335               }
 336               else if (return_code == TEST_SKIPPED)
 337               { }
 338               else if (return_code == TEST_FAILURE)
 339               {
 340                 Error << " item.flush(failure)";
 341                 signal.set_shutdown(SHUTDOWN_GRACEFUL);
 342               }
 343             }
 344             else if (return_code == TEST_SKIPPED)
 345             { }
 346             else if (return_code == TEST_FAILURE)
 347             {
 348               Error << " item.startup(failure)";
 349               signal.set_shutdown(SHUTDOWN_GRACEFUL);
 350             }
 351           }
 352
 353           catch (libtest::fatal &e)
 354           {
 355             Error << "Fatal exception was thrown: " << e.what();
 356             return_code= TEST_FAILURE;
 357           }
 358           catch (std::exception &e)
 359           {
 360             Error << "Exception was thrown: " << e.what();
 361             return_code= TEST_FAILURE;
 362           }
 363           catch (...)
 364           {
 365             Error << "Unknown exception occurred";
 366             return_code= TEST_FAILURE;
 367           }
 368
 369           stats.total++;
 370
 371           switch (return_code)
 372           {
 373           case TEST_SUCCESS:
 374             Out << "\tTesting " << run->name <<  "\t\t\t\t\t" << load_time / 1000 << "." << load_time % 1000 << "[ " << test_strerror(return_code) << " ]";
 375             stats.success++;
 376             break;
 377
 378           case TEST_FAILURE:
 379             stats.failed++;
 380             failed= true;
 381             Out << "\tTesting " << run->name <<  "\t\t\t\t\t" << "[ " << test_strerror(return_code) << " ]";
 382             break;
 383
 384           case TEST_SKIPPED:
 385             stats.skipped++;
 386             skipped= true;
 387             Out << "\tTesting " << run->name <<  "\t\t\t\t\t" << "[ " << test_strerror(return_code) << " ]";
 388             break;
 389
 390           default:
 391             throw fatal_message("invalid return code");
 392           }
 393
 394           if (test_failed(world.on_error(return_code, creators_ptr)))
 395           {
 396             Error << "Failed while running on_error()";
 397             signal.set_shutdown(SHUTDOWN_GRACEFUL);
 398             break;
 399           }
 400         }
 401
 402         (void) world.runner()->post(next->post, creators_ptr);
 403
 404 cleanup:
 405         if (failed == false and skipped == false)
 406         {
 407           stats.collection_success++;
 408         }
 409
 410         if (failed)
 411         {
 412           stats.collection_failed++;
 413         }
 414
 415         if (skipped)
 416         {
 417           stats.collection_skipped++;
 418         }
 419
 420         world.shutdown(creators_ptr);
 421         Outn();
 422       }
 423
 424       if (not signal.is_shutdown())
 425       {
 426         signal.set_shutdown(SHUTDOWN_GRACEFUL);
 427       }
 428
 429       shutdown_t status= signal.get_shutdown();
 430       if (status == SHUTDOWN_FORCED)
 431       {
 432         Out << "Tests were aborted.";
 433         exit_code= EXIT_FAILURE;
 434       }
 435       else if (stats.collection_failed)
 436       {
 437         Out << "Some test failed.";
 438         exit_code= EXIT_FAILURE;
 439       }
 440       else if (stats.collection_skipped and stats.collection_failed and stats.collection_success)
 441       {
 442         Out << "Some tests were skipped.";
 443       }
 444       else if (stats.collection_success and stats.collection_failed == 0)
 445       {
 446         Out << "All tests completed successfully.";
 447       }
 448
 449       stats_print(&stats);
 450
 451       Outn(); // Generate a blank to break up the messages if make check/test has been run
 452     } while (exit_code == EXIT_SUCCESS and opt_repeat);
 453   }
 454   catch (libtest::fatal& e)
 455   {
 456     std::cerr << e.what() << std::endl;
 457   }
 458   catch (std::exception& e)
 459   {
 460     std::cerr << e.what() << std::endl;
 461   }
 462   catch (...)
 463   {
 464     std::cerr << "Unknown exception halted execution." << std::endl;
 465   }
 466
 467   return exit_code;
 468 }