git.m6w6.name Git - m6w6/libmemcached/blob - libtest/main.cc

   1 /*  vim:expandtab:shiftwidth=2:tabstop=2:smarttab:
   2  *
   3  *  Data Differential YATL (i.e. libtest)  library
   4  *
   5  *  Copyright (C) 2012 Data Differential, http://datadifferential.com/
   6  *
   7  *  Redistribution and use in source and binary forms, with or without
   8  *  modification, are permitted provided that the following conditions are
   9  *  met:
  10  *
  11  *      * Redistributions of source code must retain the above copyright
  12  *  notice, this list of conditions and the following disclaimer.
  13  *
  14  *      * Redistributions in binary form must reproduce the above
  15  *  copyright notice, this list of conditions and the following disclaimer
  16  *  in the documentation and/or other materials provided with the
  17  *  distribution.
  18  *
  19  *      * The names of its contributors may not be used to endorse or
  20  *  promote products derived from this software without specific prior
  21  *  written permission.
  22  *
  23  *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  24  *  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  25  *  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  26  *  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  27  *  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  28  *  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  29  *  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  30  *  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  31  *  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  32  *  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  33  *  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  34  *
  35  */
  36
  37 #include <config.h>
  38 #include <libtest/common.h>
  39
  40 #include <cassert>
  41 #include <cstdlib>
  42 #include <cstring>
  43 #include <ctime>
  44 #include <fnmatch.h>
  45 #include <iostream>
  46 #include <memory>
  47 #include <sys/stat.h>
  48 #include <sys/time.h>
  49 #include <sys/types.h>
  50 #include <sys/wait.h>
  51 #include <unistd.h>
  52
  53 #include <signal.h>
  54
  55 #ifndef __INTEL_COMPILER
  56 #pragma GCC diagnostic ignored "-Wold-style-cast"
  57 #endif
  58
  59 using namespace libtest;
  60
  61 static void stats_print(Stats *stats)
  62 {
  63   if (stats->collection_failed == 0 and stats->collection_success == 0)
  64   {
  65     return;
  66   }
  67
  68   Out << "\tTotal Collections\t\t\t\t" << stats->collection_total;
  69   Out << "\tFailed Collections\t\t\t\t" << stats->collection_failed;
  70   Out << "\tSkipped Collections\t\t\t\t" << stats->collection_skipped;
  71   Out << "\tSucceeded Collections\t\t\t\t" << stats->collection_success;
  72   Outn();
  73   Out << "Total\t\t\t\t" << stats->total;
  74   Out << "\tFailed\t\t\t" << stats->failed;
  75   Out << "\tSkipped\t\t\t" << stats->skipped;
  76   Out << "\tSucceeded\t\t" << stats->success;
  77 }
  78
  79 static long int timedif(struct timeval a, struct timeval b)
  80 {
  81   long us, s;
  82
  83   us = (long)(a.tv_usec - b.tv_usec);
  84   us /= 1000;
  85   s = (long)(a.tv_sec - b.tv_sec);
  86   s *= 1000;
  87   return s + us;
  88 }
  89
  90 static test_return_t runner_code(Framework* frame,
  91                                  test_st* run,
  92                                  void* creators_ptr,
  93                                  long int& load_time)
  94 { // Runner Code
  95
  96   struct timeval start_time, end_time;
  97
  98   gettimeofday(&start_time, NULL);
  99   assert(frame->runner());
 100   assert(run->test_fn);
 101
 102   test_return_t return_code;
 103   try
 104   {
 105     return_code= frame->runner()->run(run->test_fn, creators_ptr);
 106   }
 107   // Special case where check for the testing of the exception
 108   // system.
 109   catch (libtest::fatal &e)
 110   {
 111     if (fatal::is_disabled())
 112     {
 113       fatal::increment_disabled_counter();
 114       return_code= TEST_SUCCESS;
 115     }
 116     else
 117     {
 118       throw;
 119     }
 120   }
 121
 122   gettimeofday(&end_time, NULL);
 123   load_time= timedif(end_time, start_time);
 124
 125   return return_code;
 126 }
 127
 128 #include <getopt.h>
 129 #include <unistd.h>
 130
 131 int main(int argc, char *argv[])
 132 {
 133   bool opt_massive= false;
 134   unsigned long int opt_repeat= 1; // Run all tests once
 135   bool opt_quiet= false;
 136   std::string collection_to_run;
 137
 138   // Options parsing
 139   {
 140     enum long_option_t {
 141       OPT_LIBYATL_VERSION,
 142       OPT_LIBYATL_MATCH_COLLECTION,
 143       OPT_LIBYATL_MASSIVE,
 144       OPT_LIBYATL_QUIET,
 145       OPT_LIBYATL_REPEAT
 146     };
 147
 148     static struct option long_options[]=
 149     {
 150       { "version", no_argument, NULL, OPT_LIBYATL_VERSION },
 151       { "quiet", no_argument, NULL, OPT_LIBYATL_QUIET },
 152       { "repeat", no_argument, NULL, OPT_LIBYATL_REPEAT },
 153       { "collection", required_argument, NULL, OPT_LIBYATL_MATCH_COLLECTION },
 154       { "massive", no_argument, NULL, OPT_LIBYATL_MASSIVE },
 155       { 0, 0, 0, 0 }
 156     };
 157
 158     int option_index= 0;
 159     while (1)
 160     {
 161       int option_rv= getopt_long(argc, argv, "", long_options, &option_index);
 162       if (option_rv == -1)
 163       {
 164         break;
 165       }
 166
 167       switch (option_rv)
 168       {
 169       case OPT_LIBYATL_VERSION:
 170         break;
 171
 172       case OPT_LIBYATL_QUIET:
 173         opt_quiet= true;
 174         break;
 175
 176       case OPT_LIBYATL_REPEAT:
 177         opt_repeat= strtoul(optarg, (char **) NULL, 10);
 178         break;
 179
 180       case OPT_LIBYATL_MATCH_COLLECTION:
 181         collection_to_run= optarg;
 182         break;
 183
 184       case OPT_LIBYATL_MASSIVE:
 185         opt_massive= true;
 186         break;
 187
 188       case '?':
 189         /* getopt_long already printed an error message. */
 190         Error << "unknown option to getopt_long()";
 191         exit(EXIT_FAILURE);
 192
 193       default:
 194         break;
 195       }
 196     }
 197   }
 198
 199   srandom((unsigned int)time(NULL));
 200
 201   if (bool(getenv("YATL_REPEAT")) and (strtoul(getenv("YATL_REPEAT"), (char **) NULL, 10) > 1))
 202   {
 203     opt_repeat= strtoul(getenv("YATL_REPEAT"), (char **) NULL, 10);
 204   }
 205
 206   if ((bool(getenv("YATL_QUIET")) and (strcmp(getenv("YATL_QUIET"), "0") == 0)) or opt_quiet)
 207   {
 208     opt_quiet= true;
 209   }
 210   else if (getenv("JENKINS_URL"))
 211   {
 212     if (bool(getenv("YATL_QUIET")) and (strcmp(getenv("YATL_QUIET"), "1") == 0))
 213     { }
 214     else
 215     {
 216       opt_quiet= true;
 217     }
 218   }
 219
 220   if (opt_quiet)
 221   {
 222     close(STDOUT_FILENO);
 223   }
 224
 225   char buffer[1024];
 226   if (getenv("LIBTEST_TMP"))
 227   {
 228     snprintf(buffer, sizeof(buffer), "%s", getenv("LIBTEST_TMP"));
 229   }
 230   else
 231   {
 232     snprintf(buffer, sizeof(buffer), "%s", LIBTEST_TEMP);
 233   }
 234
 235   if (chdir(buffer) == -1)
 236   {
 237     char getcwd_buffer[1024];
 238     char *dir= getcwd(getcwd_buffer, sizeof(getcwd_buffer));
 239
 240     Error << "Unable to chdir() from " << dir << " to " << buffer << " errno:" << strerror(errno);
 241     return EXIT_FAILURE;
 242   }
 243
 244   if (libtest::libtool() == NULL)
 245   {
 246     Error << "Failed to locate libtool";
 247     return EXIT_FAILURE;
 248   }
 249
 250   int exit_code;
 251
 252   try
 253   {
 254     do
 255     {
 256       exit_code= EXIT_SUCCESS;
 257       std::auto_ptr<Framework> frame(new Framework);
 258
 259       fatal_assert(sigignore(SIGPIPE) == 0);
 260
 261       libtest::SignalThread signal;
 262       if (signal.setup() == false)
 263       {
 264         Error << "Failed to setup signals";
 265         return EXIT_FAILURE;
 266       }
 267
 268       Stats stats;
 269
 270       get_world(frame.get());
 271
 272       test_return_t error;
 273       void *creators_ptr= frame->create(error);
 274
 275       switch (error)
 276       {
 277       case TEST_SUCCESS:
 278         break;
 279
 280       case TEST_SKIPPED:
 281         Out << "SKIP " << argv[0];
 282         return EXIT_SUCCESS;
 283
 284       case TEST_FAILURE:
 285         return EXIT_FAILURE;
 286       }
 287
 288       if (getenv("YATL_COLLECTION_TO_RUN"))
 289       {
 290         if (strlen(getenv("YATL_COLLECTION_TO_RUN")))
 291         {
 292           collection_to_run= getenv("YATL_COLLECTION_TO_RUN");
 293         }
 294       }
 295
 296       if (collection_to_run.compare("none") == 0)
 297       {
 298         return EXIT_SUCCESS;
 299       }
 300
 301       if (collection_to_run.empty() == false)
 302       {
 303         Out << "Only testing " <<  collection_to_run;
 304       }
 305
 306       char *wildcard= NULL;
 307       if (argc == 3)
 308       {
 309         wildcard= argv[2];
 310       }
 311
 312       for (collection_st *next= frame->collections; next and next->name and (not signal.is_shutdown()); next++)
 313       {
 314         if (collection_to_run.empty() == false and fnmatch(collection_to_run.c_str(), next->name, 0))
 315         {
 316           continue;
 317         }
 318
 319         stats.collection_total++;
 320
 321         bool failed= false;
 322         bool skipped= false;
 323         test_return_t collection_rc;
 324         if (test_success(collection_rc= frame->runner()->pre(next->pre, creators_ptr)))
 325         {
 326           Out << "Collection: " << next->name;
 327
 328           for (test_st *run= next->tests; run->name; run++)
 329           {
 330             long int load_time= 0;
 331
 332             if (wildcard && fnmatch(wildcard, run->name, 0))
 333             {
 334               continue;
 335             }
 336
 337             test_return_t return_code;
 338             try
 339             {
 340               if (run->requires_flush)
 341               {
 342                 if (test_failed(frame->runner()->flush(creators_ptr)))
 343                 {
 344                   Error << "frame->runner()->flush(creators_ptr)";
 345                   continue;
 346                 }
 347               }
 348
 349               return_code= runner_code(frame.get(), run, creators_ptr, load_time);
 350
 351               if (return_code == TEST_SKIPPED)
 352               { }
 353               else if (return_code == TEST_FAILURE)
 354               {
 355 #if 0
 356                 Error << " frame->runner()->run(failure)";
 357                 signal.set_shutdown(SHUTDOWN_GRACEFUL);
 358 #endif
 359               }
 360
 361             }
 362             catch (libtest::fatal &e)
 363             {
 364               Error << "Fatal exception was thrown: " << e.what();
 365               return_code= TEST_FAILURE;
 366               throw;
 367             }
 368             catch (std::exception &e)
 369             {
 370               Error << "Exception was thrown: " << e.what();
 371               return_code= TEST_FAILURE;
 372               throw;
 373             }
 374             catch (...)
 375             {
 376               Error << "Unknown exception occurred";
 377               return_code= TEST_FAILURE;
 378               throw;
 379             }
 380
 381             stats.total++;
 382
 383             switch (return_code)
 384             {
 385             case TEST_SUCCESS:
 386               Out << "\tTesting " << run->name <<  "\t\t\t\t\t" << load_time / 1000 << "." << load_time % 1000 << "[ " << test_strerror(return_code) << " ]";
 387               stats.success++;
 388               break;
 389
 390             case TEST_FAILURE:
 391               stats.failed++;
 392               failed= true;
 393               Out << "\tTesting " << run->name <<  "\t\t\t\t\t" << "[ " << test_strerror(return_code) << " ]";
 394               break;
 395
 396             case TEST_SKIPPED:
 397               stats.skipped++;
 398               skipped= true;
 399               Out << "\tTesting " << run->name <<  "\t\t\t\t\t" << "[ " << test_strerror(return_code) << " ]";
 400               break;
 401
 402             default:
 403               fatal_message("invalid return code");
 404             }
 405 #if 0
 406             @TODO add code here to allow for a collection to define a method to reset to allow tests to continue.
 407 #endif
 408           }
 409
 410           (void) frame->runner()->post(next->post, creators_ptr);
 411         }
 412         else if (collection_rc == TEST_FAILURE)
 413         {
 414           Out << next->name << " [ failed ]";
 415           failed= true;
 416 #if 0
 417           signal.set_shutdown(SHUTDOWN_GRACEFUL);
 418 #endif
 419         }
 420         else if (collection_rc == TEST_SKIPPED)
 421         {
 422           Out << next->name << " [ skipping ]";
 423           skipped= true;
 424         }
 425
 426         if (failed == false and skipped == false)
 427         {
 428           stats.collection_success++;
 429         }
 430
 431         if (failed)
 432         {
 433           stats.collection_failed++;
 434         }
 435
 436         if (skipped)
 437         {
 438           stats.collection_skipped++;
 439         }
 440
 441         Outn();
 442       }
 443
 444       if (signal.is_shutdown() == false)
 445       {
 446         signal.set_shutdown(SHUTDOWN_GRACEFUL);
 447       }
 448
 449       shutdown_t status= signal.get_shutdown();
 450       if (status == SHUTDOWN_FORCED)
 451       {
 452         Out << "Tests were aborted.";
 453         exit_code= EXIT_FAILURE;
 454       }
 455       else if (stats.collection_failed)
 456       {
 457         Out << "Some test failed.";
 458         exit_code= EXIT_FAILURE;
 459       }
 460       else if (stats.collection_skipped and stats.collection_failed and stats.collection_success)
 461       {
 462         Out << "Some tests were skipped.";
 463       }
 464       else if (stats.collection_success and (stats.collection_failed == 0))
 465       {
 466         Out << "All tests completed successfully.";
 467       }
 468
 469       stats_print(&stats);
 470
 471       Outn(); // Generate a blank to break up the messages if make check/test has been run
 472     } while (exit_code == EXIT_SUCCESS and --opt_repeat);
 473   }
 474   catch (libtest::fatal& e)
 475   {
 476     std::cerr << e.what() << std::endl;
 477   }
 478   catch (libtest::disconnected& e)
 479   {
 480     std::cerr << "Unhandled disconnection occurred:" << e.what() << std::endl;
 481   }
 482   catch (std::exception& e)
 483   {
 484     std::cerr << e.what() << std::endl;
 485   }
 486   catch (...)
 487   {
 488     std::cerr << "Unknown exception halted execution." << std::endl;
 489   }
 490
 491   return exit_code;
 492 }