git.m6w6.name Git - m6w6/libmemcached/blob - libtest/main.cc

   1 /*  vim:expandtab:shiftwidth=2:tabstop=2:smarttab:
   2  *
   3  *  libtest
   4  *
   5  *  Copyright (C) 2011 Data Differential, http://datadifferential.com/
   6  *
   7  *  This library is free software; you can redistribute it and/or
   8  *  modify it under the terms of the GNU Lesser General Public
   9  *  License as published by the Free Software Foundation; either
  10  *  version 3 of the License, or (at your option) any later version.
  11  *
  12  *  This library is distributed in the hope that it will be useful,
  13  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15  *  Lesser General Public License for more details.
  16  *
  17  *  You should have received a copy of the GNU Lesser General Public
  18  *  License along with this library; if not, write to the Free Software
  19  *  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
  20  */
  21
  22 #include <config.h>
  23 #include <libtest/common.h>
  24
  25 #include <cassert>
  26 #include <cstdlib>
  27 #include <cstring>
  28 #include <ctime>
  29 #include <fnmatch.h>
  30 #include <iostream>
  31 #include <memory>
  32 #include <sys/stat.h>
  33 #include <sys/time.h>
  34 #include <sys/types.h>
  35 #include <sys/wait.h>
  36 #include <unistd.h>
  37
  38 #include <signal.h>
  39
  40 #ifndef __INTEL_COMPILER
  41 #pragma GCC diagnostic ignored "-Wold-style-cast"
  42 #endif
  43
  44 using namespace libtest;
  45
  46 static void stats_print(Stats *stats)
  47 {
  48   if (stats->collection_failed == 0 and stats->collection_success == 0)
  49   {
  50     return;
  51   }
  52
  53   Out << "\tTotal Collections\t\t\t\t" << stats->collection_total;
  54   Out << "\tFailed Collections\t\t\t\t" << stats->collection_failed;
  55   Out << "\tSkipped Collections\t\t\t\t" << stats->collection_skipped;
  56   Out << "\tSucceeded Collections\t\t\t\t" << stats->collection_success;
  57   Outn();
  58   Out << "Total\t\t\t\t" << stats->total;
  59   Out << "\tFailed\t\t\t" << stats->failed;
  60   Out << "\tSkipped\t\t\t" << stats->skipped;
  61   Out << "\tSucceeded\t\t" << stats->success;
  62 }
  63
  64 static long int timedif(struct timeval a, struct timeval b)
  65 {
  66   long us, s;
  67
  68   us = (long)(a.tv_usec - b.tv_usec);
  69   us /= 1000;
  70   s = (long)(a.tv_sec - b.tv_sec);
  71   s *= 1000;
  72   return s + us;
  73 }
  74
  75 static test_return_t runner_code(Framework* frame,
  76                                  test_st* run,
  77                                  void* creators_ptr,
  78                                  long int& load_time)
  79 { // Runner Code
  80
  81   struct timeval start_time, end_time;
  82
  83   gettimeofday(&start_time, NULL);
  84   assert(frame->runner());
  85   assert(run->test_fn);
  86
  87   test_return_t return_code;
  88   try
  89   {
  90     return_code= frame->runner()->run(run->test_fn, creators_ptr);
  91   }
  92   // Special case where check for the testing of the exception
  93   // system.
  94   catch (libtest::fatal &e)
  95   {
  96     if (fatal::is_disabled())
  97     {
  98       fatal::increment_disabled_counter();
  99       return_code= TEST_SUCCESS;
 100     }
 101     else
 102     {
 103       throw;
 104     }
 105   }
 106
 107   gettimeofday(&end_time, NULL);
 108   load_time= timedif(end_time, start_time);
 109
 110   return return_code;
 111 }
 112
 113 #include <getopt.h>
 114 #include <unistd.h>
 115
 116 int main(int argc, char *argv[])
 117 {
 118   bool opt_massive= false;
 119   unsigned long int opt_repeat= 1; // Run all tests once
 120   bool opt_quiet= false;
 121   std::string collection_to_run;
 122
 123   // Options parsing
 124   {
 125     enum long_option_t {
 126       OPT_LIBYATL_VERSION,
 127       OPT_LIBYATL_MATCH_COLLECTION,
 128       OPT_LIBYATL_MASSIVE,
 129       OPT_LIBYATL_QUIET,
 130       OPT_LIBYATL_REPEAT
 131     };
 132
 133     static struct option long_options[]=
 134     {
 135       { "version", no_argument, NULL, OPT_LIBYATL_VERSION },
 136       { "quiet", no_argument, NULL, OPT_LIBYATL_QUIET },
 137       { "repeat", no_argument, NULL, OPT_LIBYATL_REPEAT },
 138       { "collection", required_argument, NULL, OPT_LIBYATL_MATCH_COLLECTION },
 139       { "massive", no_argument, NULL, OPT_LIBYATL_MASSIVE },
 140       { 0, 0, 0, 0 }
 141     };
 142
 143     int option_index= 0;
 144     while (1)
 145     {
 146       int option_rv= getopt_long(argc, argv, "", long_options, &option_index);
 147       if (option_rv == -1)
 148       {
 149         break;
 150       }
 151
 152       switch (option_rv)
 153       {
 154       case OPT_LIBYATL_VERSION:
 155         break;
 156
 157       case OPT_LIBYATL_QUIET:
 158         opt_quiet= true;
 159         break;
 160
 161       case OPT_LIBYATL_REPEAT:
 162         opt_repeat= strtoul(optarg, (char **) NULL, 10);
 163         break;
 164
 165       case OPT_LIBYATL_MATCH_COLLECTION:
 166         collection_to_run= optarg;
 167         break;
 168
 169       case OPT_LIBYATL_MASSIVE:
 170         opt_massive= true;
 171         break;
 172
 173       case '?':
 174         /* getopt_long already printed an error message. */
 175         Error << "unknown option to getopt_long()";
 176         exit(EXIT_FAILURE);
 177
 178       default:
 179         break;
 180       }
 181     }
 182   }
 183
 184   srandom((unsigned int)time(NULL));
 185
 186   if (bool(getenv("YATL_REPEAT")) and (strtoul(getenv("YATL_REPEAT"), (char **) NULL, 10) > 1))
 187   {
 188     opt_repeat= strtoul(getenv("YATL_REPEAT"), (char **) NULL, 10);
 189   }
 190
 191   if ((bool(getenv("YATL_QUIET")) and (strcmp(getenv("YATL_QUIET"), "0") == 0)) or opt_quiet)
 192   {
 193     opt_quiet= true;
 194   }
 195   else if (getenv("JENKINS_URL"))
 196   {
 197     if (bool(getenv("YATL_QUIET")) and (strcmp(getenv("YATL_QUIET"), "1") == 0))
 198     { }
 199     else
 200     {
 201       opt_quiet= true;
 202     }
 203   }
 204
 205   if (opt_quiet)
 206   {
 207     close(STDOUT_FILENO);
 208   }
 209
 210   char buffer[1024];
 211   if (getenv("LIBTEST_TMP"))
 212   {
 213     snprintf(buffer, sizeof(buffer), "%s", getenv("LIBTEST_TMP"));
 214   }
 215   else
 216   {
 217     snprintf(buffer, sizeof(buffer), "%s", LIBTEST_TEMP);
 218   }
 219
 220   if (chdir(buffer) == -1)
 221   {
 222     char getcwd_buffer[1024];
 223     char *dir= getcwd(getcwd_buffer, sizeof(getcwd_buffer));
 224
 225     Error << "Unable to chdir() from " << dir << " to " << buffer << " errno:" << strerror(errno);
 226     return EXIT_FAILURE;
 227   }
 228
 229   if (libtest::libtool() == NULL)
 230   {
 231     Error << "Failed to locate libtool";
 232     return EXIT_FAILURE;
 233   }
 234
 235   int exit_code;
 236
 237   try
 238   {
 239     do
 240     {
 241       exit_code= EXIT_SUCCESS;
 242       std::auto_ptr<Framework> frame(new Framework);
 243
 244       fatal_assert(sigignore(SIGPIPE) == 0);
 245
 246       libtest::SignalThread signal;
 247       if (signal.setup() == false)
 248       {
 249         Error << "Failed to setup signals";
 250         return EXIT_FAILURE;
 251       }
 252
 253       Stats stats;
 254
 255       get_world(frame.get());
 256
 257       test_return_t error;
 258       void *creators_ptr= frame->create(error);
 259
 260       switch (error)
 261       {
 262       case TEST_SUCCESS:
 263         break;
 264
 265       case TEST_SKIPPED:
 266         Out << "SKIP " << argv[0];
 267         return EXIT_SUCCESS;
 268
 269       case TEST_FAILURE:
 270         return EXIT_FAILURE;
 271       }
 272
 273       if (getenv("YATL_COLLECTION_TO_RUN"))
 274       {
 275         if (strlen(getenv("YATL_COLLECTION_TO_RUN")))
 276         {
 277           collection_to_run= getenv("YATL_COLLECTION_TO_RUN");
 278         }
 279       }
 280
 281       if (collection_to_run.compare("none") == 0)
 282       {
 283         return EXIT_SUCCESS;
 284       }
 285
 286       if (collection_to_run.empty() == false)
 287       {
 288         Out << "Only testing " <<  collection_to_run;
 289       }
 290
 291       char *wildcard= NULL;
 292       if (argc == 3)
 293       {
 294         wildcard= argv[2];
 295       }
 296
 297       for (collection_st *next= frame->collections; next and next->name and (not signal.is_shutdown()); next++)
 298       {
 299         if (collection_to_run.empty() == false and fnmatch(collection_to_run.c_str(), next->name, 0))
 300         {
 301           continue;
 302         }
 303
 304         stats.collection_total++;
 305
 306         bool failed= false;
 307         bool skipped= false;
 308         test_return_t collection_rc;
 309         if (test_success(collection_rc= frame->runner()->pre(next->pre, creators_ptr)))
 310         {
 311           Out << "Collection: " << next->name;
 312
 313           for (test_st *run= next->tests; run->name; run++)
 314           {
 315             long int load_time= 0;
 316
 317             if (wildcard && fnmatch(wildcard, run->name, 0))
 318             {
 319               continue;
 320             }
 321
 322             test_return_t return_code;
 323             try
 324             {
 325               if (run->requires_flush)
 326               {
 327                 if (test_failed(frame->runner()->flush(creators_ptr)))
 328                 {
 329                   Error << "frame->runner()->flush(creators_ptr)";
 330                   continue;
 331                 }
 332               }
 333
 334               return_code= runner_code(frame.get(), run, creators_ptr, load_time);
 335
 336               if (return_code == TEST_SKIPPED)
 337               { }
 338               else if (return_code == TEST_FAILURE)
 339               {
 340 #if 0
 341                 Error << " frame->runner()->run(failure)";
 342                 signal.set_shutdown(SHUTDOWN_GRACEFUL);
 343 #endif
 344               }
 345
 346             }
 347             catch (libtest::fatal &e)
 348             {
 349               Error << "Fatal exception was thrown: " << e.what();
 350               return_code= TEST_FAILURE;
 351               throw;
 352             }
 353             catch (std::exception &e)
 354             {
 355               Error << "Exception was thrown: " << e.what();
 356               return_code= TEST_FAILURE;
 357               throw;
 358             }
 359             catch (...)
 360             {
 361               Error << "Unknown exception occurred";
 362               return_code= TEST_FAILURE;
 363               throw;
 364             }
 365
 366             stats.total++;
 367
 368             switch (return_code)
 369             {
 370             case TEST_SUCCESS:
 371               Out << "\tTesting " << run->name <<  "\t\t\t\t\t" << load_time / 1000 << "." << load_time % 1000 << "[ " << test_strerror(return_code) << " ]";
 372               stats.success++;
 373               break;
 374
 375             case TEST_FAILURE:
 376               stats.failed++;
 377               failed= true;
 378               Out << "\tTesting " << run->name <<  "\t\t\t\t\t" << "[ " << test_strerror(return_code) << " ]";
 379               break;
 380
 381             case TEST_SKIPPED:
 382               stats.skipped++;
 383               skipped= true;
 384               Out << "\tTesting " << run->name <<  "\t\t\t\t\t" << "[ " << test_strerror(return_code) << " ]";
 385               break;
 386
 387             default:
 388               fatal_message("invalid return code");
 389             }
 390 #if 0
 391             @TODO add code here to allow for a collection to define a method to reset to allow tests to continue.
 392 #endif
 393           }
 394
 395           (void) frame->runner()->post(next->post, creators_ptr);
 396         }
 397         else if (collection_rc == TEST_FAILURE)
 398         {
 399           Out << next->name << " [ failed ]";
 400           failed= true;
 401 #if 0
 402           signal.set_shutdown(SHUTDOWN_GRACEFUL);
 403 #endif
 404         }
 405         else if (collection_rc == TEST_SKIPPED)
 406         {
 407           Out << next->name << " [ skipping ]";
 408           skipped= true;
 409         }
 410
 411         if (failed == false and skipped == false)
 412         {
 413           stats.collection_success++;
 414         }
 415
 416         if (failed)
 417         {
 418           stats.collection_failed++;
 419         }
 420
 421         if (skipped)
 422         {
 423           stats.collection_skipped++;
 424         }
 425
 426         Outn();
 427       }
 428
 429       if (signal.is_shutdown() == false)
 430       {
 431         signal.set_shutdown(SHUTDOWN_GRACEFUL);
 432       }
 433
 434       shutdown_t status= signal.get_shutdown();
 435       if (status == SHUTDOWN_FORCED)
 436       {
 437         Out << "Tests were aborted.";
 438         exit_code= EXIT_FAILURE;
 439       }
 440       else if (stats.collection_failed)
 441       {
 442         Out << "Some test failed.";
 443         exit_code= EXIT_FAILURE;
 444       }
 445       else if (stats.collection_skipped and stats.collection_failed and stats.collection_success)
 446       {
 447         Out << "Some tests were skipped.";
 448       }
 449       else if (stats.collection_success and (stats.collection_failed == 0))
 450       {
 451         Out << "All tests completed successfully.";
 452       }
 453
 454       stats_print(&stats);
 455
 456       Outn(); // Generate a blank to break up the messages if make check/test has been run
 457     } while (exit_code == EXIT_SUCCESS and --opt_repeat);
 458   }
 459   catch (libtest::fatal& e)
 460   {
 461     std::cerr << e.what() << std::endl;
 462   }
 463   catch (libtest::disconnected& e)
 464   {
 465     std::cerr << "Unhandled disconnection occurred:" << e.what() << std::endl;
 466   }
 467   catch (std::exception& e)
 468   {
 469     std::cerr << e.what() << std::endl;
 470   }
 471   catch (...)
 472   {
 473     std::cerr << "Unknown exception halted execution." << std::endl;
 474   }
 475
 476   return exit_code;
 477 }