git.m6w6.name Git - m6w6/libmemcached/blob - libtest/main.cc

   1 /*  vim:expandtab:shiftwidth=2:tabstop=2:smarttab:
   2  *
   3  *  libtest
   4  *
   5  *  Copyright (C) 2011 Data Differential, http://datadifferential.com/
   6  *
   7  *  This library is free software; you can redistribute it and/or
   8  *  modify it under the terms of the GNU Lesser General Public
   9  *  License as published by the Free Software Foundation; either
  10  *  version 3 of the License, or (at your option) any later version.
  11  *
  12  *  This library is distributed in the hope that it will be useful,
  13  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15  *  Lesser General Public License for more details.
  16  *
  17  *  You should have received a copy of the GNU Lesser General Public
  18  *  License along with this library; if not, write to the Free Software
  19  *  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
  20  */
  21
  22 #include <config.h>
  23 #include <libtest/common.h>
  24
  25 #include <cassert>
  26 #include <cstdlib>
  27 #include <cstring>
  28 #include <sys/time.h>
  29 #include <sys/types.h>
  30 #include <sys/stat.h>
  31 #include <sys/wait.h>
  32 #include <unistd.h>
  33 #include <ctime>
  34 #include <fnmatch.h>
  35 #include <iostream>
  36
  37 #include <signal.h>
  38
  39 #ifndef __INTEL_COMPILER
  40 #pragma GCC diagnostic ignored "-Wold-style-cast"
  41 #endif
  42
  43 using namespace libtest;
  44
  45 static void stats_print(Stats *stats)
  46 {
  47   if (stats->collection_failed == 0 and stats->collection_success == 0)
  48   {
  49     return;
  50   }
  51
  52   Out << "\tTotal Collections\t\t\t\t" << stats->collection_total;
  53   Out << "\tFailed Collections\t\t\t\t" << stats->collection_failed;
  54   Out << "\tSkipped Collections\t\t\t\t" << stats->collection_skipped;
  55   Out << "\tSucceeded Collections\t\t\t\t" << stats->collection_success;
  56   Outn();
  57   Out << "Total\t\t\t\t" << stats->total;
  58   Out << "\tFailed\t\t\t" << stats->failed;
  59   Out << "\tSkipped\t\t\t" << stats->skipped;
  60   Out << "\tSucceeded\t\t" << stats->success;
  61 }
  62
  63 static long int timedif(struct timeval a, struct timeval b)
  64 {
  65   long us, s;
  66
  67   us = (long)(a.tv_usec - b.tv_usec);
  68   us /= 1000;
  69   s = (long)(a.tv_sec - b.tv_sec);
  70   s *= 1000;
  71   return s + us;
  72 }
  73
  74 #include <getopt.h>
  75 #include <unistd.h>
  76
  77 int main(int argc, char *argv[])
  78 {
  79   bool opt_massive= false;
  80   unsigned long int opt_repeat= 1; // Run all tests once
  81   bool opt_quiet= false;
  82   std::string collection_to_run;
  83
  84   // Options parsing
  85   {
  86     enum long_option_t {
  87       OPT_LIBYATL_VERSION,
  88       OPT_LIBYATL_MATCH_COLLECTION,
  89       OPT_LIBYATL_MASSIVE,
  90       OPT_LIBYATL_QUIET,
  91       OPT_LIBYATL_REPEAT
  92     };
  93
  94     static struct option long_options[]=
  95     {
  96       { "version", no_argument, NULL, OPT_LIBYATL_VERSION },
  97       { "quiet", no_argument, NULL, OPT_LIBYATL_QUIET },
  98       { "repeat", no_argument, NULL, OPT_LIBYATL_REPEAT },
  99       { "collection", required_argument, NULL, OPT_LIBYATL_MATCH_COLLECTION },
 100       { "massive", no_argument, NULL, OPT_LIBYATL_MASSIVE },
 101       { 0, 0, 0, 0 }
 102     };
 103
 104     int option_index= 0;
 105     while (1)
 106     {
 107       int option_rv= getopt_long(argc, argv, "", long_options, &option_index);
 108       if (option_rv == -1)
 109       {
 110         break;
 111       }
 112
 113       switch (option_rv)
 114       {
 115       case OPT_LIBYATL_VERSION:
 116         break;
 117
 118       case OPT_LIBYATL_QUIET:
 119         opt_quiet= true;
 120         break;
 121
 122       case OPT_LIBYATL_REPEAT:
 123         opt_repeat= strtoul(optarg, (char **) NULL, 10);
 124         break;
 125
 126       case OPT_LIBYATL_MATCH_COLLECTION:
 127         collection_to_run= optarg;
 128         break;
 129
 130       case OPT_LIBYATL_MASSIVE:
 131         opt_massive= true;
 132         break;
 133
 134       case '?':
 135         /* getopt_long already printed an error message. */
 136         Error << "unknown option to getopt_long()";
 137         exit(EXIT_FAILURE);
 138
 139       default:
 140         break;
 141       }
 142     }
 143   }
 144
 145   srandom((unsigned int)time(NULL));
 146
 147   if (bool(getenv("YATL_REPEAT")) and (strtoul(getenv("YATL_REPEAT"), (char **) NULL, 10) > 1))
 148   {
 149     opt_repeat= strtoul(getenv("YATL_REPEAT"), (char **) NULL, 10);
 150   }
 151
 152   if ((bool(getenv("YATL_QUIET")) and (strcmp(getenv("YATL_QUIET"), "0") == 0)) or opt_quiet)
 153   {
 154     opt_quiet= true;
 155   }
 156   else if (getenv("JENKINS_URL"))
 157   {
 158     if (bool(getenv("YATL_QUIET")) and (strcmp(getenv("YATL_QUIET"), "1") == 0))
 159     { }
 160     else
 161     {
 162       opt_quiet= true;
 163     }
 164   }
 165
 166   if (opt_quiet)
 167   {
 168     close(STDOUT_FILENO);
 169   }
 170
 171   char buffer[1024];
 172   if (getenv("LIBTEST_TMP"))
 173   {
 174     snprintf(buffer, sizeof(buffer), "%s", getenv("LIBTEST_TMP"));
 175   }
 176   else
 177   {
 178     snprintf(buffer, sizeof(buffer), "%s", LIBTEST_TEMP);
 179   }
 180
 181   if (chdir(buffer) == -1)
 182   {
 183     char getcwd_buffer[1024];
 184     char *dir= getcwd(getcwd_buffer, sizeof(getcwd_buffer));
 185
 186     Error << "Unable to chdir() from " << dir << " to " << buffer << " errno:" << strerror(errno);
 187     return EXIT_FAILURE;
 188   }
 189
 190   if (libtest::libtool() == NULL)
 191   {
 192     Error << "Failed to locate libtool";
 193     return EXIT_FAILURE;
 194   }
 195
 196   int exit_code;
 197
 198   try {
 199     do {
 200       exit_code= EXIT_SUCCESS;
 201       Framework world;
 202
 203       fatal_assert(sigignore(SIGPIPE) == 0);
 204
 205       libtest::SignalThread signal;
 206       if (signal.setup() == false)
 207       {
 208         Error << "Failed to setup signals";
 209         return EXIT_FAILURE;
 210       }
 211
 212       Stats stats;
 213
 214       get_world(&world);
 215
 216       test_return_t error;
 217       void *creators_ptr= world.create(error);
 218
 219       switch (error)
 220       {
 221       case TEST_SUCCESS:
 222         break;
 223
 224       case TEST_SKIPPED:
 225         Out << "SKIP " << argv[0];
 226         return EXIT_SUCCESS;
 227
 228       case TEST_FAILURE:
 229         return EXIT_FAILURE;
 230       }
 231
 232       if (getenv("YATL_COLLECTION_TO_RUN"))
 233       {
 234         if (strlen(getenv("YATL_COLLECTION_TO_RUN")))
 235         {
 236           collection_to_run= getenv("YATL_COLLECTION_TO_RUN");
 237         }
 238       }
 239
 240       if (collection_to_run.compare("none") == 0)
 241       {
 242         return EXIT_SUCCESS;
 243       }
 244
 245       if (collection_to_run.empty() == false)
 246       {
 247         Out << "Only testing " <<  collection_to_run;
 248       }
 249
 250       char *wildcard= NULL;
 251       if (argc == 3)
 252       {
 253         wildcard= argv[2];
 254       }
 255
 256       for (collection_st *next= world.collections; next and next->name and (not signal.is_shutdown()); next++)
 257       {
 258         bool failed= false;
 259         bool skipped= false;
 260
 261         if (collection_to_run.empty() == false and fnmatch(collection_to_run.c_str(), next->name, 0))
 262         {
 263           continue;
 264         }
 265
 266         stats.collection_total++;
 267
 268         test_return_t collection_rc= world.startup(creators_ptr);
 269
 270         if (collection_rc == TEST_SUCCESS and next->pre)
 271         {
 272           collection_rc= world.runner()->pre(next->pre, creators_ptr);
 273         }
 274
 275         switch (collection_rc)
 276         {
 277         case TEST_SUCCESS:
 278           break;
 279
 280         case TEST_FAILURE:
 281           Out << next->name << " [ failed ]";
 282           failed= true;
 283           signal.set_shutdown(SHUTDOWN_GRACEFUL);
 284           goto cleanup;
 285
 286         case TEST_SKIPPED:
 287           Out << next->name << " [ skipping ]";
 288           skipped= true;
 289           goto cleanup;
 290
 291         default:
 292           fatal_message("invalid return code");
 293         }
 294
 295         Out << "Collection: " << next->name;
 296
 297         for (test_st *run= next->tests; run->name; run++)
 298         {
 299           struct timeval start_time, end_time;
 300           long int load_time= 0;
 301
 302           if (wildcard && fnmatch(wildcard, run->name, 0))
 303           {
 304             continue;
 305           }
 306
 307           test_return_t return_code;
 308           try {
 309             if (test_success(return_code= world.item.startup(creators_ptr)))
 310             {
 311               if (test_success(return_code= world.item.flush(creators_ptr, run)))
 312               {
 313                 // @note pre will fail is SKIPPED is returned
 314                 if (test_success(return_code= world.item.pre(creators_ptr)))
 315                 {
 316                   { // Runner Code
 317                     gettimeofday(&start_time, NULL);
 318                     assert(world.runner());
 319                     assert(run->test_fn);
 320                     try
 321                     {
 322                       return_code= world.runner()->run(run->test_fn, creators_ptr);
 323                     }
 324                     // Special case where check for the testing of the exception
 325                     // system.
 326                     catch (libtest::fatal &e)
 327                     {
 328                       if (fatal::is_disabled())
 329                       {
 330                         fatal::increment_disabled_counter();
 331                         return_code= TEST_SUCCESS;
 332                       }
 333                       else
 334                       {
 335                         throw;
 336                       }
 337                     }
 338
 339                     gettimeofday(&end_time, NULL);
 340                     load_time= timedif(end_time, start_time);
 341                   }
 342                 }
 343
 344                 // @todo do something if post fails
 345                 (void)world.item.post(creators_ptr);
 346               }
 347               else if (return_code == TEST_SKIPPED)
 348               { }
 349               else if (return_code == TEST_FAILURE)
 350               {
 351                 Error << " item.flush(failure)";
 352                 signal.set_shutdown(SHUTDOWN_GRACEFUL);
 353               }
 354             }
 355             else if (return_code == TEST_SKIPPED)
 356             { }
 357             else if (return_code == TEST_FAILURE)
 358             {
 359               Error << " item.startup(failure)";
 360               signal.set_shutdown(SHUTDOWN_GRACEFUL);
 361             }
 362           }
 363
 364           catch (libtest::fatal &e)
 365           {
 366             Error << "Fatal exception was thrown: " << e.what();
 367             return_code= TEST_FAILURE;
 368           }
 369           catch (std::exception &e)
 370           {
 371             Error << "Exception was thrown: " << e.what();
 372             return_code= TEST_FAILURE;
 373           }
 374           catch (...)
 375           {
 376             Error << "Unknown exception occurred";
 377             return_code= TEST_FAILURE;
 378           }
 379
 380           stats.total++;
 381
 382           switch (return_code)
 383           {
 384           case TEST_SUCCESS:
 385             Out << "\tTesting " << run->name <<  "\t\t\t\t\t" << load_time / 1000 << "." << load_time % 1000 << "[ " << test_strerror(return_code) << " ]";
 386             stats.success++;
 387             break;
 388
 389           case TEST_FAILURE:
 390             stats.failed++;
 391             failed= true;
 392             Out << "\tTesting " << run->name <<  "\t\t\t\t\t" << "[ " << test_strerror(return_code) << " ]";
 393             break;
 394
 395           case TEST_SKIPPED:
 396             stats.skipped++;
 397             skipped= true;
 398             Out << "\tTesting " << run->name <<  "\t\t\t\t\t" << "[ " << test_strerror(return_code) << " ]";
 399             break;
 400
 401           default:
 402             fatal_message("invalid return code");
 403           }
 404
 405           if (test_failed(world.on_error(return_code, creators_ptr)))
 406           {
 407             Error << "Failed while running on_error()";
 408             signal.set_shutdown(SHUTDOWN_GRACEFUL);
 409             break;
 410           }
 411         }
 412
 413         (void) world.runner()->post(next->post, creators_ptr);
 414
 415 cleanup:
 416         if (failed == false and skipped == false)
 417         {
 418           stats.collection_success++;
 419         }
 420
 421         if (failed)
 422         {
 423           stats.collection_failed++;
 424         }
 425
 426         if (skipped)
 427         {
 428           stats.collection_skipped++;
 429         }
 430
 431         world.shutdown(creators_ptr);
 432         Outn();
 433       }
 434
 435       if (not signal.is_shutdown())
 436       {
 437         signal.set_shutdown(SHUTDOWN_GRACEFUL);
 438       }
 439
 440       shutdown_t status= signal.get_shutdown();
 441       if (status == SHUTDOWN_FORCED)
 442       {
 443         Out << "Tests were aborted.";
 444         exit_code= EXIT_FAILURE;
 445       }
 446       else if (stats.collection_failed)
 447       {
 448         Out << "Some test failed.";
 449         exit_code= EXIT_FAILURE;
 450       }
 451       else if (stats.collection_skipped and stats.collection_failed and stats.collection_success)
 452       {
 453         Out << "Some tests were skipped.";
 454       }
 455       else if (stats.collection_success and stats.collection_failed == 0)
 456       {
 457         Out << "All tests completed successfully.";
 458       }
 459
 460       stats_print(&stats);
 461
 462       Outn(); // Generate a blank to break up the messages if make check/test has been run
 463     } while (exit_code == EXIT_SUCCESS and --opt_repeat);
 464   }
 465   catch (libtest::fatal& e)
 466   {
 467     std::cerr << e.what() << std::endl;
 468   }
 469   catch (libtest::disconnected& e)
 470   {
 471     std::cerr << "Unhandled disconnection occurred:" << e.what() << std::endl;
 472   }
 473   catch (std::exception& e)
 474   {
 475     std::cerr << e.what() << std::endl;
 476   }
 477   catch (...)
 478   {
 479     std::cerr << "Unknown exception halted execution." << std::endl;
 480   }
 481
 482   return exit_code;
 483 }