git.m6w6.name Git - m6w6/libmemcached/blob - libtest/test.cc

   1 /*  vim:expandtab:shiftwidth=2:tabstop=2:smarttab:
   2  *
   3  *  libtest
   4  *
   5  *  Copyright (C) 2011 Data Differential, http://datadifferential.com/
   6  *
   7  *  This library is free software; you can redistribute it and/or
   8  *  modify it under the terms of the GNU Lesser General Public
   9  *  License as published by the Free Software Foundation; either
  10  *  version 3 of the License, or (at your option) any later version.
  11  *
  12  *  This library is distributed in the hope that it will be useful,
  13  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15  *  Lesser General Public License for more details.
  16  *
  17  *  You should have received a copy of the GNU Lesser General Public
  18  *  License along with this library; if not, write to the Free Software
  19  *  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
  20  */
  21
  22 #include <config.h>
  23 #include <libtest/common.h>
  24
  25 #include <cassert>
  26 #include <cstdlib>
  27 #include <cstring>
  28 #include <sys/time.h>
  29 #include <sys/types.h>
  30 #include <sys/stat.h>
  31 #include <sys/wait.h>
  32 #include <unistd.h>
  33 #include <ctime>
  34 #include <fnmatch.h>
  35 #include <iostream>
  36
  37 #include <signal.h>
  38
  39 #ifndef __INTEL_COMPILER
  40 #pragma GCC diagnostic ignored "-Wold-style-cast"
  41 #endif
  42
  43 using namespace libtest;
  44
  45 static void stats_print(Stats *stats)
  46 {
  47   if (stats->collection_failed == 0 and stats->collection_success == 0)
  48   {
  49     return;
  50   }
  51
  52   Out << "\tTotal Collections\t\t\t\t" << stats->collection_total;
  53   Out << "\tFailed Collections\t\t\t\t" << stats->collection_failed;
  54   Out << "\tSkipped Collections\t\t\t\t" << stats->collection_skipped;
  55   Out << "\tSucceeded Collections\t\t\t\t" << stats->collection_success;
  56   Outn();
  57   Out << "Total\t\t\t\t" << stats->total;
  58   Out << "\tFailed\t\t\t" << stats->failed;
  59   Out << "\tSkipped\t\t\t" << stats->skipped;
  60   Out << "\tSucceeded\t\t" << stats->success;
  61 }
  62
  63 static long int timedif(struct timeval a, struct timeval b)
  64 {
  65   long us, s;
  66
  67   us = (long)(a.tv_usec - b.tv_usec);
  68   us /= 1000;
  69   s = (long)(a.tv_sec - b.tv_sec);
  70   s *= 1000;
  71   return s + us;
  72 }
  73
  74 #include <getopt.h>
  75 #include <unistd.h>
  76
  77 int main(int argc, char *argv[])
  78 {
  79   bool opt_repeat= false;
  80   std::string collection_to_run;
  81
  82   // Options parsing
  83   {
  84     enum long_option_t {
  85       OPT_LIBYATL_VERSION,
  86       OPT_LIBYATL_MATCH_COLLECTION,
  87       OPT_LIBYATL_REPEAT
  88     };
  89
  90     static struct option long_options[]=
  91     {
  92       {"repeat", no_argument, NULL, OPT_LIBYATL_REPEAT},
  93       {"collection", required_argument, NULL, OPT_LIBYATL_MATCH_COLLECTION},
  94       {0, 0, 0, 0}
  95     };
  96
  97     int option_index= 0;
  98     while (1)
  99     {
 100       int option_rv= getopt_long(argc, argv, "", long_options, &option_index);
 101       if (option_rv == -1)
 102       {
 103         break;
 104       }
 105
 106       switch (option_rv)
 107       {
 108       case OPT_LIBYATL_VERSION:
 109         break;
 110
 111       case OPT_LIBYATL_REPEAT:
 112         opt_repeat= true;
 113         break;
 114
 115       case OPT_LIBYATL_MATCH_COLLECTION:
 116         collection_to_run= optarg;
 117         break;
 118
 119       case '?':
 120         /* getopt_long already printed an error message. */
 121         Error << "unknown option to getopt_long()";
 122         exit(EXIT_FAILURE);
 123
 124       default:
 125         break;
 126       }
 127     }
 128   }
 129
 130   srandom((unsigned int)time(NULL));
 131
 132   if (getenv("LIBTEST_QUIET") and strcmp(getenv("LIBTEST_QUIET"), "0") == 0)
 133   {
 134     close(STDOUT_FILENO);
 135   }
 136   else if (getenv("JENKINS_URL"))
 137   {
 138     close(STDOUT_FILENO);
 139   }
 140
 141   char buffer[1024];
 142   if (getenv("LIBTEST_TMP"))
 143   {
 144     snprintf(buffer, sizeof(buffer), "%s", getenv("LIBTEST_TMP"));
 145   }
 146   else
 147   {
 148     snprintf(buffer, sizeof(buffer), "%s", LIBTEST_TEMP);
 149   }
 150
 151   if (chdir(buffer) == -1)
 152   {
 153     char getcwd_buffer[1024];
 154     char *dir= getcwd(getcwd_buffer, sizeof(getcwd_buffer));
 155
 156     Error << "Unable to chdir() from " << dir << " to " << buffer << " errno:" << strerror(errno);
 157     return EXIT_FAILURE;
 158   }
 159
 160   if (libtest::libtool() == NULL)
 161   {
 162     Error << "Failed to locate libtool";
 163     return EXIT_FAILURE;
 164   }
 165
 166   int exit_code;
 167
 168   try {
 169     do {
 170       exit_code= EXIT_SUCCESS;
 171       Framework world;
 172
 173       fatal_assert(sigignore(SIGPIPE) == 0);
 174
 175       libtest::SignalThread signal;
 176       if (not signal.setup())
 177       {
 178         Error << "Failed to setup signals";
 179         return EXIT_FAILURE;
 180       }
 181
 182       Stats stats;
 183
 184       get_world(&world);
 185
 186       test_return_t error;
 187       void *creators_ptr= world.create(error);
 188
 189       switch (error)
 190       {
 191       case TEST_SUCCESS:
 192         break;
 193
 194       case TEST_SKIPPED:
 195         Out << "SKIP " << argv[0];
 196         return EXIT_SUCCESS;
 197
 198       case TEST_FAILURE:
 199         return EXIT_FAILURE;
 200       }
 201
 202       if (getenv("TEST_COLLECTION"))
 203       {
 204         if (strlen(getenv("TEST_COLLECTION")))
 205         {
 206           collection_to_run= getenv("TEST_COLLECTION");
 207         }
 208       }
 209
 210       if (collection_to_run.empty() == false)
 211       {
 212         Out << "Only testing " <<  collection_to_run;
 213       }
 214
 215       char *wildcard= NULL;
 216       if (argc == 3)
 217       {
 218         wildcard= argv[2];
 219       }
 220
 221       for (collection_st *next= world.collections; next and next->name and (not signal.is_shutdown()); next++)
 222       {
 223         bool failed= false;
 224         bool skipped= false;
 225
 226         if (collection_to_run.empty() == false and fnmatch(collection_to_run.c_str(), next->name, 0))
 227         {
 228           continue;
 229         }
 230
 231         stats.collection_total++;
 232
 233         test_return_t collection_rc= world.startup(creators_ptr);
 234
 235         if (collection_rc == TEST_SUCCESS and next->pre)
 236         {
 237           collection_rc= world.runner()->pre(next->pre, creators_ptr);
 238         }
 239
 240         switch (collection_rc)
 241         {
 242         case TEST_SUCCESS:
 243           break;
 244
 245         case TEST_FAILURE:
 246           Out << next->name << " [ failed ]";
 247           failed= true;
 248           signal.set_shutdown(SHUTDOWN_GRACEFUL);
 249           goto cleanup;
 250
 251         case TEST_SKIPPED:
 252           Out << next->name << " [ skipping ]";
 253           skipped= true;
 254           goto cleanup;
 255
 256         default:
 257           throw fatal_message("invalid return code");
 258         }
 259
 260         Out << "Collection: " << next->name;
 261
 262         for (test_st *run= next->tests; run->name; run++)
 263         {
 264           struct timeval start_time, end_time;
 265           long int load_time= 0;
 266
 267           if (wildcard && fnmatch(wildcard, run->name, 0))
 268           {
 269             continue;
 270           }
 271
 272           test_return_t return_code;
 273           try {
 274             if (test_success(return_code= world.item.startup(creators_ptr)))
 275             {
 276               if (test_success(return_code= world.item.flush(creators_ptr, run)))
 277               {
 278                 // @note pre will fail is SKIPPED is returned
 279                 if (test_success(return_code= world.item.pre(creators_ptr)))
 280                 {
 281                   { // Runner Code
 282                     gettimeofday(&start_time, NULL);
 283                     assert(world.runner());
 284                     assert(run->test_fn);
 285                     try
 286                     {
 287                       return_code= world.runner()->run(run->test_fn, creators_ptr);
 288                     }
 289                     // Special case where check for the testing of the exception
 290                     // system.
 291                     catch (libtest::fatal &e)
 292                     {
 293                       if (fatal::is_disabled())
 294                       {
 295                         fatal::increment_disabled_counter();
 296                         return_code= TEST_SUCCESS;
 297                       }
 298                       else
 299                       {
 300                         throw;
 301                       }
 302                     }
 303
 304                     gettimeofday(&end_time, NULL);
 305                     load_time= timedif(end_time, start_time);
 306                   }
 307                 }
 308
 309                 // @todo do something if post fails
 310                 (void)world.item.post(creators_ptr);
 311               }
 312               else if (return_code == TEST_SKIPPED)
 313               { }
 314               else if (return_code == TEST_FAILURE)
 315               {
 316                 Error << " item.flush(failure)";
 317                 signal.set_shutdown(SHUTDOWN_GRACEFUL);
 318               }
 319             }
 320             else if (return_code == TEST_SKIPPED)
 321             { }
 322             else if (return_code == TEST_FAILURE)
 323             {
 324               Error << " item.startup(failure)";
 325               signal.set_shutdown(SHUTDOWN_GRACEFUL);
 326             }
 327           }
 328
 329           catch (std::exception &e)
 330           {
 331             Error << "Exception was thrown: " << e.what();
 332             return_code= TEST_FAILURE;
 333           }
 334           catch (...)
 335           {
 336             Error << "Unknown exception occurred";
 337             return_code= TEST_FAILURE;
 338           }
 339
 340           stats.total++;
 341
 342           switch (return_code)
 343           {
 344           case TEST_SUCCESS:
 345             Out << "\tTesting " << run->name <<  "\t\t\t\t\t" << load_time / 1000 << "." << load_time % 1000 << "[ " << test_strerror(return_code) << " ]";
 346             stats.success++;
 347             break;
 348
 349           case TEST_FAILURE:
 350             stats.failed++;
 351             failed= true;
 352             Out << "\tTesting " << run->name <<  "\t\t\t\t\t" << "[ " << test_strerror(return_code) << " ]";
 353             break;
 354
 355           case TEST_SKIPPED:
 356             stats.skipped++;
 357             skipped= true;
 358             Out << "\tTesting " << run->name <<  "\t\t\t\t\t" << "[ " << test_strerror(return_code) << " ]";
 359             break;
 360
 361           default:
 362             throw fatal_message("invalid return code");
 363           }
 364
 365           if (test_failed(world.on_error(return_code, creators_ptr)))
 366           {
 367             Error << "Failed while running on_error()";
 368             signal.set_shutdown(SHUTDOWN_GRACEFUL);
 369             break;
 370           }
 371         }
 372
 373         (void) world.runner()->post(next->post, creators_ptr);
 374
 375 cleanup:
 376         if (failed == false and skipped == false)
 377         {
 378           stats.collection_success++;
 379         }
 380
 381         if (failed)
 382         {
 383           stats.collection_failed++;
 384         }
 385
 386         if (skipped)
 387         {
 388           stats.collection_skipped++;
 389         }
 390
 391         world.shutdown(creators_ptr);
 392         Outn();
 393       }
 394
 395       if (not signal.is_shutdown())
 396       {
 397         signal.set_shutdown(SHUTDOWN_GRACEFUL);
 398       }
 399
 400       shutdown_t status= signal.get_shutdown();
 401       if (status == SHUTDOWN_FORCED)
 402       {
 403         Out << "Tests were aborted.";
 404         exit_code= EXIT_FAILURE;
 405       }
 406       else if (stats.collection_failed)
 407       {
 408         Out << "Some test failed.";
 409         exit_code= EXIT_FAILURE;
 410       }
 411       else if (stats.collection_skipped and stats.collection_failed and stats.collection_success)
 412       {
 413         Out << "Some tests were skipped.";
 414       }
 415       else if (stats.collection_success and stats.collection_failed == 0)
 416       {
 417         Out << "All tests completed successfully.";
 418       }
 419
 420       stats_print(&stats);
 421
 422       Outn(); // Generate a blank to break up the messages if make check/test has been run
 423     } while (exit_code == EXIT_SUCCESS and opt_repeat);
 424   }
 425   catch (libtest::fatal& e)
 426   {
 427     std::cerr << e.what() << std::endl;
 428   }
 429   catch (std::exception& e)
 430   {
 431     std::cerr << e.what() << std::endl;
 432   }
 433   catch (...)
 434   {
 435     std::cerr << "Unknown exception halted execution." << std::endl;
 436   }
 437
 438   return exit_code;
 439 }