git.m6w6.name Git - m6w6/libmemcached/blob - libtest/test.cc

   1 /*  vim:expandtab:shiftwidth=2:tabstop=2:smarttab:
   2  *
   3  *  libtest
   4  *
   5  *  Copyright (C) 2011 Data Differential, http://datadifferential.com/
   6  *
   7  *  This library is free software; you can redistribute it and/or
   8  *  modify it under the terms of the GNU Lesser General Public
   9  *  License as published by the Free Software Foundation; either
  10  *  version 3 of the License, or (at your option) any later version.
  11  *
  12  *  This library is distributed in the hope that it will be useful,
  13  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15  *  Lesser General Public License for more details.
  16  *
  17  *  You should have received a copy of the GNU Lesser General Public
  18  *  License along with this library; if not, write to the Free Software
  19  *  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
  20  */
  21
  22 #include <config.h>
  23 #include <libtest/common.h>
  24
  25 #include <cassert>
  26 #include <cstdlib>
  27 #include <cstring>
  28 #include <sys/time.h>
  29 #include <sys/types.h>
  30 #include <sys/stat.h>
  31 #include <sys/wait.h>
  32 #include <unistd.h>
  33 #include <ctime>
  34 #include <fnmatch.h>
  35 #include <iostream>
  36
  37 #include <signal.h>
  38
  39 #if defined(HAVE_CURL_CURL_H) && HAVE_CURL_CURL_H
  40 #include <curl/curl.h>
  41 #endif
  42
  43 #ifndef __INTEL_COMPILER
  44 #pragma GCC diagnostic ignored "-Wold-style-cast"
  45 #endif
  46
  47 using namespace libtest;
  48
  49 static void stats_print(Stats *stats)
  50 {
  51   if (stats->collection_failed == 0 and stats->collection_success == 0)
  52   {
  53     return;
  54   }
  55
  56   Out << "\tTotal Collections\t\t\t\t" << stats->collection_total;
  57   Out << "\tFailed Collections\t\t\t\t" << stats->collection_failed;
  58   Out << "\tSkipped Collections\t\t\t\t" << stats->collection_skipped;
  59   Out << "\tSucceeded Collections\t\t\t\t" << stats->collection_success;
  60   Outn();
  61   Out << "Total\t\t\t\t" << stats->total;
  62   Out << "\tFailed\t\t\t" << stats->failed;
  63   Out << "\tSkipped\t\t\t" << stats->skipped;
  64   Out << "\tSucceeded\t\t" << stats->success;
  65 }
  66
  67 static long int timedif(struct timeval a, struct timeval b)
  68 {
  69   long us, s;
  70
  71   us = (long)(a.tv_usec - b.tv_usec);
  72   us /= 1000;
  73   s = (long)(a.tv_sec - b.tv_sec);
  74   s *= 1000;
  75   return s + us;
  76 }
  77
  78 static void cleanup_curl(void)
  79 {
  80 #if defined(HAVE_CURL_CURL_H) && HAVE_CURL_CURL_H
  81   curl_global_cleanup();
  82 #endif
  83 }
  84
  85 #include <getopt.h>
  86 #include <unistd.h>
  87
  88 int main(int argc, char *argv[])
  89 {
  90 #if defined(HAVE_CURL_CURL_H) && HAVE_CURL_CURL_H
  91   if (curl_global_init(CURL_GLOBAL_ALL))
  92   {
  93     Error << "curl_global_init(CURL_GLOBAL_ALL) failed";
  94     return EXIT_FAILURE;
  95   }
  96 #endif
  97
  98   if (atexit(cleanup_curl))
  99   {
 100     Error << "atexit() failed";
 101     return EXIT_FAILURE;
 102   }
 103
 104   bool opt_repeat= false;
 105   std::string collection_to_run;
 106
 107   // Options parsing
 108   {
 109     enum long_option_t {
 110       OPT_LIBYATL_VERSION,
 111       OPT_LIBYATL_MATCH_COLLECTION,
 112       OPT_LIBYATL_REPEAT
 113     };
 114
 115     static struct option long_options[]=
 116     {
 117       {"repeat", no_argument, NULL, OPT_LIBYATL_REPEAT},
 118       {"collection", required_argument, NULL, OPT_LIBYATL_MATCH_COLLECTION},
 119       {0, 0, 0, 0}
 120     };
 121
 122     int option_index= 0;
 123     while (1)
 124     {
 125       int option_rv= getopt_long(argc, argv, "", long_options, &option_index);
 126       if (option_rv == -1)
 127       {
 128         break;
 129       }
 130
 131       switch (option_rv)
 132       {
 133       case OPT_LIBYATL_VERSION:
 134         break;
 135
 136       case OPT_LIBYATL_REPEAT:
 137         opt_repeat= true;
 138         break;
 139
 140       case OPT_LIBYATL_MATCH_COLLECTION:
 141         collection_to_run= optarg;
 142         break;
 143
 144       case '?':
 145         /* getopt_long already printed an error message. */
 146         Error << "unknown option to getopt_long()";
 147         exit(EXIT_FAILURE);
 148
 149       default:
 150         break;
 151       }
 152     }
 153   }
 154
 155   srandom((unsigned int)time(NULL));
 156
 157   if (getenv("LIBTEST_QUIET") and strcmp(getenv("LIBTEST_QUIET"), "0") == 0)
 158   {
 159     close(STDOUT_FILENO);
 160   }
 161   else if (getenv("JENKINS_URL"))
 162   {
 163     close(STDOUT_FILENO);
 164   }
 165
 166   char buffer[1024];
 167   if (getenv("LIBTEST_TMP"))
 168   {
 169     snprintf(buffer, sizeof(buffer), "%s", getenv("LIBTEST_TMP"));
 170   }
 171   else
 172   {
 173     snprintf(buffer, sizeof(buffer), "%s", LIBTEST_TEMP);
 174   }
 175
 176   if (chdir(buffer) == -1)
 177   {
 178     char getcwd_buffer[1024];
 179     char *dir= getcwd(getcwd_buffer, sizeof(getcwd_buffer));
 180
 181     Error << "Unable to chdir() from " << dir << " to " << buffer << " errno:" << strerror(errno);
 182     return EXIT_FAILURE;
 183   }
 184
 185   if (libtest::libtool() == NULL)
 186   {
 187     Error << "Failed to locate libtool";
 188     return EXIT_FAILURE;
 189   }
 190
 191   int exit_code;
 192
 193   try {
 194     do {
 195       exit_code= EXIT_SUCCESS;
 196       Framework world;
 197
 198       fatal_assert(sigignore(SIGPIPE) == 0);
 199
 200       libtest::SignalThread signal;
 201       if (not signal.setup())
 202       {
 203         Error << "Failed to setup signals";
 204         return EXIT_FAILURE;
 205       }
 206
 207       Stats stats;
 208
 209       get_world(&world);
 210
 211       test_return_t error;
 212       void *creators_ptr= world.create(error);
 213
 214       switch (error)
 215       {
 216       case TEST_SUCCESS:
 217         break;
 218
 219       case TEST_SKIPPED:
 220         Out << "SKIP " << argv[0];
 221         return EXIT_SUCCESS;
 222
 223       case TEST_FAILURE:
 224         return EXIT_FAILURE;
 225       }
 226
 227       if (getenv("TEST_COLLECTION"))
 228       {
 229         if (strlen(getenv("TEST_COLLECTION")))
 230         {
 231           collection_to_run= getenv("TEST_COLLECTION");
 232         }
 233       }
 234
 235       if (collection_to_run.empty() == false)
 236       {
 237         Out << "Only testing " <<  collection_to_run;
 238       }
 239
 240       char *wildcard= NULL;
 241       if (argc == 3)
 242       {
 243         wildcard= argv[2];
 244       }
 245
 246       for (collection_st *next= world.collections; next and next->name and (not signal.is_shutdown()); next++)
 247       {
 248         bool failed= false;
 249         bool skipped= false;
 250
 251         if (collection_to_run.empty() == false and fnmatch(collection_to_run.c_str(), next->name, 0))
 252         {
 253           continue;
 254         }
 255
 256         stats.collection_total++;
 257
 258         test_return_t collection_rc= world.startup(creators_ptr);
 259
 260         if (collection_rc == TEST_SUCCESS and next->pre)
 261         {
 262           collection_rc= world.runner()->pre(next->pre, creators_ptr);
 263         }
 264
 265         switch (collection_rc)
 266         {
 267         case TEST_SUCCESS:
 268           break;
 269
 270         case TEST_FAILURE:
 271           Out << next->name << " [ failed ]";
 272           failed= true;
 273           signal.set_shutdown(SHUTDOWN_GRACEFUL);
 274           goto cleanup;
 275
 276         case TEST_SKIPPED:
 277           Out << next->name << " [ skipping ]";
 278           skipped= true;
 279           goto cleanup;
 280
 281         default:
 282           throw fatal_message("invalid return code");
 283         }
 284
 285         Out << "Collection: " << next->name;
 286
 287         for (test_st *run= next->tests; run->name; run++)
 288         {
 289           struct timeval start_time, end_time;
 290           long int load_time= 0;
 291
 292           if (wildcard && fnmatch(wildcard, run->name, 0))
 293           {
 294             continue;
 295           }
 296
 297           test_return_t return_code;
 298           try {
 299             if (test_success(return_code= world.item.startup(creators_ptr)))
 300             {
 301               if (test_success(return_code= world.item.flush(creators_ptr, run)))
 302               {
 303                 // @note pre will fail is SKIPPED is returned
 304                 if (test_success(return_code= world.item.pre(creators_ptr)))
 305                 {
 306                   { // Runner Code
 307                     gettimeofday(&start_time, NULL);
 308                     assert(world.runner());
 309                     assert(run->test_fn);
 310                     try
 311                     {
 312                       return_code= world.runner()->run(run->test_fn, creators_ptr);
 313                     }
 314                     // Special case where check for the testing of the exception
 315                     // system.
 316                     catch (libtest::fatal &e)
 317                     {
 318                       if (fatal::is_disabled())
 319                       {
 320                         fatal::increment_disabled_counter();
 321                         return_code= TEST_SUCCESS;
 322                       }
 323                       else
 324                       {
 325                         throw;
 326                       }
 327                     }
 328
 329                     gettimeofday(&end_time, NULL);
 330                     load_time= timedif(end_time, start_time);
 331                   }
 332                 }
 333
 334                 // @todo do something if post fails
 335                 (void)world.item.post(creators_ptr);
 336               }
 337               else if (return_code == TEST_SKIPPED)
 338               { }
 339               else if (return_code == TEST_FAILURE)
 340               {
 341                 Error << " item.flush(failure)";
 342                 signal.set_shutdown(SHUTDOWN_GRACEFUL);
 343               }
 344             }
 345             else if (return_code == TEST_SKIPPED)
 346             { }
 347             else if (return_code == TEST_FAILURE)
 348             {
 349               Error << " item.startup(failure)";
 350               signal.set_shutdown(SHUTDOWN_GRACEFUL);
 351             }
 352           }
 353
 354           catch (std::exception &e)
 355           {
 356             Error << "Exception was thrown: " << e.what();
 357             return_code= TEST_FAILURE;
 358           }
 359           catch (...)
 360           {
 361             Error << "Unknown exception occurred";
 362             return_code= TEST_FAILURE;
 363           }
 364
 365           stats.total++;
 366
 367           switch (return_code)
 368           {
 369           case TEST_SUCCESS:
 370             Out << "\tTesting " << run->name <<  "\t\t\t\t\t" << load_time / 1000 << "." << load_time % 1000 << "[ " << test_strerror(return_code) << " ]";
 371             stats.success++;
 372             break;
 373
 374           case TEST_FAILURE:
 375             stats.failed++;
 376             failed= true;
 377             Out << "\tTesting " << run->name <<  "\t\t\t\t\t" << "[ " << test_strerror(return_code) << " ]";
 378             break;
 379
 380           case TEST_SKIPPED:
 381             stats.skipped++;
 382             skipped= true;
 383             Out << "\tTesting " << run->name <<  "\t\t\t\t\t" << "[ " << test_strerror(return_code) << " ]";
 384             break;
 385
 386           default:
 387             throw fatal_message("invalid return code");
 388           }
 389
 390           if (test_failed(world.on_error(return_code, creators_ptr)))
 391           {
 392             Error << "Failed while running on_error()";
 393             signal.set_shutdown(SHUTDOWN_GRACEFUL);
 394             break;
 395           }
 396         }
 397
 398         (void) world.runner()->post(next->post, creators_ptr);
 399
 400 cleanup:
 401         if (failed == false and skipped == false)
 402         {
 403           stats.collection_success++;
 404         }
 405
 406         if (failed)
 407         {
 408           stats.collection_failed++;
 409         }
 410
 411         if (skipped)
 412         {
 413           stats.collection_skipped++;
 414         }
 415
 416         world.shutdown(creators_ptr);
 417         Outn();
 418       }
 419
 420       if (not signal.is_shutdown())
 421       {
 422         signal.set_shutdown(SHUTDOWN_GRACEFUL);
 423       }
 424
 425       shutdown_t status= signal.get_shutdown();
 426       if (status == SHUTDOWN_FORCED)
 427       {
 428         Out << "Tests were aborted.";
 429         exit_code= EXIT_FAILURE;
 430       }
 431       else if (stats.collection_failed)
 432       {
 433         Out << "Some test failed.";
 434         exit_code= EXIT_FAILURE;
 435       }
 436       else if (stats.collection_skipped and stats.collection_failed and stats.collection_success)
 437       {
 438         Out << "Some tests were skipped.";
 439       }
 440       else if (stats.collection_success and stats.collection_failed == 0)
 441       {
 442         Out << "All tests completed successfully.";
 443       }
 444
 445       stats_print(&stats);
 446
 447       Outn(); // Generate a blank to break up the messages if make check/test has been run
 448     } while (exit_code == EXIT_SUCCESS and opt_repeat);
 449   }
 450   catch (libtest::fatal& e)
 451   {
 452     std::cerr << e.what() << std::endl;
 453   }
 454   catch (std::bad_alloc& e)
 455   {
 456     std::cerr << e.what() << std::endl;
 457   }
 458   catch (...)
 459   {
 460     std::cerr << "Unknown exception halted execution" << std::endl;
 461   }
 462
 463   return exit_code;
 464 }