git.m6w6.name Git - awesomized/libmemcached/blob - libtest/test.cc

   1 /*  vim:expandtab:shiftwidth=2:tabstop=2:smarttab:
   2  *
   3  *  libtest
   4  *
   5  *  Copyright (C) 2011 Data Differential, http://datadifferential.com/
   6  *
   7  *  This library is free software; you can redistribute it and/or
   8  *  modify it under the terms of the GNU Lesser General Public
   9  *  License as published by the Free Software Foundation; either
  10  *  version 3 of the License, or (at your option) any later version.
  11  *
  12  *  This library is distributed in the hope that it will be useful,
  13  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15  *  Lesser General Public License for more details.
  16  *
  17  *  You should have received a copy of the GNU Lesser General Public
  18  *  License along with this library; if not, write to the Free Software
  19  *  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
  20  */
  21
  22 #include <libtest/common.h>
  23
  24 #include <cassert>
  25 #include <cstdlib>
  26 #include <cstring>
  27 #include <sys/time.h>
  28 #include <sys/types.h>
  29 #include <sys/stat.h>
  30 #include <sys/wait.h>
  31 #include <unistd.h>
  32 #include <ctime>
  33 #include <fnmatch.h>
  34 #include <iostream>
  35
  36 #include <signal.h>
  37
  38 #if defined(HAVE_CURL_CURL_H) && HAVE_CURL_CURL_H
  39 #include <curl/curl.h>
  40 #endif
  41
  42 #ifndef __INTEL_COMPILER
  43 #pragma GCC diagnostic ignored "-Wold-style-cast"
  44 #endif
  45
  46 using namespace libtest;
  47
  48 static void stats_print(Stats *stats)
  49 {
  50   if (stats->collection_failed == 0 and stats->collection_success == 0)
  51   {
  52     return;
  53   }
  54
  55   Out << "\tTotal Collections\t\t\t\t" << stats->collection_total;
  56   Out << "\tFailed Collections\t\t\t\t" << stats->collection_failed;
  57   Out << "\tSkipped Collections\t\t\t\t" << stats->collection_skipped;
  58   Out << "\tSucceeded Collections\t\t\t\t" << stats->collection_success;
  59   Outn();
  60   Out << "Total\t\t\t\t" << stats->total;
  61   Out << "\tFailed\t\t\t" << stats->failed;
  62   Out << "\tSkipped\t\t\t" << stats->skipped;
  63   Out << "\tSucceeded\t\t" << stats->success;
  64 }
  65
  66 static long int timedif(struct timeval a, struct timeval b)
  67 {
  68   long us, s;
  69
  70   us = (long)(a.tv_usec - b.tv_usec);
  71   us /= 1000;
  72   s = (long)(a.tv_sec - b.tv_sec);
  73   s *= 1000;
  74   return s + us;
  75 }
  76
  77 static void cleanup_curl(void)
  78 {
  79 #if defined(HAVE_CURL_CURL_H) && HAVE_CURL_CURL_H
  80   curl_global_cleanup();
  81 #endif
  82 }
  83
  84 #include <getopt.h>
  85 #include <unistd.h>
  86
  87 int main(int argc, char *argv[])
  88 {
  89 #if defined(HAVE_CURL_CURL_H) && HAVE_CURL_CURL_H
  90   if (curl_global_init(CURL_GLOBAL_ALL))
  91   {
  92     Error << "curl_global_init(CURL_GLOBAL_ALL) failed";
  93     return EXIT_FAILURE;
  94   }
  95 #endif
  96
  97   if (atexit(cleanup_curl))
  98   {
  99     Error << "atexit() failed";
 100     return EXIT_FAILURE;
 101   }
 102
 103   bool opt_repeat= false;
 104   std::string collection_to_run;
 105
 106   // Options parsing
 107   {
 108     enum long_option_t {
 109       OPT_LIBYATL_VERSION,
 110       OPT_LIBYATL_MATCH_COLLECTION,
 111       OPT_LIBYATL_REPEAT
 112     };
 113
 114     static struct option long_options[]=
 115     {
 116       {"repeat", no_argument, NULL, OPT_LIBYATL_REPEAT},
 117       {"collection", required_argument, NULL, OPT_LIBYATL_MATCH_COLLECTION},
 118       {0, 0, 0, 0}
 119     };
 120
 121     int option_index= 0;
 122     while (1)
 123     {
 124       int option_rv= getopt_long(argc, argv, "", long_options, &option_index);
 125       if (option_rv == -1)
 126       {
 127         break;
 128       }
 129
 130       switch (option_rv)
 131       {
 132       case OPT_LIBYATL_VERSION:
 133         break;
 134
 135       case OPT_LIBYATL_REPEAT:
 136         opt_repeat= true;
 137         break;
 138
 139       case OPT_LIBYATL_MATCH_COLLECTION:
 140         collection_to_run= optarg;
 141         break;
 142
 143       case '?':
 144         /* getopt_long already printed an error message. */
 145         Error << "unknown option to getopt_long()";
 146         exit(EXIT_FAILURE);
 147
 148       default:
 149         break;
 150       }
 151     }
 152   }
 153
 154   srandom((unsigned int)time(NULL));
 155
 156   if (getenv("LIBTEST_QUIET") and strcmp(getenv("LIBTEST_QUIET"), "0") == 0)
 157   {
 158     close(STDOUT_FILENO);
 159   }
 160   else if (getenv("JENKINS_URL"))
 161   {
 162     close(STDOUT_FILENO);
 163   }
 164
 165   char buffer[1024];
 166   if (getenv("LIBTEST_TMP"))
 167   {
 168     snprintf(buffer, sizeof(buffer), "%s", getenv("LIBTEST_TMP"));
 169   }
 170   else
 171   {
 172     snprintf(buffer, sizeof(buffer), "%s", LIBTEST_TEMP);
 173   }
 174
 175   if (chdir(buffer) == -1)
 176   {
 177     char getcwd_buffer[1024];
 178     char *dir= getcwd(getcwd_buffer, sizeof(getcwd_buffer));
 179
 180     Error << "Unable to chdir() from " << dir << " to " << buffer << " errno:" << strerror(errno);
 181     return EXIT_FAILURE;
 182   }
 183
 184   if (libtest::libtool() == NULL)
 185   {
 186     Error << "Failed to locate libtool";
 187     return EXIT_FAILURE;
 188   }
 189
 190   int exit_code;
 191
 192   try {
 193     do {
 194       exit_code= EXIT_SUCCESS;
 195       Framework *world= new Framework();
 196
 197       if (world == NULL)
 198       {
 199         Error << "Failed to create Framework()";
 200         return EXIT_FAILURE;
 201       }
 202
 203       assert(sigignore(SIGPIPE) == 0);
 204
 205       libtest::SignalThread signal;
 206       if (not signal.setup())
 207       {
 208         Error << "Failed to setup signals";
 209         return EXIT_FAILURE;
 210       }
 211
 212       Stats stats;
 213
 214       get_world(world);
 215
 216       test_return_t error;
 217       void *creators_ptr= world->create(error);
 218
 219       switch (error)
 220       {
 221       case TEST_SUCCESS:
 222         break;
 223
 224       case TEST_SKIPPED:
 225         Out << "SKIP " << argv[0];
 226         delete world;
 227         return EXIT_SUCCESS;
 228
 229       case TEST_FATAL:
 230       case TEST_FAILURE:
 231       case TEST_MEMORY_ALLOCATION_FAILURE:
 232         delete world;
 233         return EXIT_FAILURE;
 234       }
 235
 236       if (getenv("TEST_COLLECTION"))
 237       {
 238         if (strlen(getenv("TEST_COLLECTION")))
 239         {
 240           collection_to_run= getenv("TEST_COLLECTION");
 241         }
 242       }
 243
 244       if (collection_to_run.empty() == false)
 245       {
 246         Out << "Only testing " <<  collection_to_run;
 247       }
 248
 249       char *wildcard= NULL;
 250       if (argc == 3)
 251       {
 252         wildcard= argv[2];
 253       }
 254
 255       for (collection_st *next= world->collections; next and next->name and (not signal.is_shutdown()); next++)
 256       {
 257         bool failed= false;
 258         bool skipped= false;
 259
 260         if (collection_to_run.empty() == false and fnmatch(collection_to_run.c_str(), next->name, 0))
 261         {
 262           continue;
 263         }
 264
 265         stats.collection_total++;
 266
 267         test_return_t collection_rc= world->startup(creators_ptr);
 268
 269         if (collection_rc == TEST_SUCCESS and next->pre)
 270         {
 271           collection_rc= world->runner()->pre(next->pre, creators_ptr);
 272         }
 273
 274         switch (collection_rc)
 275         {
 276         case TEST_SUCCESS:
 277           break;
 278
 279         case TEST_FATAL:
 280         case TEST_FAILURE:
 281           Out << next->name << " [ failed ]";
 282           failed= true;
 283           signal.set_shutdown(SHUTDOWN_GRACEFUL);
 284           goto cleanup;
 285
 286         case TEST_SKIPPED:
 287           Out << next->name << " [ skipping ]";
 288           skipped= true;
 289           goto cleanup;
 290
 291         case TEST_MEMORY_ALLOCATION_FAILURE:
 292           test_assert(0, "Allocation failure, or unknown return");
 293         }
 294
 295         Out << "Collection: " << next->name;
 296
 297         for (test_st *run= next->tests; run->name; run++)
 298         {
 299           struct timeval start_time, end_time;
 300           long int load_time= 0;
 301
 302           if (wildcard && fnmatch(wildcard, run->name, 0))
 303           {
 304             continue;
 305           }
 306
 307           test_return_t return_code;
 308           try {
 309             if (test_success(return_code= world->item.startup(creators_ptr)))
 310             {
 311               if (test_success(return_code= world->item.flush(creators_ptr, run)))
 312               {
 313                 // @note pre will fail is SKIPPED is returned
 314                 if (test_success(return_code= world->item.pre(creators_ptr)))
 315                 {
 316                   { // Runner Code
 317                     gettimeofday(&start_time, NULL);
 318                     assert(world->runner());
 319                     assert(run->test_fn);
 320                     try
 321                     {
 322                       return_code= world->runner()->run(run->test_fn, creators_ptr);
 323                     }
 324                     // Special case where check for the testing of the exception
 325                     // system.
 326                     catch (libtest::fatal &e)
 327                     {
 328                       if (fatal::is_disabled())
 329                       {
 330                         fatal::increment_disabled_counter();
 331                         return_code= TEST_SUCCESS;
 332                       }
 333                       else
 334                       {
 335                         throw;
 336                       }
 337                     }
 338
 339                     gettimeofday(&end_time, NULL);
 340                     load_time= timedif(end_time, start_time);
 341                   }
 342                 }
 343
 344                 // @todo do something if post fails
 345                 (void)world->item.post(creators_ptr);
 346               }
 347               else if (return_code == TEST_SKIPPED)
 348               { }
 349               else if (return_code == TEST_FAILURE)
 350               {
 351                 Error << " item.flush(failure)";
 352                 signal.set_shutdown(SHUTDOWN_GRACEFUL);
 353               }
 354             }
 355             else if (return_code == TEST_SKIPPED)
 356             { }
 357             else if (return_code == TEST_FAILURE)
 358             {
 359               Error << " item.startup(failure)";
 360               signal.set_shutdown(SHUTDOWN_GRACEFUL);
 361             }
 362           }
 363
 364           catch (std::exception &e)
 365           {
 366             Error << "Exception was thrown: " << e.what();
 367             return_code= TEST_FAILURE;
 368           }
 369           catch (...)
 370           {
 371             Error << "Unknown exception occurred";
 372             return_code= TEST_FAILURE;
 373           }
 374
 375           stats.total++;
 376
 377           switch (return_code)
 378           {
 379           case TEST_SUCCESS:
 380             Out << "\tTesting " << run->name <<  "\t\t\t\t\t" << load_time / 1000 << "." << load_time % 1000 << "[ " << test_strerror(return_code) << " ]";
 381             stats.success++;
 382             break;
 383
 384           case TEST_FATAL:
 385           case TEST_FAILURE:
 386             stats.failed++;
 387             failed= true;
 388             Out << "\tTesting " << run->name <<  "\t\t\t\t\t" << "[ " << test_strerror(return_code) << " ]";
 389             break;
 390
 391           case TEST_SKIPPED:
 392             stats.skipped++;
 393             skipped= true;
 394             Out << "\tTesting " << run->name <<  "\t\t\t\t\t" << "[ " << test_strerror(return_code) << " ]";
 395             break;
 396
 397           case TEST_MEMORY_ALLOCATION_FAILURE:
 398             test_assert(0, "Memory Allocation Error");
 399           }
 400
 401           if (test_failed(world->on_error(return_code, creators_ptr)))
 402           {
 403             Error << "Failed while running on_error()";
 404             signal.set_shutdown(SHUTDOWN_GRACEFUL);
 405             break;
 406           }
 407         }
 408
 409         (void) world->runner()->post(next->post, creators_ptr);
 410
 411 cleanup:
 412         if (failed == false and skipped == false)
 413         {
 414           stats.collection_success++;
 415         }
 416
 417         if (failed)
 418         {
 419           stats.collection_failed++;
 420         }
 421
 422         if (skipped)
 423         {
 424           stats.collection_skipped++;
 425         }
 426
 427         world->shutdown(creators_ptr);
 428         Outn();
 429       }
 430
 431       if (not signal.is_shutdown())
 432       {
 433         signal.set_shutdown(SHUTDOWN_GRACEFUL);
 434       }
 435
 436       shutdown_t status= signal.get_shutdown();
 437       if (status == SHUTDOWN_FORCED)
 438       {
 439         Out << "Tests were aborted.";
 440         exit_code= EXIT_FAILURE;
 441       }
 442       else if (stats.collection_failed)
 443       {
 444         Out << "Some test failed.";
 445         exit_code= EXIT_FAILURE;
 446       }
 447       else if (stats.collection_skipped and stats.collection_failed and stats.collection_success)
 448       {
 449         Out << "Some tests were skipped.";
 450       }
 451       else if (stats.collection_success and stats.collection_failed == 0)
 452       {
 453         Out << "All tests completed successfully.";
 454       }
 455
 456       stats_print(&stats);
 457
 458       delete world;
 459
 460       Outn(); // Generate a blank to break up the messages if make check/test has been run
 461     } while (exit_code == EXIT_SUCCESS and opt_repeat);
 462   }
 463   catch (libtest::fatal& e)
 464   {
 465     std::cerr << e.what() << std::endl;
 466   }
 467   catch (...)
 468   {
 469     std::cerr << "Unknown exception halted execution" << std::endl;
 470   }
 471
 472   return exit_code;
 473 }