git.m6w6.name Git - awesomized/libmemcached/blob - libtest/main.cc

   1 /*  vim:expandtab:shiftwidth=2:tabstop=2:smarttab:
   2  *
   3  *  Data Differential YATL (i.e. libtest)  library
   4  *
   5  *  Copyright (C) 2012-2013 Data Differential, http://datadifferential.com/
   6  *
   7  *  Redistribution and use in source and binary forms, with or without
   8  *  modification, are permitted provided that the following conditions are
   9  *  met:
  10  *
  11  *      * Redistributions of source code must retain the above copyright
  12  *  notice, this list of conditions and the following disclaimer.
  13  *
  14  *      * Redistributions in binary form must reproduce the above
  15  *  copyright notice, this list of conditions and the following disclaimer
  16  *  in the documentation and/or other materials provided with the
  17  *  distribution.
  18  *
  19  *      * The names of its contributors may not be used to endorse or
  20  *  promote products derived from this software without specific prior
  21  *  written permission.
  22  *
  23  *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  24  *  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  25  *  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  26  *  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  27  *  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  28  *  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  29  *  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  30  *  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  31  *  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  32  *  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  33  *  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  34  *
  35  */
  36
  37 #include "libtest/yatlcon.h"
  38 #include <libtest/common.h>
  39
  40 #include <cassert>
  41 #include <cstdlib>
  42 #include <cstring>
  43 #include <ctime>
  44 #include <fnmatch.h>
  45 #include <iostream>
  46 #ifdef HAVE_STRINGS_H
  47 # include <strings.h>
  48 #endif
  49 #include <fstream>
  50 #include <memory>
  51 #include <sys/stat.h>
  52 #include <sys/time.h>
  53 #include <sys/types.h>
  54 #include <sys/wait.h>
  55 #include <unistd.h>
  56
  57 #include <signal.h>
  58
  59 #ifndef __INTEL_COMPILER
  60 #pragma GCC diagnostic ignored "-Wold-style-cast"
  61 #endif
  62
  63 using namespace libtest;
  64
  65 static void stats_print(libtest::Framework *frame)
  66 {
  67   if (frame->failed() == 0 and frame->success() == 0)
  68   {
  69     return;
  70   }
  71
  72   Outn();
  73   Out << "Collections\t\t\t\t\t" << frame->total();
  74   Out << "\tFailed\t\t\t\t\t" << frame->failed();
  75   Out << "\tSkipped\t\t\t\t\t" << frame->skipped();
  76   Out << "\tSucceeded\t\t\t\t" << frame->success();
  77   Outn();
  78   Out << "Tests\t\t\t\t\t" << frame->sum_total();
  79   Out << "\tFailed\t\t\t\t" << frame->sum_failed();
  80   Out << "\tSkipped\t\t\t\t" << frame->sum_skipped();
  81   Out << "\tSucceeded\t\t\t" << frame->sum_success();
  82 }
  83
  84 #include <getopt.h>
  85 #include <unistd.h>
  86
  87 int main(int argc, char *argv[])
  88 {
  89   bool opt_massive= false;
  90   unsigned long int opt_repeat= 1; // Run all tests once
  91   bool opt_quiet= false;
  92   std::string collection_to_run;
  93   std::string wildcard;
  94   std::string binary_name;
  95
  96   const char *just_filename= rindex(argv[0], '/');
  97   if (just_filename)
  98   {
  99     just_filename++;
 100   }
 101   else
 102   {
 103     just_filename= argv[0];
 104   }
 105
 106   if (just_filename[0] == 'l' and just_filename[1] == 't' and  just_filename[2] == '-')
 107   {
 108     just_filename+= 3;
 109   }
 110   binary_name.append(just_filename);
 111
 112   /*
 113     Valgrind does not currently work reliably, or sometimes at all, on OSX
 114     - Fri Jun 15 11:24:07 EDT 2012
 115   */
 116 #if defined(__APPLE__) && __APPLE__
 117   if (valgrind_is_caller())
 118   {
 119     return EXIT_SKIP;
 120   }
 121 #endif
 122
 123   // Options parsing
 124   {
 125     enum long_option_t {
 126       OPT_LIBYATL_VERSION,
 127       OPT_LIBYATL_MATCH_COLLECTION,
 128       OPT_LIBYATL_MASSIVE,
 129       OPT_LIBYATL_QUIET,
 130       OPT_LIBYATL_MATCH_WILDCARD,
 131       OPT_LIBYATL_REPEAT
 132     };
 133
 134     static struct option long_options[]=
 135     {
 136       { "version", no_argument, NULL, OPT_LIBYATL_VERSION },
 137       { "quiet", no_argument, NULL, OPT_LIBYATL_QUIET },
 138       { "repeat", required_argument, NULL, OPT_LIBYATL_REPEAT },
 139       { "collection", required_argument, NULL, OPT_LIBYATL_MATCH_COLLECTION },
 140       { "wildcard", required_argument, NULL, OPT_LIBYATL_MATCH_WILDCARD },
 141       { "massive", no_argument, NULL, OPT_LIBYATL_MASSIVE },
 142       { 0, 0, 0, 0 }
 143     };
 144
 145     int option_index= 0;
 146     while (1)
 147     {
 148       int option_rv= getopt_long(argc, argv, "", long_options, &option_index);
 149       if (option_rv == -1)
 150       {
 151         break;
 152       }
 153
 154       switch (option_rv)
 155       {
 156       case OPT_LIBYATL_VERSION:
 157         break;
 158
 159       case OPT_LIBYATL_QUIET:
 160         opt_quiet= true;
 161         break;
 162
 163       case OPT_LIBYATL_REPEAT:
 164         errno= 0;
 165         opt_repeat= strtoul(optarg, (char **) NULL, 10);
 166         if (errno != 0)
 167         {
 168           Error << "unknown value passed to --repeat: `" << optarg << "`";
 169           exit(EXIT_FAILURE);
 170         }
 171         break;
 172
 173       case OPT_LIBYATL_MATCH_COLLECTION:
 174         collection_to_run= optarg;
 175         break;
 176
 177       case OPT_LIBYATL_MATCH_WILDCARD:
 178         wildcard= optarg;
 179         break;
 180
 181       case OPT_LIBYATL_MASSIVE:
 182         opt_massive= true;
 183         break;
 184
 185       case '?':
 186         /* getopt_long already printed an error message. */
 187         Error << "unknown option to getopt_long()";
 188         exit(EXIT_FAILURE);
 189
 190       default:
 191         break;
 192       }
 193     }
 194   }
 195
 196   srandom((unsigned int)time(NULL));
 197
 198   errno= 0;
 199   if (bool(getenv("YATL_REPEAT")))
 200   {
 201     errno= 0;
 202     opt_repeat= strtoul(getenv("YATL_REPEAT"), (char **) NULL, 10);
 203     if (errno != 0)
 204     {
 205       Error << "ENV YATL_REPEAT passed an invalid value: `" << getenv("YATL_REPEAT") << "`";
 206       exit(EXIT_FAILURE);
 207     }
 208   }
 209
 210   if ((bool(getenv("YATL_QUIET")) and (strcmp(getenv("YATL_QUIET"), "0") == 0)) or opt_quiet)
 211   {
 212     opt_quiet= true;
 213   }
 214   else if (getenv("JENKINS_URL"))
 215   {
 216     if (bool(getenv("YATL_QUIET")) and (strcmp(getenv("YATL_QUIET"), "1") == 0))
 217     { }
 218     else
 219     {
 220       opt_quiet= true;
 221     }
 222   }
 223
 224   if ((bool(getenv("YATL_RUN_MASSIVE_TESTS"))) or opt_massive)
 225   {
 226     opt_massive= true;
 227   }
 228
 229   if (opt_quiet)
 230   {
 231     close(STDOUT_FILENO);
 232   }
 233
 234   if (opt_massive)
 235   {
 236     is_massive(opt_massive);
 237   }
 238
 239   libtest::vchar_t tmp_directory;
 240   tmp_directory.resize(1024);
 241   if (getenv("LIBTEST_TMP"))
 242   {
 243     snprintf(&tmp_directory[0], tmp_directory.size(), "%s", getenv("LIBTEST_TMP"));
 244   }
 245   else
 246   {
 247     snprintf(&tmp_directory[0], tmp_directory.size(), "%s", LIBTEST_TEMP);
 248   }
 249
 250   if (chdir(&tmp_directory[0]) == -1)
 251   {
 252     libtest::vchar_t getcwd_buffer;
 253     getcwd_buffer.resize(1024);
 254     char *dir= getcwd(&getcwd_buffer[0], getcwd_buffer.size());
 255
 256     Error << "Unable to chdir() from " << dir << " to " << &tmp_directory[0] << " errno:" << strerror(errno);
 257     return EXIT_FAILURE;
 258   }
 259
 260   if (libtest::libtool() == NULL)
 261   {
 262     Error << "Failed to locate libtool";
 263     return EXIT_FAILURE;
 264   }
 265
 266   if (getenv("YATL_COLLECTION_TO_RUN"))
 267   {
 268     if (strlen(getenv("YATL_COLLECTION_TO_RUN")))
 269     {
 270       collection_to_run= getenv("YATL_COLLECTION_TO_RUN");
 271     }
 272   }
 273
 274   if (collection_to_run.compare("none") == 0)
 275   {
 276     return EXIT_SUCCESS;
 277   }
 278
 279   if (collection_to_run.empty() == false)
 280   {
 281     Out << "Only testing " <<  collection_to_run;
 282   }
 283
 284   int exit_code;
 285
 286   try
 287   {
 288     do
 289     {
 290       exit_code= EXIT_SUCCESS;
 291       fatal_assert(sigignore(SIGPIPE) == 0);
 292
 293       libtest::SignalThread signal;
 294       if (signal.setup() == false)
 295       {
 296         Error << "Failed to setup signals";
 297         return EXIT_FAILURE;
 298       }
 299
 300       std::auto_ptr<libtest::Framework> frame(new libtest::Framework(signal, binary_name, collection_to_run, wildcard));
 301
 302       // Run create(), bail on error.
 303       {
 304         switch (frame->create())
 305         {
 306         case TEST_SUCCESS:
 307           break;
 308
 309         case TEST_SKIPPED:
 310           SKIP("SKIP was returned from framework create()");
 311           break;
 312
 313         case TEST_FAILURE:
 314           std::cerr << "Could not call frame->create()" << std::endl;
 315           return EXIT_FAILURE;
 316         }
 317       }
 318
 319       frame->exec();
 320
 321       if (signal.is_shutdown() == false)
 322       {
 323         signal.set_shutdown(SHUTDOWN_GRACEFUL);
 324       }
 325
 326       shutdown_t status= signal.get_shutdown();
 327       if (status == SHUTDOWN_FORCED)
 328       {
 329         Out << "Tests were aborted.";
 330         exit_code= EXIT_FAILURE;
 331       }
 332       else if (frame->failed())
 333       {
 334         Out << "Some test failed.";
 335         exit_code= EXIT_FAILURE;
 336       }
 337       else if (frame->skipped() and frame->failed() and frame->success())
 338       {
 339         Out << "Some tests were skipped.";
 340       }
 341       else if (frame->success() and (frame->failed() == 0))
 342       {
 343         Out;
 344         Out << "All tests completed successfully.";
 345       }
 346
 347       stats_print(frame.get());
 348
 349       std::ofstream xml_file;
 350       std::string file_name;
 351       file_name.append(&tmp_directory[0]);
 352       file_name.append(frame->name());
 353       file_name.append(".xml");
 354       xml_file.open(file_name.c_str(), std::ios::trunc);
 355       libtest::Formatter::xml(*frame, xml_file);
 356
 357       Outn(); // Generate a blank to break up the messages if make check/test has been run
 358     } while (exit_code == EXIT_SUCCESS and --opt_repeat);
 359   }
 360   catch (const libtest::__skipped& e)
 361   {
 362     return EXIT_SKIP;
 363   }
 364   catch (const libtest::__failure& e)
 365   {
 366     libtest::stream::make_cout(e.file(), e.line(), e.func()) << e.what();
 367     exit_code= EXIT_FAILURE;
 368   }
 369   catch (const libtest::fatal& e)
 370   {
 371     std::cerr << "FATAL:" << e.what() << std::endl;
 372     exit_code= EXIT_FAILURE;
 373   }
 374   catch (const libtest::disconnected& e)
 375   {
 376     std::cerr << "Unhandled disconnection occurred:" << e.what() << std::endl;
 377     exit_code= EXIT_FAILURE;
 378   }
 379   catch (const std::exception& e)
 380   {
 381     std::cerr << "std::exception:" << e.what() << std::endl;
 382     exit_code= EXIT_FAILURE;
 383   }
 384   catch (char const* s)
 385   {
 386     std::cerr << "Exception:" << s << std::endl;
 387     exit_code= EXIT_FAILURE;
 388   }
 389   catch (...)
 390   {
 391     std::cerr << "Unknown exception halted execution." << std::endl;
 392     exit_code= EXIT_FAILURE;
 393   }
 394
 395   return exit_code;
 396 }