git.m6w6.name Git - awesomized/libmemcached/blob - libtest/main.cc

   1 /*  vim:expandtab:shiftwidth=2:tabstop=2:smarttab:
   2  *
   3  *  Data Differential YATL (i.e. libtest)  library
   4  *
   5  *  Copyright (C) 2012-2013 Data Differential, http://datadifferential.com/
   6  *
   7  *  Redistribution and use in source and binary forms, with or without
   8  *  modification, are permitted provided that the following conditions are
   9  *  met:
  10  *
  11  *      * Redistributions of source code must retain the above copyright
  12  *  notice, this list of conditions and the following disclaimer.
  13  *
  14  *      * Redistributions in binary form must reproduce the above
  15  *  copyright notice, this list of conditions and the following disclaimer
  16  *  in the documentation and/or other materials provided with the
  17  *  distribution.
  18  *
  19  *      * The names of its contributors may not be used to endorse or
  20  *  promote products derived from this software without specific prior
  21  *  written permission.
  22  *
  23  *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  24  *  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  25  *  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  26  *  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  27  *  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  28  *  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  29  *  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  30  *  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  31  *  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  32  *  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  33  *  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  34  *
  35  */
  36
  37 #include "libtest/yatlcon.h"
  38 #include <libtest/common.h>
  39
  40 #include <cassert>
  41 #include <cstdlib>
  42 #include <cstring>
  43 #include <ctime>
  44 #include <fnmatch.h>
  45 #include <iostream>
  46 #ifdef HAVE_STRINGS_H
  47 # include <strings.h>
  48 #endif
  49 #include <fstream>
  50 #include <memory>
  51 #include <sys/stat.h>
  52 #include <sys/time.h>
  53 #include <sys/types.h>
  54 #include <sys/wait.h>
  55 #include <unistd.h>
  56
  57 #include <signal.h>
  58
  59 #ifndef __INTEL_COMPILER
  60 #pragma GCC diagnostic ignored "-Wold-style-cast"
  61 #endif
  62
  63 #if __cplusplus >= 201103L
  64 # define UNIQUE_PTR std::unique_ptr
  65 #else
  66 # define UNIQUE_PTR std::auto_ptr
  67 #endif
  68
  69 using namespace libtest;
  70
  71 static void stats_print(libtest::Framework *frame)
  72 {
  73   if (frame->failed() == 0 and frame->success() == 0)
  74   {
  75     return;
  76   }
  77
  78   Outn();
  79   Out << "Collections\t\t\t\t\t" << frame->total();
  80   Out << "\tFailed\t\t\t\t\t" << frame->failed();
  81   Out << "\tSkipped\t\t\t\t\t" << frame->skipped();
  82   Out << "\tSucceeded\t\t\t\t" << frame->success();
  83   Outn();
  84   Out << "Tests\t\t\t\t\t" << frame->sum_total();
  85   Out << "\tFailed\t\t\t\t" << frame->sum_failed();
  86   Out << "\tSkipped\t\t\t\t" << frame->sum_skipped();
  87   Out << "\tSucceeded\t\t\t" << frame->sum_success();
  88 }
  89
  90 #include <getopt.h>
  91 #include <unistd.h>
  92
  93 int main(int argc, char *argv[])
  94 {
  95   bool opt_massive= false;
  96   unsigned long int opt_repeat= 1; // Run all tests once
  97   bool opt_quiet= false;
  98   std::string collection_to_run;
  99   std::string wildcard;
 100   std::string binary_name;
 101
 102   const char *just_filename= rindex(argv[0], '/');
 103   if (just_filename)
 104   {
 105     just_filename++;
 106   }
 107   else
 108   {
 109     just_filename= argv[0];
 110   }
 111
 112   if (just_filename[0] == 'l' and just_filename[1] == 't' and  just_filename[2] == '-')
 113   {
 114     just_filename+= 3;
 115   }
 116   binary_name.append(just_filename);
 117
 118   /*
 119     Valgrind does not currently work reliably, or sometimes at all, on OSX
 120     - Fri Jun 15 11:24:07 EDT 2012
 121   */
 122 #if defined(__APPLE__) && __APPLE__
 123   if (valgrind_is_caller())
 124   {
 125     return EXIT_SKIP;
 126   }
 127 #endif
 128
 129   // Options parsing
 130   {
 131     enum long_option_t {
 132       OPT_LIBYATL_VERSION,
 133       OPT_LIBYATL_MATCH_COLLECTION,
 134       OPT_LIBYATL_MASSIVE,
 135       OPT_LIBYATL_QUIET,
 136       OPT_LIBYATL_MATCH_WILDCARD,
 137       OPT_LIBYATL_REPEAT
 138     };
 139
 140     static struct option long_options[]=
 141     {
 142       { "version", no_argument, NULL, OPT_LIBYATL_VERSION },
 143       { "quiet", no_argument, NULL, OPT_LIBYATL_QUIET },
 144       { "repeat", required_argument, NULL, OPT_LIBYATL_REPEAT },
 145       { "collection", required_argument, NULL, OPT_LIBYATL_MATCH_COLLECTION },
 146       { "wildcard", required_argument, NULL, OPT_LIBYATL_MATCH_WILDCARD },
 147       { "massive", no_argument, NULL, OPT_LIBYATL_MASSIVE },
 148       { 0, 0, 0, 0 }
 149     };
 150
 151     int option_index= 0;
 152     while (1)
 153     {
 154       int option_rv= getopt_long(argc, argv, "", long_options, &option_index);
 155       if (option_rv == -1)
 156       {
 157         break;
 158       }
 159
 160       switch (option_rv)
 161       {
 162       case OPT_LIBYATL_VERSION:
 163         break;
 164
 165       case OPT_LIBYATL_QUIET:
 166         opt_quiet= true;
 167         break;
 168
 169       case OPT_LIBYATL_REPEAT:
 170         errno= 0;
 171         opt_repeat= strtoul(optarg, (char **) NULL, 10);
 172         if (errno != 0)
 173         {
 174           Error << "unknown value passed to --repeat: `" << optarg << "`";
 175           exit(EXIT_FAILURE);
 176         }
 177         break;
 178
 179       case OPT_LIBYATL_MATCH_COLLECTION:
 180         collection_to_run= optarg;
 181         break;
 182
 183       case OPT_LIBYATL_MATCH_WILDCARD:
 184         wildcard= optarg;
 185         break;
 186
 187       case OPT_LIBYATL_MASSIVE:
 188         opt_massive= true;
 189         break;
 190
 191       case '?':
 192         /* getopt_long already printed an error message. */
 193         Error << "unknown option to getopt_long()";
 194         exit(EXIT_FAILURE);
 195
 196       default:
 197         break;
 198       }
 199     }
 200   }
 201
 202   srandom((unsigned int)time(NULL));
 203
 204   errno= 0;
 205   if (bool(getenv("YATL_REPEAT")))
 206   {
 207     errno= 0;
 208     opt_repeat= strtoul(getenv("YATL_REPEAT"), (char **) NULL, 10);
 209     if (errno != 0)
 210     {
 211       Error << "ENV YATL_REPEAT passed an invalid value: `" << getenv("YATL_REPEAT") << "`";
 212       exit(EXIT_FAILURE);
 213     }
 214   }
 215
 216   if ((bool(getenv("YATL_QUIET")) and (strcmp(getenv("YATL_QUIET"), "0") == 0)) or opt_quiet)
 217   {
 218     opt_quiet= true;
 219   }
 220   else if (getenv("JENKINS_URL"))
 221   {
 222     if (bool(getenv("YATL_QUIET")) and (strcmp(getenv("YATL_QUIET"), "1") == 0))
 223     { }
 224     else
 225     {
 226       opt_quiet= true;
 227     }
 228   }
 229
 230   if ((bool(getenv("YATL_RUN_MASSIVE_TESTS"))) or opt_massive)
 231   {
 232     opt_massive= true;
 233   }
 234
 235   if (opt_quiet)
 236   {
 237     close(STDOUT_FILENO);
 238   }
 239
 240   if (opt_massive)
 241   {
 242     is_massive(opt_massive);
 243   }
 244
 245   libtest::vchar_t tmp_directory;
 246   tmp_directory.resize(1024);
 247   if (getenv("LIBTEST_TMP"))
 248   {
 249     snprintf(&tmp_directory[0], tmp_directory.size(), "%s", getenv("LIBTEST_TMP"));
 250   }
 251   else
 252   {
 253     snprintf(&tmp_directory[0], tmp_directory.size(), "%s", LIBTEST_TEMP);
 254   }
 255
 256   if (chdir(&tmp_directory[0]) == -1)
 257   {
 258     libtest::vchar_t getcwd_buffer;
 259     getcwd_buffer.resize(1024);
 260     char *dir= getcwd(&getcwd_buffer[0], getcwd_buffer.size());
 261
 262     Error << "Unable to chdir() from " << dir << " to " << &tmp_directory[0] << " errno:" << strerror(errno);
 263     return EXIT_FAILURE;
 264   }
 265
 266   if (libtest::libtool() == NULL)
 267   {
 268     Error << "Failed to locate libtool";
 269     return EXIT_FAILURE;
 270   }
 271
 272   if (getenv("YATL_COLLECTION_TO_RUN"))
 273   {
 274     if (strlen(getenv("YATL_COLLECTION_TO_RUN")))
 275     {
 276       collection_to_run= getenv("YATL_COLLECTION_TO_RUN");
 277     }
 278   }
 279
 280   if (collection_to_run.compare("none") == 0)
 281   {
 282     return EXIT_SUCCESS;
 283   }
 284
 285   if (collection_to_run.empty() == false)
 286   {
 287     Out << "Only testing " <<  collection_to_run;
 288   }
 289
 290   int exit_code;
 291
 292   try
 293   {
 294     do
 295     {
 296       exit_code= EXIT_SUCCESS;
 297       fatal_assert(sigignore(SIGPIPE) == 0);
 298
 299       libtest::SignalThread signal;
 300       if (signal.setup() == false)
 301       {
 302         Error << "Failed to setup signals";
 303         return EXIT_FAILURE;
 304       }
 305
 306       UNIQUE_PTR<libtest::Framework> frame(new libtest::Framework(signal, binary_name, collection_to_run, wildcard));
 307
 308       // Run create(), bail on error.
 309       {
 310         switch (frame->create())
 311         {
 312         case TEST_SUCCESS:
 313           break;
 314
 315         case TEST_SKIPPED:
 316           SKIP("SKIP was returned from framework create()");
 317           break;
 318
 319         case TEST_FAILURE:
 320           std::cerr << "Could not call frame->create()" << std::endl;
 321           return EXIT_FAILURE;
 322         }
 323       }
 324
 325       frame->exec();
 326
 327       if (signal.is_shutdown() == false)
 328       {
 329         signal.set_shutdown(SHUTDOWN_GRACEFUL);
 330       }
 331
 332       shutdown_t status= signal.get_shutdown();
 333       if (status == SHUTDOWN_FORCED)
 334       {
 335         Out << "Tests were aborted.";
 336         exit_code= EXIT_FAILURE;
 337       }
 338       else if (frame->failed())
 339       {
 340         Out << "Some test failed.";
 341         exit_code= EXIT_FAILURE;
 342       }
 343       else if (frame->skipped() and frame->failed() and frame->success())
 344       {
 345         Out << "Some tests were skipped.";
 346       }
 347       else if (frame->success() and (frame->failed() == 0))
 348       {
 349         Out;
 350         Out << "All tests completed successfully.";
 351       }
 352
 353       stats_print(frame.get());
 354
 355       std::ofstream xml_file;
 356       std::string file_name;
 357       file_name.append(&tmp_directory[0]);
 358       file_name.append(frame->name());
 359       file_name.append(".xml");
 360       xml_file.open(file_name.c_str(), std::ios::trunc);
 361       libtest::Formatter::xml(*frame, xml_file);
 362
 363       Outn(); // Generate a blank to break up the messages if make check/test has been run
 364     } while (exit_code == EXIT_SUCCESS and --opt_repeat);
 365   }
 366   catch (const libtest::__skipped& e)
 367   {
 368     return EXIT_SKIP;
 369   }
 370   catch (const libtest::__failure& e)
 371   {
 372     libtest::stream::make_cout(e.file(), e.line(), e.func()) << e.what();
 373     exit_code= EXIT_FAILURE;
 374   }
 375   catch (const libtest::fatal& e)
 376   {
 377     std::cerr << "FATAL:" << e.what() << std::endl;
 378     exit_code= EXIT_FAILURE;
 379   }
 380   catch (const libtest::disconnected& e)
 381   {
 382     std::cerr << "Unhandled disconnection occurred:" << e.what() << std::endl;
 383     exit_code= EXIT_FAILURE;
 384   }
 385   catch (const std::exception& e)
 386   {
 387     std::cerr << "std::exception:" << e.what() << std::endl;
 388     exit_code= EXIT_FAILURE;
 389   }
 390   catch (char const* s)
 391   {
 392     std::cerr << "Exception:" << s << std::endl;
 393     exit_code= EXIT_FAILURE;
 394   }
 395   catch (...)
 396   {
 397     std::cerr << "Unknown exception halted execution." << std::endl;
 398     exit_code= EXIT_FAILURE;
 399   }
 400
 401   return exit_code;
 402 }