git.m6w6.name Git - m6w6/libmemcached/blob - libtest/test.cc

   1 /*  vim:expandtab:shiftwidth=2:tabstop=2:smarttab:
   2  *
   3  *  libtest
   4  *
   5  *  Copyright (C) 2011 Data Differential, http://datadifferential.com/
   6  *
   7  *  This library is free software; you can redistribute it and/or
   8  *  modify it under the terms of the GNU Lesser General Public
   9  *  License as published by the Free Software Foundation; either
  10  *  version 3 of the License, or (at your option) any later version.
  11  *
  12  *  This library is distributed in the hope that it will be useful,
  13  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15  *  Lesser General Public License for more details.
  16  *
  17  *  You should have received a copy of the GNU Lesser General Public
  18  *  License along with this library; if not, write to the Free Software
  19  *  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
  20  */
  21
  22 #include <libtest/common.h>
  23
  24 #include <cassert>
  25 #include <cstdlib>
  26 #include <cstring>
  27 #include <sys/time.h>
  28 #include <sys/types.h>
  29 #include <sys/stat.h>
  30 #include <sys/wait.h>
  31 #include <unistd.h>
  32 #include <ctime>
  33 #include <fnmatch.h>
  34 #include <iostream>
  35
  36 #include <signal.h>
  37
  38 #ifndef __INTEL_COMPILER
  39 #pragma GCC diagnostic ignored "-Wold-style-cast"
  40 #endif
  41
  42 using namespace libtest;
  43
  44 static void stats_print(Stats *stats)
  45 {
  46   if (stats->collection_failed == 0 and stats->collection_success == 0)
  47   {
  48     return;
  49   }
  50
  51   Out << "\tTotal Collections\t\t\t\t" << stats->collection_total;
  52   Out << "\tFailed Collections\t\t\t\t" << stats->collection_failed;
  53   Out << "\tSkipped Collections\t\t\t\t" << stats->collection_skipped;
  54   Out << "\tSucceeded Collections\t\t\t\t" << stats->collection_success;
  55   Outn();
  56   Out << "Total\t\t\t\t" << stats->total;
  57   Out << "\tFailed\t\t\t" << stats->failed;
  58   Out << "\tSkipped\t\t\t" << stats->skipped;
  59   Out << "\tSucceeded\t\t" << stats->success;
  60 }
  61
  62 static long int timedif(struct timeval a, struct timeval b)
  63 {
  64   long us, s;
  65
  66   us = (long)(a.tv_usec - b.tv_usec);
  67   us /= 1000;
  68   s = (long)(a.tv_sec - b.tv_sec);
  69   s *= 1000;
  70   return s + us;
  71 }
  72
  73 static Framework *world= NULL;
  74 int main(int argc, char *argv[])
  75 {
  76   srandom((unsigned int)time(NULL));
  77
  78   if (getenv("LIBTEST_QUIET"))
  79   {
  80     close(STDOUT_FILENO);
  81   }
  82   else if (getenv("JENKINS_URL"))
  83   {
  84     close(STDOUT_FILENO);
  85   }
  86
  87   char buffer[1024];
  88   if (getenv("LIBTEST_TMP"))
  89   {
  90     snprintf(buffer, sizeof(buffer), "%s", getenv("LIBTEST_TMP"));
  91   }
  92   else
  93   {
  94     snprintf(buffer, sizeof(buffer), "%s", LIBTEST_TEMP);
  95   }
  96
  97   if (chdir(buffer) == -1)
  98   {
  99     char getcwd_buffer[1024];
 100     char *dir= getcwd(getcwd_buffer, sizeof(getcwd_buffer));
 101
 102     Error << "Unable to chdir() from " << dir << " to " << buffer << " errno:" << strerror(errno);
 103     return EXIT_FAILURE;
 104   }
 105
 106   if (libtest::libtool() == NULL)
 107   {
 108     Error << "Failed to locate libtool";
 109     return EXIT_FAILURE;
 110   }
 111
 112   world= new Framework();
 113
 114   if (world == NULL)
 115   {
 116     Error << "Failed to create Framework()";
 117     return EXIT_FAILURE;
 118   }
 119
 120   libtest::SignalThread signal;
 121   if (not signal.setup())
 122   {
 123     return EXIT_FAILURE;
 124   }
 125
 126   Stats stats;
 127
 128   get_world(world);
 129
 130   test_return_t error;
 131   void *creators_ptr= world->create(error);
 132
 133   switch (error)
 134   {
 135   case TEST_SUCCESS:
 136     break;
 137
 138   case TEST_SKIPPED:
 139     Out << "SKIP " << argv[0];
 140     delete world;
 141     return EXIT_SUCCESS;
 142
 143   case TEST_FATAL:
 144   case TEST_FAILURE:
 145   case TEST_MEMORY_ALLOCATION_FAILURE:
 146     delete world;
 147     return EXIT_FAILURE;
 148   }
 149
 150   char *collection_to_run= NULL;
 151   if (argc > 1)
 152   {
 153     collection_to_run= argv[1];
 154   }
 155   else if (getenv("TEST_COLLECTION"))
 156   {
 157     if (strlen(getenv("TEST_COLLECTION")))
 158     {
 159       collection_to_run= getenv("TEST_COLLECTION");
 160     }
 161   }
 162
 163   if (collection_to_run)
 164   {
 165     Out << "Only testing " <<  collection_to_run;
 166   }
 167
 168   char *wildcard= NULL;
 169   if (argc == 3)
 170   {
 171     wildcard= argv[2];
 172   }
 173
 174   for (collection_st *next= world->collections; next->name and (not signal.is_shutdown()); next++)
 175   {
 176     test_return_t collection_rc= TEST_SUCCESS;
 177     bool failed= false;
 178     bool skipped= false;
 179
 180     if (collection_to_run && fnmatch(collection_to_run, next->name, 0))
 181       continue;
 182
 183     stats.collection_total++;
 184
 185     collection_rc= world->startup(creators_ptr);
 186
 187     if (collection_rc == TEST_SUCCESS and next->pre)
 188     {
 189       collection_rc= world->runner()->pre(next->pre, creators_ptr);
 190     }
 191
 192     switch (collection_rc)
 193     {
 194     case TEST_SUCCESS:
 195       break;
 196
 197     case TEST_FATAL:
 198     case TEST_FAILURE:
 199       Out << next->name << " [ failed ]";
 200       failed= true;
 201       signal.set_shutdown(SHUTDOWN_GRACEFUL);
 202       goto cleanup;
 203
 204     case TEST_SKIPPED:
 205       Out << next->name << " [ skipping ]";
 206       skipped= true;
 207       goto cleanup;
 208
 209     case TEST_MEMORY_ALLOCATION_FAILURE:
 210       test_assert(0, "Allocation failure, or unknown return");
 211     }
 212
 213     Out << "Collection: " << next->name;
 214
 215     for (test_st *run= next->tests; run->name; run++)
 216     {
 217       struct timeval start_time, end_time;
 218       long int load_time= 0;
 219
 220       if (wildcard && fnmatch(wildcard, run->name, 0))
 221       {
 222         continue;
 223       }
 224
 225       test_return_t return_code;
 226       if (test_success(return_code= world->item.startup(creators_ptr)))
 227       {
 228         if (test_success(return_code= world->item.flush(creators_ptr, run)))
 229         {
 230           // @note pre will fail is SKIPPED is returned
 231           if (test_success(return_code= world->item.pre(creators_ptr)))
 232           {
 233             { // Runner Code
 234               gettimeofday(&start_time, NULL);
 235               assert(world->runner());
 236               assert(run->test_fn);
 237               return_code= world->runner()->run(run->test_fn, creators_ptr);
 238               gettimeofday(&end_time, NULL);
 239               load_time= timedif(end_time, start_time);
 240             }
 241           }
 242
 243           // @todo do something if post fails
 244           (void)world->item.post(creators_ptr);
 245         }
 246         else if (return_code == TEST_SKIPPED)
 247         { }
 248         else if (return_code == TEST_FAILURE)
 249         {
 250           Error << " item.flush(failure)";
 251           signal.set_shutdown(SHUTDOWN_GRACEFUL);
 252         }
 253       }
 254       else if (return_code == TEST_SKIPPED)
 255       { }
 256       else if (return_code == TEST_FAILURE)
 257       {
 258         Error << " item.startup(failure)";
 259         signal.set_shutdown(SHUTDOWN_GRACEFUL);
 260       }
 261
 262       stats.total++;
 263
 264       switch (return_code)
 265       {
 266       case TEST_SUCCESS:
 267         Out << "\tTesting " << run->name <<  "\t\t\t\t\t" << load_time / 1000 << "." << load_time % 1000 << "[ " << test_strerror(return_code) << " ]";
 268         stats.success++;
 269         break;
 270
 271       case TEST_FATAL:
 272       case TEST_FAILURE:
 273         stats.failed++;
 274         failed= true;
 275         Out << "\tTesting " << run->name <<  "\t\t\t\t\t" << "[ " << test_strerror(return_code) << " ]";
 276         break;
 277
 278       case TEST_SKIPPED:
 279         stats.skipped++;
 280         skipped= true;
 281         Out << "\tTesting " << run->name <<  "\t\t\t\t\t" << "[ " << test_strerror(return_code) << " ]";
 282         break;
 283
 284       case TEST_MEMORY_ALLOCATION_FAILURE:
 285         test_assert(0, "Memory Allocation Error");
 286       }
 287
 288       if (test_failed(world->on_error(return_code, creators_ptr)))
 289       {
 290         Error << "Failed while running on_error()";
 291         signal.set_shutdown(SHUTDOWN_GRACEFUL);
 292         break;
 293       }
 294     }
 295
 296     (void) world->runner()->post(next->post, creators_ptr);
 297
 298 cleanup:
 299     if (failed == false and skipped == false)
 300     {
 301       stats.collection_success++;
 302     }
 303
 304     if (failed)
 305     {
 306       stats.collection_failed++;
 307     }
 308
 309     if (skipped)
 310     {
 311       stats.collection_skipped++;
 312     }
 313
 314     world->shutdown(creators_ptr);
 315     Outn();
 316   }
 317
 318   if (not signal.is_shutdown())
 319   {
 320     signal.set_shutdown(SHUTDOWN_GRACEFUL);
 321   }
 322
 323   int exit_code= EXIT_SUCCESS;
 324   shutdown_t status= signal.get_shutdown();
 325   if (status == SHUTDOWN_FORCED)
 326   {
 327     Out << "Tests were aborted.";
 328     exit_code= EXIT_FAILURE;
 329   }
 330   else if (stats.collection_failed)
 331   {
 332     Out << "Some test failed.";
 333     exit_code= EXIT_FAILURE;
 334   }
 335   else if (stats.collection_skipped and stats.collection_failed and stats.collection_success)
 336   {
 337     Out << "Some tests were skipped.";
 338   }
 339   else if (stats.collection_success and stats.collection_failed == 0)
 340   {
 341     Out << "All tests completed successfully.";
 342   }
 343
 344   stats_print(&stats);
 345
 346   delete world;
 347
 348   Outn(); // Generate a blank to break up the messages if make check/test has been run
 349
 350   return exit_code;
 351 }