git.m6w6.name Git - m6w6/libmemcached/blob - libtest/test.cc

   1 /*  vim:expandtab:shiftwidth=2:tabstop=2:smarttab:
   2  *
   3  *  uTest, libtest
   4  *
   5  *  Copyright (C) 2011 Data Differential, http://datadifferential.com/
   6  *  Copyright (C) 2006-2009 Brian Aker
   7  *
   8  *  Redistribution and use in source and binary forms, with or without
   9  *  modification, are permitted provided that the following conditions are
  10  *  met:
  11  *
  12  *      * Redistributions of source code must retain the above copyright
  13  *  notice, this list of conditions and the following disclaimer.
  14  *
  15  *      * Redistributions in binary form must reproduce the above
  16  *  copyright notice, this list of conditions and the following disclaimer
  17  *  in the documentation and/or other materials provided with the
  18  *  distribution.
  19  *
  20  *      * The names of its contributors may not be used to endorse or
  21  *  promote products derived from this software without specific prior
  22  *  written permission.
  23  *
  24  *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  25  *  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  26  *  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  27  *  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  28  *  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  29  *  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  30  *  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  31  *  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  32  *  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  33  *  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  34  *  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  35  *
  36  */
  37
  38 #include <libtest/common.h>
  39
  40 #include <cassert>
  41 #include <cstdlib>
  42 #include <cstring>
  43 #include <sys/time.h>
  44 #include <sys/types.h>
  45 #include <sys/stat.h>
  46 #include <sys/wait.h>
  47 #include <unistd.h>
  48 #include <ctime>
  49 #include <fnmatch.h>
  50 #include <iostream>
  51
  52 #include <signal.h>
  53
  54 #include <libtest/stats.h>
  55 #include <libtest/signal.h>
  56
  57 #ifndef __INTEL_COMPILER
  58 #pragma GCC diagnostic ignored "-Wold-style-cast"
  59 #endif
  60
  61 using namespace libtest;
  62
  63 static in_port_t global_port= 0;
  64 static char global_socket[1024];
  65
  66 in_port_t default_port()
  67 {
  68   return global_port;
  69 }
  70
  71 void set_default_port(in_port_t port)
  72 {
  73   global_port= port;
  74 }
  75
  76 const char *default_socket()
  77 {
  78   assert(global_socket[0]);
  79   return global_socket;
  80 }
  81
  82 bool test_is_local()
  83 {
  84   return (getenv("LIBTEST_LOCAL"));
  85 }
  86
  87 void set_default_socket(const char *socket)
  88 {
  89   if (socket)
  90   {
  91     strncpy(global_socket, socket, strlen(socket));
  92   }
  93 }
  94
  95 static void stats_print(Stats *stats)
  96 {
  97   Out << "\tTotal Collections\t\t\t\t" << stats->collection_total;
  98   Out << "\tFailed Collections\t\t\t\t" << stats->collection_failed;
  99   Out << "\tSkipped Collections\t\t\t\t" << stats->collection_skipped;
 100   Out << "\tSucceeded Collections\t\t\t\t" << stats->collection_success;
 101   Outn();
 102   Out << "Total\t\t\t\t" << stats->total;
 103   Out << "\tFailed\t\t\t" << stats->failed;
 104   Out << "\tSkipped\t\t\t" << stats->skipped;
 105   Out << "\tSucceeded\t\t" << stats->success;
 106 }
 107
 108 static long int timedif(struct timeval a, struct timeval b)
 109 {
 110   long us, s;
 111
 112   us = (long)(a.tv_usec - b.tv_usec);
 113   us /= 1000;
 114   s = (long)(a.tv_sec - b.tv_sec);
 115   s *= 1000;
 116   return s + us;
 117 }
 118
 119 const char *test_strerror(test_return_t code)
 120 {
 121   switch (code) {
 122   case TEST_SUCCESS:
 123     return "ok";
 124
 125   case TEST_FAILURE:
 126     return "failed";
 127
 128   case TEST_MEMORY_ALLOCATION_FAILURE:
 129     return "memory allocation";
 130
 131   case TEST_SKIPPED:
 132     return "skipped";
 133
 134   case TEST_FATAL:
 135     break;
 136   }
 137
 138   return "failed";
 139 }
 140
 141 void create_core(void)
 142 {
 143   if (getenv("LIBMEMCACHED_NO_COREDUMP") == NULL)
 144   {
 145     pid_t pid= fork();
 146
 147     if (pid == 0)
 148     {
 149       abort();
 150     }
 151     else
 152     {
 153       while (waitpid(pid, NULL, 0) != pid) {};
 154     }
 155   }
 156 }
 157
 158 static Framework *world= NULL;
 159 int main(int argc, char *argv[])
 160 {
 161   srandom((unsigned int)time(NULL));
 162
 163   world= new Framework();
 164
 165   if (not world)
 166   {
 167     return EXIT_FAILURE;
 168   }
 169
 170   setup_signals();
 171
 172   Stats stats;
 173
 174   get_world(world);
 175
 176   test_return_t error;
 177   void *creators_ptr= world->create(error);
 178   if (test_failed(error))
 179   {
 180     Error << "create() failed";
 181     delete world;
 182     return EXIT_FAILURE;
 183   }
 184
 185   char *collection_to_run= NULL;
 186   if (argc > 1)
 187   {
 188     collection_to_run= argv[1];
 189   }
 190   else if (getenv("TEST_COLLECTION"))
 191   {
 192     collection_to_run= getenv("TEST_COLLECTION");
 193   }
 194
 195   if (collection_to_run)
 196   {
 197     Out << "Only testing " <<  collection_to_run;
 198   }
 199
 200   char *wildcard= NULL;
 201   if (argc == 3)
 202   {
 203     wildcard= argv[2];
 204   }
 205
 206   for (collection_st *next= world->collections; next->name and (not is_shutdown()); next++)
 207   {
 208     test_return_t collection_rc= TEST_SUCCESS;
 209     bool failed= false;
 210     bool skipped= false;
 211
 212     if (collection_to_run && fnmatch(collection_to_run, next->name, 0))
 213       continue;
 214
 215     stats.collection_total++;
 216
 217     collection_rc= world->startup(creators_ptr);
 218
 219     if (collection_rc == TEST_SUCCESS and next->pre)
 220     {
 221       collection_rc= world->runner()->pre(next->pre, creators_ptr);
 222     }
 223
 224     switch (collection_rc)
 225     {
 226     case TEST_SUCCESS:
 227       break;
 228
 229     case TEST_FATAL:
 230     case TEST_FAILURE:
 231       Out << next->name << " [ failed ]";
 232       failed= true;
 233       set_shutdown(SHUTDOWN_GRACEFUL);
 234       goto cleanup;
 235
 236     case TEST_SKIPPED:
 237       Out << next->name << " [ skipping ]";
 238       skipped= true;
 239       goto cleanup;
 240
 241     case TEST_MEMORY_ALLOCATION_FAILURE:
 242       test_assert(0, "Allocation failure, or unknown return");
 243     }
 244
 245     Out << "Collection: " << next->name;
 246
 247     for (test_st *run= next->tests; run->name; run++)
 248     {
 249       struct timeval start_time, end_time;
 250       long int load_time= 0;
 251
 252       if (wildcard && fnmatch(wildcard, run->name, 0))
 253       {
 254         continue;
 255       }
 256
 257       test_return_t return_code;
 258       if (test_success(return_code= world->item.startup(creators_ptr)))
 259       {
 260         if (test_success(return_code= world->item.flush(creators_ptr, run)))
 261         {
 262           // @note pre will fail is SKIPPED is returned
 263           if (test_success(return_code= world->item.pre(creators_ptr)))
 264           {
 265             { // Runner Code
 266               gettimeofday(&start_time, NULL);
 267               assert(world->runner());
 268               assert(run->test_fn);
 269               return_code= world->runner()->run(run->test_fn, creators_ptr);
 270               gettimeofday(&end_time, NULL);
 271               load_time= timedif(end_time, start_time);
 272             }
 273           }
 274
 275           // @todo do something if post fails
 276           (void)world->item.post(creators_ptr);
 277         }
 278         else if (return_code == TEST_SKIPPED)
 279         { }
 280         else if (return_code == TEST_FAILURE)
 281         {
 282           Error << " item.flush(failure)";
 283           set_shutdown(SHUTDOWN_GRACEFUL);
 284         }
 285       }
 286       else if (return_code == TEST_SKIPPED)
 287       { }
 288       else if (return_code == TEST_FAILURE)
 289       {
 290         Error << " item.startup(failure)";
 291         set_shutdown(SHUTDOWN_GRACEFUL);
 292       }
 293
 294       stats.total++;
 295
 296       switch (return_code)
 297       {
 298       case TEST_SUCCESS:
 299         Out << "\tTesting " << run->name <<  "\t\t\t\t\t" << load_time / 1000 << "." << load_time % 1000 << "[ " << test_strerror(return_code) << " ]";
 300         stats.success++;
 301         break;
 302
 303       case TEST_FATAL:
 304       case TEST_FAILURE:
 305         stats.failed++;
 306         failed= true;
 307         Out << "\tTesting " << run->name <<  "\t\t\t\t\t" << "[ " << test_strerror(return_code) << " ]";
 308         break;
 309
 310       case TEST_SKIPPED:
 311         stats.skipped++;
 312         skipped= true;
 313         Out << "\tTesting " << run->name <<  "\t\t\t\t\t" << "[ " << test_strerror(return_code) << " ]";
 314         break;
 315
 316       case TEST_MEMORY_ALLOCATION_FAILURE:
 317         test_assert(0, "Memory Allocation Error");
 318       }
 319
 320       if (test_failed(world->on_error(return_code, creators_ptr)))
 321       {
 322         Error << "Failed while running on_error()";
 323         set_shutdown(SHUTDOWN_GRACEFUL);
 324         break;
 325       }
 326     }
 327
 328     (void) world->runner()->post(next->post, creators_ptr);
 329
 330 cleanup:
 331     if (failed == false and skipped == false)
 332     {
 333       stats.collection_success++;
 334     }
 335
 336     if (failed)
 337     {
 338       stats.collection_failed++;
 339     }
 340
 341     if (skipped)
 342     {
 343       stats.collection_skipped++;
 344     }
 345
 346     world->shutdown(creators_ptr);
 347     Outn();
 348   }
 349
 350   if (not is_shutdown())
 351   {
 352     set_shutdown(SHUTDOWN_GRACEFUL);
 353   }
 354
 355   int exit_code= EXIT_SUCCESS;
 356   shutdown_t status= get_shutdown();
 357   if (status == SHUTDOWN_FORCED)
 358   {
 359     Out << "Tests were aborted.";
 360     exit_code= EXIT_FAILURE;
 361   }
 362   else if (stats.collection_failed)
 363   {
 364     Out << "Some test failed.";
 365     exit_code= EXIT_FAILURE;
 366   }
 367   else if (stats.collection_skipped)
 368   {
 369     Out << "Some tests were skipped.";
 370   }
 371   else
 372   {
 373     Out << "All tests completed successfully.";
 374   }
 375
 376   stats_print(&stats);
 377
 378   delete world;
 379
 380   Outn(); // Generate a blank to break up the messages if make check/test has been run
 381
 382   return exit_code;
 383 }