git.m6w6.name Git - m6w6/libmemcached/blob - clients/memslap.cc

   1 /*  vim:expandtab:shiftwidth=2:tabstop=2:smarttab:
   2  *
   3  *  Libmemcached library
   4  *
   5  *  Copyright (C) 2011 Data Differential, http://datadifferential.com/
   6  *  Copyright (C) 2006-2009 Brian Aker All rights reserved.
   7  *
   8  *  Redistribution and use in source and binary forms, with or without
   9  *  modification, are permitted provided that the following conditions are
  10  *  met:
  11  *
  12  *      * Redistributions of source code must retain the above copyright
  13  *  notice, this list of conditions and the following disclaimer.
  14  *
  15  *      * Redistributions in binary form must reproduce the above
  16  *  copyright notice, this list of conditions and the following disclaimer
  17  *  in the documentation and/or other materials provided with the
  18  *  distribution.
  19  *
  20  *      * The names of its contributors may not be used to endorse or
  21  *  promote products derived from this software without specific prior
  22  *  written permission.
  23  *
  24  *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  25  *  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  26  *  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  27  *  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  28  *  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  29  *  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  30  *  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  31  *  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  32  *  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  33  *  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  34  *  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  35  *
  36  */
  37
  38
  39 #include <config.h>
  40
  41 #include <cassert>
  42 #include <cstdio>
  43 #include <cstdlib>
  44 #include <cstring>
  45 #include <fcntl.h>
  46 #include <getopt.h>
  47 #include <memory>
  48 #include <pthread.h>
  49 #include <sys/mman.h>
  50 #include <sys/stat.h>
  51 #include <sys/time.h>
  52 #include <sys/types.h>
  53 #include <sys/types.h>
  54 #include <unistd.h>
  55
  56 #include <iostream>
  57
  58 #include <libmemcached/memcached.h>
  59
  60 #include "client_options.h"
  61 #include "utilities.h"
  62 #include "generator.h"
  63 #include "execute.h"
  64
  65 #define DEFAULT_INITIAL_LOAD 10000
  66 #define DEFAULT_EXECUTE_NUMBER 10000
  67 #define DEFAULT_CONCURRENCY 1
  68
  69 #define PROGRAM_NAME "memslap"
  70 #define PROGRAM_DESCRIPTION "Generates a load against a memcached custer of servers."
  71
  72 /* Global Thread counter */
  73 volatile unsigned int master_wakeup;
  74 pthread_mutex_t sleeper_mutex;
  75 pthread_cond_t sleep_threshhold;
  76
  77 /* Types */
  78 enum test_t {
  79   SET_TEST,
  80   GET_TEST,
  81   MGET_TEST
  82 };
  83
  84 struct thread_context_st {
  85   unsigned int key_count;
  86   pairs_st *initial_pairs;
  87   unsigned int initial_number;
  88   pairs_st *execute_pairs;
  89   unsigned int execute_number;
  90   char **keys;
  91   size_t *key_lengths;
  92   test_t test;
  93   memcached_st *memc;
  94   const memcached_st* root;
  95
  96   thread_context_st(const memcached_st* memc_arg, test_t test_arg) :
  97     key_count(0),
  98     initial_pairs(NULL),
  99     initial_number(0),
 100     execute_pairs(NULL),
 101     execute_number(0),
 102     keys(0),
 103     key_lengths(NULL),
 104     test(test_arg),
 105     memc(NULL),
 106     root(memc_arg)
 107   {
 108   }
 109
 110   void init()
 111   {
 112     memc= memcached_clone(NULL, root);
 113   }
 114
 115   ~thread_context_st()
 116   {
 117     if (execute_pairs)
 118     {
 119       pairs_free(execute_pairs);
 120     }
 121     memcached_free(memc);
 122   }
 123 };
 124
 125 struct conclusions_st {
 126   long int load_time;
 127   long int read_time;
 128   unsigned int rows_loaded;
 129   unsigned int rows_read;
 130
 131   conclusions_st() :
 132     load_time(0),
 133     read_time(0),
 134     rows_loaded(0),
 135     rows_read()
 136   { }
 137 };
 138
 139 /* Prototypes */
 140 void options_parse(int argc, char *argv[]);
 141 void conclusions_print(conclusions_st *conclusion);
 142 void scheduler(memcached_server_st *servers, conclusions_st *conclusion);
 143 pairs_st *load_create_data(memcached_st *memc, unsigned int number_of,
 144                            unsigned int *actual_loaded);
 145 void flush_all(memcached_st *memc);
 146
 147 static bool opt_binary= 0;
 148 static int opt_verbose= 0;
 149 static int opt_flush= 0;
 150 static int opt_non_blocking_io= 0;
 151 static int opt_tcp_nodelay= 0;
 152 static unsigned int opt_execute_number= 0;
 153 static unsigned int opt_createial_load= 0;
 154 static unsigned int opt_concurrency= 0;
 155 static int opt_displayflag= 0;
 156 static char *opt_servers= NULL;
 157 static bool opt_udp_io= false;
 158 test_t opt_test= SET_TEST;
 159
 160 extern "C" {
 161
 162 static void *run_task(void *p)
 163 {
 164   thread_context_st *context= (thread_context_st *)p;
 165
 166   context->init();
 167
 168   pthread_mutex_lock(&sleeper_mutex);
 169   while (master_wakeup)
 170   {
 171     pthread_cond_wait(&sleep_threshhold, &sleeper_mutex);
 172   }
 173   pthread_mutex_unlock(&sleeper_mutex);
 174
 175   /* Do Stuff */
 176   switch (context->test)
 177   {
 178   case SET_TEST:
 179     assert(context->execute_pairs);
 180     execute_set(context->memc, context->execute_pairs, context->execute_number);
 181     break;
 182
 183   case GET_TEST:
 184     execute_get(context->memc, context->initial_pairs, context->initial_number);
 185     break;
 186
 187   case MGET_TEST:
 188     execute_mget(context->memc, (const char*const*)context->keys, context->key_lengths, context->initial_number);
 189     break;
 190   }
 191
 192   delete context;
 193
 194   pthread_exit(0);
 195 }
 196
 197 }
 198
 199
 200 int main(int argc, char *argv[])
 201 {
 202   conclusions_st conclusion;
 203
 204   srandom((unsigned int)time(NULL));
 205   options_parse(argc, argv);
 206
 207   if (opt_servers == NULL)
 208   {
 209     char *temp;
 210
 211     if ((temp= getenv("MEMCACHED_SERVERS")))
 212     {
 213       opt_servers= strdup(temp);
 214     }
 215     else
 216     {
 217       fprintf(stderr, "No Servers provided\n");
 218       return EXIT_FAILURE;
 219     }
 220   }
 221
 222   memcached_server_st *servers= memcached_servers_parse(opt_servers);
 223
 224   pthread_mutex_init(&sleeper_mutex, NULL);
 225   pthread_cond_init(&sleep_threshhold, NULL);
 226
 227   scheduler(servers, &conclusion);
 228
 229   free(opt_servers);
 230
 231   (void)pthread_mutex_destroy(&sleeper_mutex);
 232   (void)pthread_cond_destroy(&sleep_threshhold);
 233   conclusions_print(&conclusion);
 234   memcached_server_list_free(servers);
 235
 236   return EXIT_SUCCESS;
 237 }
 238
 239 void scheduler(memcached_server_st *servers, conclusions_st *conclusion)
 240 {
 241   unsigned int actual_loaded= 0; /* Fix warning */
 242
 243   struct timeval start_time, end_time;
 244   pairs_st *pairs= NULL;
 245
 246   memcached_st *memc= memcached_create(NULL);
 247
 248   memcached_server_push(memc, servers);
 249
 250   /* We need to set udp behavior before adding servers to the client */
 251   if (opt_udp_io)
 252   {
 253     if (memcached_failed(memcached_behavior_set(memc, MEMCACHED_BEHAVIOR_USE_UDP, opt_udp_io)))
 254     {
 255       std::cerr << "Failed to enable UDP." << std::endl;
 256       memcached_free(memc);
 257       exit(EXIT_FAILURE);
 258     }
 259   }
 260
 261   memcached_behavior_set(memc, MEMCACHED_BEHAVIOR_BINARY_PROTOCOL,
 262                          (uint64_t)opt_binary);
 263
 264   if (opt_flush)
 265   {
 266     flush_all(memc);
 267   }
 268
 269   if (opt_createial_load)
 270   {
 271     pairs= load_create_data(memc, opt_createial_load, &actual_loaded);
 272   }
 273
 274   char **keys= static_cast<char **>(calloc(actual_loaded, sizeof(char*)));
 275   size_t *key_lengths= static_cast<size_t *>(calloc(actual_loaded, sizeof(size_t)));
 276
 277   if (keys == NULL or key_lengths == NULL)
 278   {
 279     free(keys);
 280     free(key_lengths);
 281     keys= NULL;
 282     key_lengths= NULL;
 283   }
 284   else
 285   {
 286     for (uint32_t x= 0; x < actual_loaded; ++x)
 287     {
 288       keys[x]= pairs[x].key;
 289       key_lengths[x]= pairs[x].key_length;
 290     }
 291   }
 292
 293   /* We set this after we have loaded */
 294   {
 295     if (opt_non_blocking_io)
 296       memcached_behavior_set(memc, MEMCACHED_BEHAVIOR_NO_BLOCK, 1);
 297
 298     if (opt_tcp_nodelay)
 299       memcached_behavior_set(memc, MEMCACHED_BEHAVIOR_TCP_NODELAY, 1);
 300   }
 301
 302   pthread_mutex_lock(&sleeper_mutex);
 303   master_wakeup= 1;
 304   pthread_mutex_unlock(&sleeper_mutex);
 305
 306   pthread_t *threads= new  (std::nothrow) pthread_t[opt_concurrency];
 307
 308   if (not threads)
 309   {
 310     exit(EXIT_FAILURE);
 311   }
 312
 313   for (uint32_t x= 0; x < opt_concurrency; x++)
 314   {
 315     thread_context_st *context= new thread_context_st(memc, opt_test);
 316     context->test= opt_test;
 317
 318     context->initial_pairs= pairs;
 319     context->initial_number= actual_loaded;
 320     context->keys= keys;
 321     context->key_lengths= key_lengths;
 322
 323     if (opt_test == SET_TEST)
 324     {
 325       context->execute_pairs= pairs_generate(opt_execute_number, 400);
 326       context->execute_number= opt_execute_number;
 327     }
 328
 329     /* now you create the thread */
 330     if (pthread_create(threads +x, NULL, run_task, (void *)context) != 0)
 331     {
 332       fprintf(stderr,"Could not create thread\n");
 333       exit(1);
 334     }
 335   }
 336
 337   pthread_mutex_lock(&sleeper_mutex);
 338   master_wakeup= 0;
 339   pthread_mutex_unlock(&sleeper_mutex);
 340   pthread_cond_broadcast(&sleep_threshhold);
 341   gettimeofday(&start_time, NULL);
 342
 343   for (uint32_t x= 0; x < opt_concurrency; x++)
 344   {
 345     void *retval;
 346     pthread_join(threads[x], &retval);
 347   }
 348   delete [] threads;
 349
 350   gettimeofday(&end_time, NULL);
 351
 352   conclusion->load_time= timedif(end_time, start_time);
 353   conclusion->read_time= timedif(end_time, start_time);
 354   free(keys);
 355   free(key_lengths);
 356   pairs_free(pairs);
 357   memcached_free(memc);
 358 }
 359
 360 void options_parse(int argc, char *argv[])
 361 {
 362   memcached_programs_help_st help_options[]=
 363   {
 364     {0},
 365   };
 366
 367   static struct option long_options[]=
 368     {
 369       {(OPTIONSTRING)"concurrency", required_argument, NULL, OPT_SLAP_CONCURRENCY},
 370       {(OPTIONSTRING)"debug", no_argument, &opt_verbose, OPT_DEBUG},
 371       {(OPTIONSTRING)"quiet", no_argument, NULL, OPT_QUIET},
 372       {(OPTIONSTRING)"execute-number", required_argument, NULL, OPT_SLAP_EXECUTE_NUMBER},
 373       {(OPTIONSTRING)"flag", no_argument, &opt_displayflag, OPT_FLAG},
 374       {(OPTIONSTRING)"flush", no_argument, &opt_flush, OPT_FLUSH},
 375       {(OPTIONSTRING)"help", no_argument, NULL, OPT_HELP},
 376       {(OPTIONSTRING)"initial-load", required_argument, NULL, OPT_SLAP_INITIAL_LOAD}, /* Number to load initially */
 377       {(OPTIONSTRING)"non-blocking", no_argument, &opt_non_blocking_io, OPT_SLAP_NON_BLOCK},
 378       {(OPTIONSTRING)"servers", required_argument, NULL, OPT_SERVERS},
 379       {(OPTIONSTRING)"tcp-nodelay", no_argument, &opt_tcp_nodelay, OPT_SLAP_TCP_NODELAY},
 380       {(OPTIONSTRING)"test", required_argument, NULL, OPT_SLAP_TEST},
 381       {(OPTIONSTRING)"verbose", no_argument, &opt_verbose, OPT_VERBOSE},
 382       {(OPTIONSTRING)"version", no_argument, NULL, OPT_VERSION},
 383       {(OPTIONSTRING)"binary", no_argument, NULL, OPT_BINARY},
 384       {(OPTIONSTRING)"udp", no_argument, NULL, OPT_UDP},
 385       {0, 0, 0, 0},
 386     };
 387
 388   bool opt_help= false;
 389   bool opt_version= false;
 390   int option_index= 0;
 391   while (1)
 392   {
 393     int option_rv= getopt_long(argc, argv, "Vhvds:", long_options, &option_index);
 394
 395     if (option_rv == -1) break;
 396
 397     switch (option_rv)
 398     {
 399     case 0:
 400       break;
 401
 402     case OPT_UDP:
 403       if (opt_test == GET_TEST)
 404       {
 405         fprintf(stderr, "You can not run a get test in UDP mode. UDP mode "
 406                   "does not currently support get ops.\n");
 407         exit(1);
 408       }
 409       opt_udp_io= true;
 410       break;
 411
 412     case OPT_BINARY:
 413       opt_binary= true;
 414       break;
 415
 416     case OPT_VERBOSE: /* --verbose or -v */
 417       opt_verbose= OPT_VERBOSE;
 418       break;
 419
 420     case OPT_DEBUG: /* --debug or -d */
 421       opt_verbose = OPT_DEBUG;
 422       break;
 423
 424     case OPT_VERSION: /* --version or -V */
 425       opt_version= true;
 426       break;
 427
 428     case OPT_HELP: /* --help or -h */
 429       opt_help= true;
 430       break;
 431
 432     case OPT_SERVERS: /* --servers or -s */
 433       opt_servers= strdup(optarg);
 434       break;
 435
 436     case OPT_SLAP_TEST:
 437       if (strcmp(optarg, "get") == 0)
 438       {
 439         if (opt_udp_io == 1)
 440         {
 441           fprintf(stderr, "You can not run a get test in UDP mode. UDP mode "
 442                   "does not currently support get ops.\n");
 443           exit(EXIT_FAILURE);
 444         }
 445         opt_test= GET_TEST ;
 446       }
 447       else if (strcmp(optarg, "set") == 0)
 448       {
 449         opt_test= SET_TEST;
 450       }
 451       else if (strcmp(optarg, "mget") == 0)
 452       {
 453         opt_test= MGET_TEST;
 454       }
 455       else
 456       {
 457         fprintf(stderr, "Your test, %s, is not a known test\n", optarg);
 458         exit(EXIT_FAILURE);
 459       }
 460       break;
 461
 462     case OPT_SLAP_CONCURRENCY:
 463       opt_concurrency= (unsigned int)strtoul(optarg, (char **)NULL, 10);
 464       break;
 465
 466     case OPT_SLAP_EXECUTE_NUMBER:
 467       opt_execute_number= (unsigned int)strtoul(optarg, (char **)NULL, 10);
 468       break;
 469
 470     case OPT_SLAP_INITIAL_LOAD:
 471       opt_createial_load= (unsigned int)strtoul(optarg, (char **)NULL, 10);
 472       break;
 473
 474     case OPT_QUIET:
 475       close_stdio();
 476       break;
 477
 478
 479     case '?':
 480       /* getopt_long already printed an error message. */
 481       exit(EXIT_FAILURE);
 482
 483     default:
 484       abort();
 485     }
 486   }
 487
 488   if (opt_version)
 489   {
 490     version_command(PROGRAM_NAME);
 491     exit(EXIT_SUCCESS);
 492   }
 493
 494   if (opt_help)
 495   {
 496     help_command(PROGRAM_NAME, PROGRAM_DESCRIPTION, long_options, help_options);
 497     exit(EXIT_SUCCESS);
 498   }
 499
 500   if ((opt_test == GET_TEST or opt_test == MGET_TEST) and opt_createial_load == 0)
 501     opt_createial_load= DEFAULT_INITIAL_LOAD;
 502
 503   if (opt_execute_number == 0)
 504     opt_execute_number= DEFAULT_EXECUTE_NUMBER;
 505
 506   if (opt_concurrency == 0)
 507     opt_concurrency= DEFAULT_CONCURRENCY;
 508 }
 509
 510 void conclusions_print(conclusions_st *conclusion)
 511 {
 512   printf("\tThreads connecting to servers %u\n", opt_concurrency);
 513 #ifdef NOT_FINISHED
 514   printf("\tLoaded %u rows\n", conclusion->rows_loaded);
 515   printf("\tRead %u rows\n", conclusion->rows_read);
 516 #endif
 517   if (opt_test == SET_TEST)
 518     printf("\tTook %ld.%03ld seconds to load data\n", conclusion->load_time / 1000,
 519            conclusion->load_time % 1000);
 520   else
 521     printf("\tTook %ld.%03ld seconds to read data\n", conclusion->read_time / 1000,
 522            conclusion->read_time % 1000);
 523 }
 524
 525 void flush_all(memcached_st *memc)
 526 {
 527   memcached_flush(memc, 0);
 528 }
 529
 530 pairs_st *load_create_data(memcached_st *memc, unsigned int number_of,
 531                            unsigned int *actual_loaded)
 532 {
 533   memcached_st *memc_clone= memcached_clone(NULL, memc);
 534   /* We always used non-blocking IO for load since it is faster */
 535   memcached_behavior_set(memc_clone, MEMCACHED_BEHAVIOR_NO_BLOCK, 0);
 536
 537   pairs_st *pairs= pairs_generate(number_of, 400);
 538   *actual_loaded= execute_set(memc_clone, pairs, number_of);
 539
 540   memcached_free(memc_clone);
 541
 542   return pairs;
 543 }