git.m6w6.name Git - awesomized/libmemcached/blob - src/bin/memslap.cc

   1 /*
   2     +--------------------------------------------------------------------+
   3     | libmemcached - C/C++ Client Library for memcached                  |
   4     +--------------------------------------------------------------------+
   5     | Redistribution and use in source and binary forms, with or without |
   6     | modification, are permitted under the terms of the BSD license.    |
   7     | You should have received a copy of the license in a bundled file   |
   8     | named LICENSE; in case you did not receive a copy you can review   |
   9     | the terms online at: https://opensource.org/licenses/BSD-3-Clause  |
  10     +--------------------------------------------------------------------+
  11     | Copyright (c) 2006-2014 Brian Aker   https://datadifferential.com/ |
  12     | Copyright (c) 2020 Michael Wallner   <mike@php.net>                |
  13     +--------------------------------------------------------------------+
  14 */
  15
  16 #include "mem_config.h"
  17
  18 #include <cassert>
  19 #include <cerrno>
  20 #include <cstdio>
  21 #include <cstdlib>
  22 #include <cstring>
  23 #include <fcntl.h>
  24 #include <getopt.h>
  25 #include <memory>
  26 #include <pthread.h>
  27 #include <sys/mman.h>
  28 #include <sys/stat.h>
  29 #include <sys/time.h>
  30 #include <sys/types.h>
  31 #include <unistd.h>
  32
  33 #include <iostream>
  34
  35 #include "libmemcached-1.0/memcached.h"
  36
  37 #include "client_options.h"
  38 #include "utilities.h"
  39 #include "generator.h"
  40 #include "execute.h"
  41
  42 #define DEFAULT_INITIAL_LOAD   10000
  43 #define DEFAULT_EXECUTE_NUMBER 10000
  44 #define DEFAULT_CONCURRENCY    1
  45
  46 #define VALUE_BYTES 4096
  47
  48 #define PROGRAM_NAME        "memslap"
  49 #define PROGRAM_DESCRIPTION "Generates a load against a memcached custer of servers."
  50
  51 /* Global Thread counter */
  52 volatile unsigned int master_wakeup;
  53 pthread_mutex_t sleeper_mutex;
  54 pthread_cond_t sleep_threshhold;
  55
  56 /* Types */
  57 enum test_t { SET_TEST, GET_TEST, MGET_TEST };
  58
  59 struct thread_context_st {
  60   unsigned int key_count;
  61   pairs_st *initial_pairs;
  62   unsigned int initial_number;
  63   pairs_st *execute_pairs;
  64   unsigned int execute_number;
  65   char **keys;
  66   size_t *key_lengths;
  67   test_t test;
  68   memcached_st *memc;
  69   const memcached_st *root;
  70
  71   thread_context_st(const memcached_st *memc_arg, test_t test_arg)
  72   : key_count(0)
  73   , initial_pairs(NULL)
  74   , initial_number(0)
  75   , execute_pairs(NULL)
  76   , execute_number(0)
  77   , keys(0)
  78   , key_lengths(NULL)
  79   , test(test_arg)
  80   , memc(NULL)
  81   , root(memc_arg) {}
  82
  83   void init() { memc = memcached_clone(NULL, root); }
  84
  85   ~thread_context_st() {
  86     if (execute_pairs) {
  87       pairs_free(execute_pairs);
  88     }
  89     memcached_free(memc);
  90   }
  91 };
  92
  93 struct conclusions_st {
  94   long int load_time;
  95   long int read_time;
  96   unsigned int rows_loaded;
  97   unsigned int rows_read;
  98
  99   conclusions_st()
 100   : load_time(0)
 101   , read_time(0)
 102   , rows_loaded(0)
 103   , rows_read() {}
 104 };
 105
 106 /* Prototypes */
 107 void options_parse(int argc, char *argv[]);
 108 void conclusions_print(conclusions_st *conclusion);
 109 void scheduler(memcached_server_st *servers, conclusions_st *conclusion);
 110 pairs_st *load_create_data(memcached_st *memc, unsigned int number_of, unsigned int *actual_loaded);
 111 void flush_all(memcached_st *memc);
 112
 113 static bool opt_binary = 0;
 114 static int opt_verbose = 0;
 115 static int opt_flush = 0;
 116 static int opt_non_blocking_io = 0;
 117 static int opt_tcp_nodelay = 0;
 118 static unsigned int opt_execute_number = 0;
 119 static unsigned int opt_createial_load = 0;
 120 static unsigned int opt_concurrency = 0;
 121 static int opt_displayflag = 0;
 122 static char *opt_servers = NULL;
 123 static bool opt_udp_io = false;
 124 test_t opt_test = SET_TEST;
 125
 126 extern "C" {
 127
 128 static __attribute__((noreturn)) void *run_task(void *p) {
 129   thread_context_st *context = (thread_context_st *) p;
 130
 131   context->init();
 132
 133   pthread_mutex_lock(&sleeper_mutex);
 134   while (master_wakeup) {
 135     pthread_cond_wait(&sleep_threshhold, &sleeper_mutex);
 136   }
 137   pthread_mutex_unlock(&sleeper_mutex);
 138
 139   /* Do Stuff */
 140   switch (context->test) {
 141   case SET_TEST:
 142     assert(context->execute_pairs);
 143     execute_set(context->memc, context->execute_pairs, context->execute_number);
 144     break;
 145
 146   case GET_TEST: execute_get(context->memc, context->initial_pairs, context->initial_number); break;
 147
 148   case MGET_TEST:
 149     execute_mget(context->memc, (const char *const *) context->keys, context->key_lengths,
 150                  context->initial_number);
 151     break;
 152   }
 153
 154   delete context;
 155
 156   pthread_exit(0);
 157 }
 158 }
 159
 160 int main(int argc, char *argv[]) {
 161   conclusions_st conclusion;
 162
 163   srandom((unsigned int) time(NULL));
 164   options_parse(argc, argv);
 165
 166   if (opt_servers == NULL) {
 167     char *temp;
 168
 169     if ((temp = getenv("MEMCACHED_SERVERS"))) {
 170       opt_servers = strdup(temp);
 171     }
 172
 173     if (opt_servers == NULL) {
 174       std::cerr << "No servers provided" << std::endl;
 175       exit(EXIT_FAILURE);
 176     }
 177   }
 178
 179   memcached_server_st *servers = memcached_servers_parse(opt_servers);
 180   if (servers == NULL or memcached_server_list_count(servers) == 0) {
 181     std::cerr << "Invalid server list provided:" << opt_servers << std::endl;
 182     return EXIT_FAILURE;
 183   }
 184
 185   pthread_mutex_init(&sleeper_mutex, NULL);
 186   pthread_cond_init(&sleep_threshhold, NULL);
 187
 188   int error_code = EXIT_SUCCESS;
 189   try {
 190     scheduler(servers, &conclusion);
 191   } catch (std::exception &e) {
 192     std::cerr << "Died with exception: " << e.what() << std::endl;
 193     error_code = EXIT_FAILURE;
 194   }
 195
 196   free(opt_servers);
 197
 198   (void) pthread_mutex_destroy(&sleeper_mutex);
 199   (void) pthread_cond_destroy(&sleep_threshhold);
 200   conclusions_print(&conclusion);
 201   memcached_server_list_free(servers);
 202
 203   return error_code;
 204 }
 205
 206 void scheduler(memcached_server_st *servers, conclusions_st *conclusion) {
 207   unsigned int actual_loaded = 0; /* Fix warning */
 208
 209   struct timeval start_time, end_time;
 210   pairs_st *pairs = NULL;
 211
 212   memcached_st *memc = memcached_create(NULL);
 213
 214   memcached_server_push(memc, servers);
 215
 216   /* We need to set udp behavior before adding servers to the client */
 217   if (opt_udp_io) {
 218     if (memcached_failed(memcached_behavior_set(memc, MEMCACHED_BEHAVIOR_USE_UDP, opt_udp_io))) {
 219       std::cerr << "Failed to enable UDP." << std::endl;
 220       memcached_free(memc);
 221       exit(EXIT_FAILURE);
 222     }
 223   }
 224
 225   memcached_behavior_set(memc, MEMCACHED_BEHAVIOR_BINARY_PROTOCOL, (uint64_t) opt_binary);
 226
 227   if (opt_flush) {
 228     flush_all(memc);
 229   }
 230
 231   if (opt_createial_load) {
 232     pairs = load_create_data(memc, opt_createial_load, &actual_loaded);
 233   }
 234
 235   char **keys = static_cast<char **>(calloc(actual_loaded, sizeof(char *)));
 236   size_t *key_lengths = static_cast<size_t *>(calloc(actual_loaded, sizeof(size_t)));
 237
 238   if (keys == NULL or key_lengths == NULL) {
 239     free(keys);
 240     free(key_lengths);
 241     keys = NULL;
 242     key_lengths = NULL;
 243   } else {
 244     for (uint32_t x = 0; x < actual_loaded; ++x) {
 245       keys[x] = pairs[x].key;
 246       key_lengths[x] = pairs[x].key_length;
 247     }
 248   }
 249
 250   /* We set this after we have loaded */
 251   {
 252     if (opt_non_blocking_io)
 253       memcached_behavior_set(memc, MEMCACHED_BEHAVIOR_NO_BLOCK, 1);
 254
 255     if (opt_tcp_nodelay)
 256       memcached_behavior_set(memc, MEMCACHED_BEHAVIOR_TCP_NODELAY, 1);
 257   }
 258
 259   pthread_mutex_lock(&sleeper_mutex);
 260   master_wakeup = 1;
 261   pthread_mutex_unlock(&sleeper_mutex);
 262
 263   pthread_t *threads = new (std::nothrow) pthread_t[opt_concurrency];
 264
 265   if (threads == NULL) {
 266     exit(EXIT_FAILURE);
 267   }
 268
 269   for (uint32_t x = 0; x < opt_concurrency; x++) {
 270     thread_context_st *context = new thread_context_st(memc, opt_test);
 271     context->test = opt_test;
 272
 273     context->initial_pairs = pairs;
 274     context->initial_number = actual_loaded;
 275     context->keys = keys;
 276     context->key_lengths = key_lengths;
 277
 278     if (opt_test == SET_TEST) {
 279       context->execute_pairs = pairs_generate(opt_execute_number, VALUE_BYTES);
 280       context->execute_number = opt_execute_number;
 281     }
 282
 283     /* now you create the thread */
 284     if (pthread_create(threads + x, NULL, run_task, (void *) context) != 0) {
 285       fprintf(stderr, "Could not create thread\n");
 286       exit(1);
 287     }
 288   }
 289
 290   pthread_mutex_lock(&sleeper_mutex);
 291   master_wakeup = 0;
 292   pthread_mutex_unlock(&sleeper_mutex);
 293   pthread_cond_broadcast(&sleep_threshhold);
 294   gettimeofday(&start_time, NULL);
 295
 296   for (uint32_t x = 0; x < opt_concurrency; x++) {
 297     void *retval;
 298     pthread_join(threads[x], &retval);
 299   }
 300   delete[] threads;
 301
 302   gettimeofday(&end_time, NULL);
 303
 304   conclusion->load_time = timedif(end_time, start_time);
 305   conclusion->read_time = timedif(end_time, start_time);
 306   free(keys);
 307   free(key_lengths);
 308   pairs_free(pairs);
 309   memcached_free(memc);
 310 }
 311
 312 void options_parse(int argc, char *argv[]) {
 313   memcached_programs_help_st help_options[] = {
 314       {0},
 315   };
 316
 317   static struct option long_options[] = {
 318       {(OPTIONSTRING) "concurrency", required_argument, NULL, OPT_SLAP_CONCURRENCY},
 319       {(OPTIONSTRING) "debug", no_argument, &opt_verbose, OPT_DEBUG},
 320       {(OPTIONSTRING) "quiet", no_argument, NULL, OPT_QUIET},
 321       {(OPTIONSTRING) "execute-number", required_argument, NULL, OPT_SLAP_EXECUTE_NUMBER},
 322       {(OPTIONSTRING) "flag", no_argument, &opt_displayflag, OPT_FLAG},
 323       {(OPTIONSTRING) "flush", no_argument, &opt_flush, OPT_FLUSH},
 324       {(OPTIONSTRING) "help", no_argument, NULL, OPT_HELP},
 325       {(OPTIONSTRING) "initial-load", required_argument, NULL,
 326        OPT_SLAP_INITIAL_LOAD}, /* Number to load initially */
 327       {(OPTIONSTRING) "non-blocking", no_argument, &opt_non_blocking_io, OPT_SLAP_NON_BLOCK},
 328       {(OPTIONSTRING) "servers", required_argument, NULL, OPT_SERVERS},
 329       {(OPTIONSTRING) "tcp-nodelay", no_argument, &opt_tcp_nodelay, OPT_SLAP_TCP_NODELAY},
 330       {(OPTIONSTRING) "test", required_argument, NULL, OPT_SLAP_TEST},
 331       {(OPTIONSTRING) "verbose", no_argument, &opt_verbose, OPT_VERBOSE},
 332       {(OPTIONSTRING) "version", no_argument, NULL, OPT_VERSION},
 333       {(OPTIONSTRING) "binary", no_argument, NULL, OPT_BINARY},
 334       {(OPTIONSTRING) "udp", no_argument, NULL, OPT_UDP},
 335       {0, 0, 0, 0},
 336   };
 337
 338   bool opt_help = false;
 339   bool opt_version = false;
 340   int option_index = 0;
 341   while (1) {
 342     int option_rv = getopt_long(argc, argv, "Vhvds:", long_options, &option_index);
 343
 344     if (option_rv == -1)
 345       break;
 346
 347     switch (option_rv) {
 348     case 0: break;
 349
 350     case OPT_UDP:
 351       if (opt_test == GET_TEST) {
 352         fprintf(stderr,
 353                 "You can not run a get test in UDP mode. UDP mode "
 354                 "does not currently support get ops.\n");
 355         exit(1);
 356       }
 357       opt_udp_io = true;
 358       break;
 359
 360     case OPT_BINARY: opt_binary = true; break;
 361
 362     case OPT_VERBOSE: /* --verbose or -v */ opt_verbose = OPT_VERBOSE; break;
 363
 364     case OPT_DEBUG: /* --debug or -d */ opt_verbose = OPT_DEBUG; break;
 365
 366     case OPT_VERSION: /* --version or -V */ opt_version = true; break;
 367
 368     case OPT_HELP: /* --help or -h */ opt_help = true; break;
 369
 370     case OPT_SERVERS: /* --servers or -s */ opt_servers = strdup(optarg); break;
 371
 372     case OPT_SLAP_TEST:
 373       if (strcmp(optarg, "get") == 0) {
 374         if (opt_udp_io == 1) {
 375           fprintf(stderr,
 376                   "You can not run a get test in UDP mode. UDP mode "
 377                   "does not currently support get ops.\n");
 378           exit(EXIT_FAILURE);
 379         }
 380         opt_test = GET_TEST;
 381       } else if (strcmp(optarg, "set") == 0) {
 382         opt_test = SET_TEST;
 383       } else if (strcmp(optarg, "mget") == 0) {
 384         opt_test = MGET_TEST;
 385       } else {
 386         fprintf(stderr, "Your test, %s, is not a known test\n", optarg);
 387         exit(EXIT_FAILURE);
 388       }
 389       break;
 390
 391     case OPT_SLAP_CONCURRENCY:
 392       errno = 0;
 393       opt_concurrency = (unsigned int) strtoul(optarg, (char **) NULL, 10);
 394       if (errno != 0) {
 395         fprintf(stderr, "Invalid value for concurrency: %s\n", optarg);
 396         exit(EXIT_FAILURE);
 397       }
 398       break;
 399
 400     case OPT_SLAP_EXECUTE_NUMBER:
 401       errno = 0;
 402       opt_execute_number = (unsigned int) strtoul(optarg, (char **) NULL, 10);
 403       if (errno != 0) {
 404         fprintf(stderr, "Invalid value for execute: %s\n", optarg);
 405         exit(EXIT_FAILURE);
 406       }
 407       break;
 408
 409     case OPT_SLAP_INITIAL_LOAD:
 410       errno = 0;
 411       opt_createial_load = (unsigned int) strtoul(optarg, (char **) NULL, 10);
 412       if (errno != 0) {
 413         fprintf(stderr, "Invalid value for initial load: %s\n", optarg);
 414         exit(EXIT_FAILURE);
 415       }
 416       break;
 417
 418     case OPT_QUIET: close_stdio(); break;
 419
 420     case '?':
 421       /* getopt_long already printed an error message. */
 422       exit(EXIT_FAILURE);
 423
 424     default: abort();
 425     }
 426   }
 427
 428   if (opt_version) {
 429     version_command(PROGRAM_NAME);
 430     exit(EXIT_SUCCESS);
 431   }
 432
 433   if (opt_help) {
 434     help_command(PROGRAM_NAME, PROGRAM_DESCRIPTION, long_options, help_options);
 435     exit(EXIT_SUCCESS);
 436   }
 437
 438   if ((opt_test == GET_TEST or opt_test == MGET_TEST) and opt_createial_load == 0)
 439     opt_createial_load = DEFAULT_INITIAL_LOAD;
 440
 441   if (opt_execute_number == 0)
 442     opt_execute_number = DEFAULT_EXECUTE_NUMBER;
 443
 444   if (opt_concurrency == 0)
 445     opt_concurrency = DEFAULT_CONCURRENCY;
 446 }
 447
 448 void conclusions_print(conclusions_st *conclusion) {
 449   printf("\tThreads connecting to servers %u\n", opt_concurrency);
 450 #ifdef NOT_FINISHED
 451   printf("\tLoaded %u rows\n", conclusion->rows_loaded);
 452   printf("\tRead %u rows\n", conclusion->rows_read);
 453 #endif
 454   if (opt_test == SET_TEST)
 455     printf("\tTook %ld.%03ld seconds to load data\n", conclusion->load_time / 1000,
 456            conclusion->load_time % 1000);
 457   else
 458     printf("\tTook %ld.%03ld seconds to read data\n", conclusion->read_time / 1000,
 459            conclusion->read_time % 1000);
 460 }
 461
 462 void flush_all(memcached_st *memc) {
 463   memcached_flush(memc, 0);
 464 }
 465
 466 pairs_st *load_create_data(memcached_st *memc, unsigned int number_of,
 467                            unsigned int *actual_loaded) {
 468   memcached_st *memc_clone = memcached_clone(NULL, memc);
 469   /* We always used non-blocking IO for load since it is faster */
 470   memcached_behavior_set(memc_clone, MEMCACHED_BEHAVIOR_NO_BLOCK, 0);
 471
 472   pairs_st *pairs = pairs_generate(number_of, VALUE_BYTES);
 473   *actual_loaded = execute_set(memc_clone, pairs, number_of);
 474
 475   memcached_free(memc_clone);
 476
 477   return pairs;
 478 }