git.m6w6.name Git - awesomized/libmemcached/blob - clients/memslap.c

   1 #include "libmemcached/common.h"
   2 #include <stdio.h>
   3 #include <stdlib.h>
   4 #include <string.h>
   5 #include <sys/types.h>
   6 #include <sys/stat.h>
   7 #include <sys/types.h>
   8 #include <sys/mman.h>
   9 #include <fcntl.h>
  10 #include <sys/time.h>
  11 #include <getopt.h>
  12 #include <pthread.h>
  13
  14 #include <libmemcached/memcached.h>
  15
  16 #include "client_options.h"
  17 #include "utilities.h"
  18 #include "generator.h"
  19 #include "execute.h"
  20
  21 #define DEFAULT_INITIAL_LOAD 10000
  22 #define DEFAULT_EXECUTE_NUMBER 10000
  23 #define DEFAULT_CONCURRENCY 1
  24
  25 #define PROGRAM_NAME "memslap"
  26 #define PROGRAM_DESCRIPTION "Generates a load against a memcached custer of servers."
  27
  28 /* Global Thread counter */
  29 volatile unsigned int thread_counter;
  30 pthread_mutex_t counter_mutex;
  31 pthread_cond_t count_threshhold;
  32 volatile unsigned int master_wakeup;
  33 pthread_mutex_t sleeper_mutex;
  34 pthread_cond_t sleep_threshhold;
  35
  36 void *run_task(void *p);
  37
  38 /* Types */
  39 typedef struct conclusions_st conclusions_st;
  40 typedef struct thread_context_st thread_context_st;
  41 typedef enum {
  42   SET_TEST,
  43   GET_TEST,
  44 } test_type;
  45
  46 struct thread_context_st {
  47   unsigned int key_count;
  48   pairs_st *initial_pairs;
  49   unsigned int initial_number;
  50   pairs_st *execute_pairs;
  51   unsigned int execute_number;
  52   test_type test;
  53   memcached_st *memc;
  54 };
  55
  56 struct conclusions_st {
  57   long int load_time;
  58   long int read_time;
  59   unsigned int rows_loaded;
  60   unsigned int rows_read;
  61 };
  62
  63 /* Prototypes */
  64 void options_parse(int argc, char *argv[]);
  65 void conclusions_print(conclusions_st *conclusion);
  66 void scheduler(memcached_server_st *servers, conclusions_st *conclusion);
  67 pairs_st *load_create_data(memcached_st *memc, unsigned int number_of,
  68                            unsigned int *actual_loaded);
  69 void flush_all(memcached_st *memc);
  70
  71 static int opt_binary= 0;
  72 static int opt_verbose= 0;
  73 static int opt_flush= 0;
  74 static int opt_non_blocking_io= 0;
  75 static int opt_tcp_nodelay= 0;
  76 static unsigned int opt_execute_number= 0;
  77 static unsigned int opt_createial_load= 0;
  78 static unsigned int opt_concurrency= 0;
  79 static int opt_displayflag= 0;
  80 static char *opt_servers= NULL;
  81 static int opt_udp_io= 0;
  82 test_type opt_test= SET_TEST;
  83
  84 int main(int argc, char *argv[])
  85 {
  86   conclusions_st conclusion;
  87   memcached_server_st *servers;
  88
  89   memset(&conclusion, 0, sizeof(conclusions_st));
  90
  91   srandom(time(NULL));
  92   options_parse(argc, argv);
  93
  94   if (!opt_servers)
  95   {
  96     char *temp;
  97
  98     if ((temp= getenv("MEMCACHED_SERVERS")))
  99       opt_servers= strdup(temp);
 100     else
 101     {
 102       fprintf(stderr, "No Servers provided\n");
 103       exit(1);
 104     }
 105   }
 106
 107   servers= memcached_servers_parse(opt_servers);
 108
 109   pthread_mutex_init(&counter_mutex, NULL);
 110   pthread_cond_init(&count_threshhold, NULL);
 111   pthread_mutex_init(&sleeper_mutex, NULL);
 112   pthread_cond_init(&sleep_threshhold, NULL);
 113
 114   scheduler(servers, &conclusion);
 115
 116   free(opt_servers);
 117
 118   (void)pthread_mutex_destroy(&counter_mutex);
 119   (void)pthread_cond_destroy(&count_threshhold);
 120   (void)pthread_mutex_destroy(&sleeper_mutex);
 121   (void)pthread_cond_destroy(&sleep_threshhold);
 122   conclusions_print(&conclusion);
 123   memcached_server_list_free(servers);
 124
 125   return 0;
 126 }
 127
 128 void scheduler(memcached_server_st *servers, conclusions_st *conclusion)
 129 {
 130   unsigned int x;
 131   unsigned int actual_loaded= 0; /* Fix warning */
 132   memcached_st *memc;
 133
 134   struct timeval start_time, end_time;
 135   pthread_t mainthread;            /* Thread descriptor */
 136   pthread_attr_t attr;          /* Thread attributes */
 137   pairs_st *pairs= NULL;
 138
 139   pthread_attr_init(&attr);
 140   pthread_attr_setdetachstate(&attr,
 141                               PTHREAD_CREATE_DETACHED);
 142
 143   memc= memcached_create(NULL);
 144
 145   /* We need to set udp behavior before adding servers to the client */
 146   if (opt_udp_io)
 147   {
 148     memcached_behavior_set(memc, MEMCACHED_BEHAVIOR_USE_UDP, opt_udp_io);
 149     unsigned int x= 0;
 150     for(x= 0; x < servers[0].count; x++ )
 151       servers[x].type= MEMCACHED_CONNECTION_UDP;
 152   }
 153   memcached_server_push(memc, servers);
 154
 155   memcached_behavior_set(memc, MEMCACHED_BEHAVIOR_BINARY_PROTOCOL, opt_binary);
 156
 157   if (opt_flush)
 158     flush_all(memc);
 159   if (opt_createial_load)
 160     pairs= load_create_data(memc, opt_createial_load, &actual_loaded);
 161
 162   /* We set this after we have loaded */
 163   {
 164     if (opt_non_blocking_io)
 165       memcached_behavior_set(memc, MEMCACHED_BEHAVIOR_NO_BLOCK, 1);
 166     if (opt_tcp_nodelay)
 167       memcached_behavior_set(memc, MEMCACHED_BEHAVIOR_TCP_NODELAY, 1);
 168   }
 169
 170
 171   pthread_mutex_lock(&counter_mutex);
 172   thread_counter= 0;
 173
 174   pthread_mutex_lock(&sleeper_mutex);
 175   master_wakeup= 1;
 176   pthread_mutex_unlock(&sleeper_mutex);
 177
 178   for (x= 0; x < opt_concurrency; x++)
 179   {
 180     thread_context_st *context;
 181     context= (thread_context_st *)calloc(1, sizeof(thread_context_st));
 182
 183     context->memc= memcached_clone(NULL, memc);
 184     context->test= opt_test;
 185
 186     context->initial_pairs= pairs;
 187     context->initial_number= actual_loaded;
 188
 189     if (opt_test == SET_TEST)
 190     {
 191       context->execute_pairs= pairs_generate(opt_execute_number, 400);
 192       context->execute_number= opt_execute_number;
 193     }
 194
 195     /* now you create the thread */
 196     if (pthread_create(&mainthread, &attr, run_task,
 197                        (void *)context) != 0)
 198     {
 199       fprintf(stderr,"Could not create thread\n");
 200       exit(1);
 201     }
 202     thread_counter++;
 203   }
 204
 205   pthread_mutex_unlock(&counter_mutex);
 206   pthread_attr_destroy(&attr);
 207
 208   pthread_mutex_lock(&sleeper_mutex);
 209   master_wakeup= 0;
 210   pthread_mutex_unlock(&sleeper_mutex);
 211   pthread_cond_broadcast(&sleep_threshhold);
 212
 213   gettimeofday(&start_time, NULL);
 214   /*
 215     We loop until we know that all children have cleaned up.
 216   */
 217   pthread_mutex_lock(&counter_mutex);
 218   while (thread_counter)
 219     pthread_cond_wait(&count_threshhold, &counter_mutex);
 220   pthread_mutex_unlock(&counter_mutex);
 221
 222   gettimeofday(&end_time, NULL);
 223
 224   conclusion->load_time= timedif(end_time, start_time);
 225   conclusion->read_time= timedif(end_time, start_time);
 226   pairs_free(pairs);
 227   memcached_free(memc);
 228 }
 229
 230 void options_parse(int argc, char *argv[])
 231 {
 232   memcached_programs_help_st help_options[]=
 233   {
 234     {0},
 235   };
 236
 237   static struct option long_options[]=
 238     {
 239       {"concurrency", required_argument, NULL, OPT_SLAP_CONCURRENCY},
 240       {"debug", no_argument, &opt_verbose, OPT_DEBUG},
 241       {"execute-number", required_argument, NULL, OPT_SLAP_EXECUTE_NUMBER},
 242       {"flag", no_argument, &opt_displayflag, OPT_FLAG},
 243       {"flush", no_argument, &opt_flush, OPT_FLUSH},
 244       {"help", no_argument, NULL, OPT_HELP},
 245       {"initial-load", required_argument, NULL, OPT_SLAP_INITIAL_LOAD}, /* Number to load initially */
 246       {"non-blocking", no_argument, &opt_non_blocking_io, OPT_SLAP_NON_BLOCK},
 247       {"servers", required_argument, NULL, OPT_SERVERS},
 248       {"tcp-nodelay", no_argument, &opt_tcp_nodelay, OPT_SLAP_TCP_NODELAY},
 249       {"test", required_argument, NULL, OPT_SLAP_TEST},
 250       {"verbose", no_argument, &opt_verbose, OPT_VERBOSE},
 251       {"version", no_argument, NULL, OPT_VERSION},
 252       {"binary", no_argument, NULL, OPT_BINARY},
 253       {"udp", no_argument, NULL, OPT_UDP},
 254       {0, 0, 0, 0},
 255     };
 256
 257   int option_index= 0;
 258   int option_rv;
 259
 260   while (1)
 261   {
 262     option_rv= getopt_long(argc, argv, "Vhvds:", long_options, &option_index);
 263     if (option_rv == -1) break;
 264     switch (option_rv)
 265     {
 266     case 0:
 267       break;
 268     case OPT_UDP:
 269       if (opt_test == GET_TEST)
 270       {
 271         fprintf(stderr, "You can not run a get test in UDP mode. UDP mode "
 272                   "does not currently support get ops.\n");
 273         exit(1);
 274       }
 275       opt_udp_io= 1;
 276       break;
 277     case OPT_BINARY:
 278       opt_binary = 1;
 279       break;
 280     case OPT_VERBOSE: /* --verbose or -v */
 281       opt_verbose = OPT_VERBOSE;
 282       break;
 283     case OPT_DEBUG: /* --debug or -d */
 284       opt_verbose = OPT_DEBUG;
 285       break;
 286     case OPT_VERSION: /* --version or -V */
 287       version_command(PROGRAM_NAME);
 288       break;
 289     case OPT_HELP: /* --help or -h */
 290       help_command(PROGRAM_NAME, PROGRAM_DESCRIPTION, long_options, help_options);
 291       break;
 292     case OPT_SERVERS: /* --servers or -s */
 293       opt_servers= strdup(optarg);
 294       break;
 295     case OPT_SLAP_TEST:
 296       if (!strcmp(optarg, "get"))
 297       {
 298         if (opt_udp_io == 1)
 299         {
 300           fprintf(stderr, "You can not run a get test in UDP mode. UDP mode "
 301                   "does not currently support get ops.\n");
 302           exit(1);
 303         }
 304         opt_test= GET_TEST ;
 305       }
 306       else if (!strcmp(optarg, "set"))
 307         opt_test= SET_TEST;
 308       else
 309       {
 310         fprintf(stderr, "Your test, %s, is not a known test\n", optarg);
 311         exit(1);
 312       }
 313       break;
 314     case OPT_SLAP_CONCURRENCY:
 315       opt_concurrency= strtol(optarg, (char **)NULL, 10);
 316       break;
 317     case OPT_SLAP_EXECUTE_NUMBER:
 318       opt_execute_number= strtol(optarg, (char **)NULL, 10);
 319       break;
 320     case OPT_SLAP_INITIAL_LOAD:
 321       opt_createial_load= strtol(optarg, (char **)NULL, 10);
 322       break;
 323     case '?':
 324       /* getopt_long already printed an error message. */
 325       exit(1);
 326     default:
 327       abort();
 328     }
 329   }
 330
 331   if (opt_test == GET_TEST && opt_createial_load == 0)
 332     opt_createial_load= DEFAULT_INITIAL_LOAD;
 333
 334   if (opt_execute_number == 0)
 335     opt_execute_number= DEFAULT_EXECUTE_NUMBER;
 336
 337   if (opt_concurrency == 0)
 338     opt_concurrency= DEFAULT_CONCURRENCY;
 339 }
 340
 341 void conclusions_print(conclusions_st *conclusion)
 342 {
 343   printf("\tThreads connecting to servers %u\n", opt_concurrency);
 344 #ifdef NOT_FINISHED
 345   printf("\tLoaded %u rows\n", conclusion->rows_loaded);
 346   printf("\tRead %u rows\n", conclusion->rows_read);
 347 #endif
 348   if (opt_test == SET_TEST)
 349     printf("\tTook %ld.%03ld seconds to load data\n", conclusion->load_time / 1000,
 350            conclusion->load_time % 1000);
 351   else
 352     printf("\tTook %ld.%03ld seconds to read data\n", conclusion->read_time / 1000,
 353            conclusion->read_time % 1000);
 354 }
 355
 356 void *run_task(void *p)
 357 {
 358   thread_context_st *context= (thread_context_st *)p;
 359   memcached_st *memc;
 360
 361   memc= context->memc;
 362
 363   pthread_mutex_lock(&sleeper_mutex);
 364   while (master_wakeup)
 365   {
 366     pthread_cond_wait(&sleep_threshhold, &sleeper_mutex);
 367   }
 368   pthread_mutex_unlock(&sleeper_mutex);
 369
 370   /* Do Stuff */
 371   switch (context->test)
 372   {
 373   case SET_TEST:
 374     execute_set(memc, context->execute_pairs, context->execute_number);
 375     break;
 376   case GET_TEST:
 377     execute_get(memc, context->initial_pairs, context->initial_number);
 378     break;
 379   }
 380
 381   memcached_free(memc);
 382
 383   if (context->execute_pairs)
 384     pairs_free(context->execute_pairs);
 385
 386   free(context);
 387
 388   pthread_mutex_lock(&counter_mutex);
 389   thread_counter--;
 390   pthread_cond_signal(&count_threshhold);
 391   pthread_mutex_unlock(&counter_mutex);
 392
 393   return NULL;
 394 }
 395
 396 void flush_all(memcached_st *memc)
 397 {
 398   memcached_flush(memc, 0);
 399 }
 400
 401 pairs_st *load_create_data(memcached_st *memc, unsigned int number_of,
 402                            unsigned int *actual_loaded)
 403 {
 404   memcached_st *clone;
 405   pairs_st *pairs;
 406
 407   clone= memcached_clone(NULL, memc);
 408   /* We always used non-blocking IO for load since it is faster */
 409   memcached_behavior_set(clone, MEMCACHED_BEHAVIOR_NO_BLOCK, 0);
 410
 411   pairs= pairs_generate(number_of, 400);
 412   *actual_loaded= execute_set(clone, pairs, number_of);
 413
 414   memcached_free(clone);
 415
 416   return pairs;
 417 }