git.m6w6.name Git - m6w6/libmemcached/blob - clients/memslap.c

   1 #include <stdio.h>
   2 #include <stdlib.h>
   3 #include <string.h>
   4 #include <sys/types.h>
   5 #include <sys/stat.h>
   6 #include <sys/types.h>
   7 #include <sys/mman.h>
   8 #include <fcntl.h>
   9 #include <sys/time.h>
  10 #include <getopt.h>
  11 #include <pthread.h>
  12
  13 #include <libmemcached/memcached.h>
  14
  15 #include "client_options.h"
  16 #include "utilities.h"
  17 #include "generator.h"
  18 #include "execute.h"
  19
  20 #define DEFAULT_INITIAL_LOAD 10000
  21 #define DEFAULT_EXECUTE_NUMBER 10000
  22 #define DEFAULT_CONCURRENCY 1
  23
  24 #define PROGRAM_NAME "memslap"
  25 #define PROGRAM_DESCRIPTION "Generates a load against a memcached custer of servers."
  26
  27 /* Global Thread counter */
  28 volatile unsigned int thread_counter;
  29 pthread_mutex_t counter_mutex;
  30 pthread_cond_t count_threshhold;
  31 volatile unsigned int master_wakeup;
  32 pthread_mutex_t sleeper_mutex;
  33 pthread_cond_t sleep_threshhold;
  34
  35 void *run_task(void *p);
  36
  37 /* Types */
  38 typedef struct conclusions_st conclusions_st;
  39 typedef struct thread_context_st thread_context_st;
  40 typedef enum {
  41   SET_TEST,
  42   GET_TEST,
  43 } test_type;
  44
  45 struct thread_context_st {
  46   unsigned int key_count;
  47   pairs_st *initial_pairs;
  48   unsigned int initial_number;
  49   pairs_st *execute_pairs;
  50   unsigned int execute_number;
  51   test_type test;
  52   memcached_st *memc;
  53 };
  54
  55 struct conclusions_st {
  56   long int load_time;
  57   long int read_time;
  58   unsigned int rows_loaded;
  59   unsigned int rows_read;
  60 };
  61
  62 /* Prototypes */
  63 void options_parse(int argc, char *argv[]);
  64 void conclusions_print(conclusions_st *conclusion);
  65 void scheduler(memcached_server_st *servers, conclusions_st *conclusion);
  66 pairs_st *load_create_data(memcached_st *memc, unsigned int number_of,
  67                            unsigned int *actual_loaded);
  68 void flush_all(memcached_st *memc);
  69
  70 static int opt_binary= 0;
  71 static int opt_verbose= 0;
  72 static int opt_flush= 0;
  73 static int opt_non_blocking_io= 0;
  74 static int opt_tcp_nodelay= 0;
  75 static unsigned int opt_execute_number= 0;
  76 static unsigned int opt_createial_load= 0;
  77 static unsigned int opt_concurrency= 0;
  78 static int opt_displayflag= 0;
  79 static char *opt_servers= NULL;
  80 test_type opt_test= SET_TEST;
  81
  82 int main(int argc, char *argv[])
  83 {
  84   conclusions_st conclusion;
  85   memcached_server_st *servers;
  86
  87   memset(&conclusion, 0, sizeof(conclusions_st));
  88
  89   srandom(time(NULL));
  90   options_parse(argc, argv);
  91
  92   if (!opt_servers)
  93   {
  94     char *temp;
  95
  96     if ((temp= getenv("MEMCACHED_SERVERS")))
  97       opt_servers= strdup(temp);
  98     else
  99     {
 100       fprintf(stderr, "No Servers provided\n");
 101       exit(1);
 102     }
 103   }
 104
 105   servers= memcached_servers_parse(opt_servers);
 106
 107   pthread_mutex_init(&counter_mutex, NULL);
 108   pthread_cond_init(&count_threshhold, NULL);
 109   pthread_mutex_init(&sleeper_mutex, NULL);
 110   pthread_cond_init(&sleep_threshhold, NULL);
 111
 112   scheduler(servers, &conclusion);
 113
 114   free(opt_servers);
 115
 116   (void)pthread_mutex_destroy(&counter_mutex);
 117   (void)pthread_cond_destroy(&count_threshhold);
 118   (void)pthread_mutex_destroy(&sleeper_mutex);
 119   (void)pthread_cond_destroy(&sleep_threshhold);
 120   conclusions_print(&conclusion);
 121   memcached_server_list_free(servers);
 122
 123   return 0;
 124 }
 125
 126 void scheduler(memcached_server_st *servers, conclusions_st *conclusion)
 127 {
 128   unsigned int x;
 129   unsigned int actual_loaded= 0; /* Fix warning */
 130   memcached_st *memc;
 131
 132   struct timeval start_time, end_time;
 133   pthread_t mainthread;            /* Thread descriptor */
 134   pthread_attr_t attr;          /* Thread attributes */
 135   pairs_st *pairs= NULL;
 136
 137   pthread_attr_init(&attr);
 138   pthread_attr_setdetachstate(&attr,
 139                               PTHREAD_CREATE_DETACHED);
 140
 141   memc= memcached_create(NULL);
 142   memcached_server_push(memc, servers);
 143
 144   memcached_behavior_set(memc, MEMCACHED_BEHAVIOR_BINARY_PROTOCOL, opt_binary);
 145
 146   if (opt_flush)
 147     flush_all(memc);
 148   if (opt_createial_load)
 149     pairs= load_create_data(memc, opt_createial_load, &actual_loaded);
 150
 151   /* We set this after we have loaded */
 152   {
 153     if (opt_non_blocking_io)
 154       memcached_behavior_set(memc, MEMCACHED_BEHAVIOR_NO_BLOCK, 1);
 155     if (opt_tcp_nodelay)
 156       memcached_behavior_set(memc, MEMCACHED_BEHAVIOR_TCP_NODELAY, 1);
 157   }
 158
 159
 160   pthread_mutex_lock(&counter_mutex);
 161   thread_counter= 0;
 162
 163   pthread_mutex_lock(&sleeper_mutex);
 164   master_wakeup= 1;
 165   pthread_mutex_unlock(&sleeper_mutex);
 166
 167   for (x= 0; x < opt_concurrency; x++)
 168   {
 169     thread_context_st *context;
 170     context= (thread_context_st *)malloc(sizeof(thread_context_st));
 171     memset(context, 0, sizeof(thread_context_st));
 172
 173     context->memc= memcached_clone(NULL, memc);
 174     context->test= opt_test;
 175
 176     context->initial_pairs= pairs;
 177     context->initial_number= actual_loaded;
 178
 179     if (opt_test == SET_TEST)
 180     {
 181       context->execute_pairs= pairs_generate(opt_execute_number, 400);
 182       context->execute_number= opt_execute_number;
 183     }
 184
 185     /* now you create the thread */
 186     if (pthread_create(&mainthread, &attr, run_task,
 187                        (void *)context) != 0)
 188     {
 189       fprintf(stderr,"Could not create thread\n");
 190       exit(1);
 191     }
 192     thread_counter++;
 193   }
 194
 195   pthread_mutex_unlock(&counter_mutex);
 196   pthread_attr_destroy(&attr);
 197
 198   pthread_mutex_lock(&sleeper_mutex);
 199   master_wakeup= 0;
 200   pthread_mutex_unlock(&sleeper_mutex);
 201   pthread_cond_broadcast(&sleep_threshhold);
 202
 203   gettimeofday(&start_time, NULL);
 204   /*
 205     We loop until we know that all children have cleaned up.
 206   */
 207   pthread_mutex_lock(&counter_mutex);
 208   while (thread_counter)
 209     pthread_cond_wait(&count_threshhold, &counter_mutex);
 210   pthread_mutex_unlock(&counter_mutex);
 211
 212   gettimeofday(&end_time, NULL);
 213
 214   conclusion->load_time= timedif(end_time, start_time);
 215   conclusion->read_time= timedif(end_time, start_time);
 216   pairs_free(pairs);
 217   memcached_free(memc);
 218 }
 219
 220 void options_parse(int argc, char *argv[])
 221 {
 222   memcached_programs_help_st help_options[]=
 223   {
 224     {0},
 225   };
 226
 227   static struct option long_options[]=
 228     {
 229       {"concurrency", required_argument, NULL, OPT_SLAP_CONCURRENCY},
 230       {"debug", no_argument, &opt_verbose, OPT_DEBUG},
 231       {"execute-number", required_argument, NULL, OPT_SLAP_EXECUTE_NUMBER},
 232       {"flag", no_argument, &opt_displayflag, OPT_FLAG},
 233       {"flush", no_argument, &opt_flush, OPT_FLUSH},
 234       {"help", no_argument, NULL, OPT_HELP},
 235       {"initial-load", required_argument, NULL, OPT_SLAP_INITIAL_LOAD}, /* Number to load initially */
 236       {"non-blocking", no_argument, &opt_non_blocking_io, OPT_SLAP_NON_BLOCK},
 237       {"servers", required_argument, NULL, OPT_SERVERS},
 238       {"tcp-nodelay", no_argument, &opt_tcp_nodelay, OPT_SLAP_TCP_NODELAY},
 239       {"test", required_argument, NULL, OPT_SLAP_TEST},
 240       {"verbose", no_argument, &opt_verbose, OPT_VERBOSE},
 241       {"version", no_argument, NULL, OPT_VERSION},
 242       {"binary", no_argument, NULL, OPT_BINARY},
 243       {0, 0, 0, 0},
 244     };
 245
 246   int option_index= 0;
 247   int option_rv;
 248
 249   while (1)
 250   {
 251     option_rv= getopt_long(argc, argv, "Vhvds:", long_options, &option_index);
 252     if (option_rv == -1) break;
 253     switch (option_rv)
 254     {
 255     case 0:
 256       break;
 257     case OPT_BINARY:
 258       opt_binary = 1;
 259       break;
 260     case OPT_VERBOSE: /* --verbose or -v */
 261       opt_verbose = OPT_VERBOSE;
 262       break;
 263     case OPT_DEBUG: /* --debug or -d */
 264       opt_verbose = OPT_DEBUG;
 265       break;
 266     case OPT_VERSION: /* --version or -V */
 267       version_command(PROGRAM_NAME);
 268       break;
 269     case OPT_HELP: /* --help or -h */
 270       help_command(PROGRAM_NAME, PROGRAM_DESCRIPTION, long_options, help_options);
 271       break;
 272     case OPT_SERVERS: /* --servers or -s */
 273       opt_servers= strdup(optarg);
 274       break;
 275     case OPT_SLAP_TEST:
 276       if (!strcmp(optarg, "get"))
 277         opt_test= GET_TEST ;
 278       else if (!strcmp(optarg, "set"))
 279         opt_test= SET_TEST;
 280       else
 281       {
 282         fprintf(stderr, "Your test, %s, is not a known test\n", optarg);
 283         exit(1);
 284       }
 285       break;
 286     case OPT_SLAP_CONCURRENCY:
 287       opt_concurrency= strtol(optarg, (char **)NULL, 10);
 288       break;
 289     case OPT_SLAP_EXECUTE_NUMBER:
 290       opt_execute_number= strtol(optarg, (char **)NULL, 10);
 291       break;
 292     case OPT_SLAP_INITIAL_LOAD:
 293       opt_createial_load= strtol(optarg, (char **)NULL, 10);
 294       break;
 295     case '?':
 296       /* getopt_long already printed an error message. */
 297       exit(1);
 298     default:
 299       abort();
 300     }
 301   }
 302
 303   if (opt_test == GET_TEST && opt_createial_load == 0)
 304     opt_createial_load= DEFAULT_INITIAL_LOAD;
 305
 306   if (opt_execute_number == 0)
 307     opt_execute_number= DEFAULT_EXECUTE_NUMBER;
 308
 309   if (opt_concurrency == 0)
 310     opt_concurrency= DEFAULT_CONCURRENCY;
 311 }
 312
 313 void conclusions_print(conclusions_st *conclusion)
 314 {
 315   printf("\tThreads connecting to servers %u\n", opt_concurrency);
 316 #ifdef NOT_FINISHED
 317   printf("\tLoaded %u rows\n", conclusion->rows_loaded);
 318   printf("\tRead %u rows\n", conclusion->rows_read);
 319 #endif
 320   if (opt_test == SET_TEST)
 321     printf("\tTook %ld.%03ld seconds to load data\n", conclusion->load_time / 1000,
 322            conclusion->load_time % 1000);
 323   else
 324     printf("\tTook %ld.%03ld seconds to read data\n", conclusion->read_time / 1000,
 325            conclusion->read_time % 1000);
 326 }
 327
 328 void *run_task(void *p)
 329 {
 330   thread_context_st *context= (thread_context_st *)p;
 331   memcached_st *memc;
 332
 333   memc= context->memc;
 334
 335   pthread_mutex_lock(&sleeper_mutex);
 336   while (master_wakeup)
 337   {
 338     pthread_cond_wait(&sleep_threshhold, &sleeper_mutex);
 339   }
 340   pthread_mutex_unlock(&sleeper_mutex);
 341
 342   /* Do Stuff */
 343   switch (context->test)
 344   {
 345   case SET_TEST:
 346     execute_set(memc, context->execute_pairs, context->execute_number);
 347     break;
 348   case GET_TEST:
 349     execute_get(memc, context->initial_pairs, context->initial_number);
 350     break;
 351   }
 352
 353   memcached_free(memc);
 354
 355   if (context->execute_pairs)
 356     pairs_free(context->execute_pairs);
 357
 358   free(context);
 359
 360   pthread_mutex_lock(&counter_mutex);
 361   thread_counter--;
 362   pthread_cond_signal(&count_threshhold);
 363   pthread_mutex_unlock(&counter_mutex);
 364
 365   return NULL;
 366 }
 367
 368 void flush_all(memcached_st *memc)
 369 {
 370   memcached_flush(memc, 0);
 371 }
 372
 373 pairs_st *load_create_data(memcached_st *memc, unsigned int number_of,
 374                            unsigned int *actual_loaded)
 375 {
 376   memcached_st *clone;
 377   pairs_st *pairs;
 378
 379   clone= memcached_clone(NULL, memc);
 380   /* We always used non-blocking IO for load since it is faster */
 381   memcached_behavior_set(clone, MEMCACHED_BEHAVIOR_NO_BLOCK, 0);
 382
 383   pairs= pairs_generate(number_of, 400);
 384   *actual_loaded= execute_set(clone, pairs, number_of);
 385
 386   memcached_free(clone);
 387
 388   return pairs;
 389 }