Updating for 1.0.2 release
[awesomized/libmemcached] / clients / memslap.cc
1 /* vim:expandtab:shiftwidth=2:tabstop=2:smarttab:
2 *
3 * Libmemcached library
4 *
5 * Copyright (C) 2011 Data Differential, http://datadifferential.com/
6 * Copyright (C) 2006-2009 Brian Aker All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions are
10 * met:
11 *
12 * * Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 *
15 * * Redistributions in binary form must reproduce the above
16 * copyright notice, this list of conditions and the following disclaimer
17 * in the documentation and/or other materials provided with the
18 * distribution.
19 *
20 * * The names of its contributors may not be used to endorse or
21 * promote products derived from this software without specific prior
22 * written permission.
23 *
24 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
25 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
26 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
27 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
28 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
29 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
30 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
31 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
32 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
33 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
34 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
35 *
36 */
37
38
39 #include <config.h>
40
41 #include <cassert>
42 #include <cstdio>
43 #include <cstdlib>
44 #include <cstring>
45 #include <fcntl.h>
46 #include <getopt.h>
47 #include <memory>
48 #include <pthread.h>
49 #include <sys/mman.h>
50 #include <sys/stat.h>
51 #include <sys/time.h>
52 #include <sys/types.h>
53 #include <sys/types.h>
54 #include <unistd.h>
55
56 #include <libmemcached/memcached.h>
57
58 #include "client_options.h"
59 #include "utilities.h"
60 #include "generator.h"
61 #include "execute.h"
62
63 #define DEFAULT_INITIAL_LOAD 10000
64 #define DEFAULT_EXECUTE_NUMBER 10000
65 #define DEFAULT_CONCURRENCY 1
66
67 #define PROGRAM_NAME "memslap"
68 #define PROGRAM_DESCRIPTION "Generates a load against a memcached custer of servers."
69
70 /* Global Thread counter */
71 volatile unsigned int master_wakeup;
72 pthread_mutex_t sleeper_mutex;
73 pthread_cond_t sleep_threshhold;
74
75 /* Types */
76 enum test_t {
77 SET_TEST,
78 GET_TEST,
79 MGET_TEST
80 };
81
82 struct thread_context_st {
83 unsigned int key_count;
84 pairs_st *initial_pairs;
85 unsigned int initial_number;
86 pairs_st *execute_pairs;
87 unsigned int execute_number;
88 char **keys;
89 size_t *key_lengths;
90 test_t test;
91 memcached_st *memc;
92 const memcached_st* root;
93
94 thread_context_st(const memcached_st* memc_arg, test_t test_arg) :
95 key_count(0),
96 initial_pairs(NULL),
97 initial_number(0),
98 execute_pairs(NULL),
99 execute_number(0),
100 keys(0),
101 key_lengths(NULL),
102 test(test_arg),
103 memc(NULL),
104 root(memc_arg)
105 {
106 }
107
108 void init()
109 {
110 memc= memcached_clone(NULL, root);
111 }
112
113 ~thread_context_st()
114 {
115 if (execute_pairs)
116 {
117 pairs_free(execute_pairs);
118 }
119 memcached_free(memc);
120 }
121 };
122
123 struct conclusions_st {
124 long int load_time;
125 long int read_time;
126 unsigned int rows_loaded;
127 unsigned int rows_read;
128
129 conclusions_st() :
130 load_time(0),
131 read_time(0),
132 rows_loaded(0),
133 rows_read()
134 { }
135 };
136
137 /* Prototypes */
138 void options_parse(int argc, char *argv[]);
139 void conclusions_print(conclusions_st *conclusion);
140 void scheduler(memcached_server_st *servers, conclusions_st *conclusion);
141 pairs_st *load_create_data(memcached_st *memc, unsigned int number_of,
142 unsigned int *actual_loaded);
143 void flush_all(memcached_st *memc);
144
145 static bool opt_binary= 0;
146 static int opt_verbose= 0;
147 static int opt_flush= 0;
148 static int opt_non_blocking_io= 0;
149 static int opt_tcp_nodelay= 0;
150 static unsigned int opt_execute_number= 0;
151 static unsigned int opt_createial_load= 0;
152 static unsigned int opt_concurrency= 0;
153 static int opt_displayflag= 0;
154 static char *opt_servers= NULL;
155 static int opt_udp_io= 0;
156 test_t opt_test= SET_TEST;
157
158 extern "C" {
159
160 static void *run_task(void *p)
161 {
162 thread_context_st *context= (thread_context_st *)p;
163
164 context->init();
165
166 pthread_mutex_lock(&sleeper_mutex);
167 while (master_wakeup)
168 {
169 pthread_cond_wait(&sleep_threshhold, &sleeper_mutex);
170 }
171 pthread_mutex_unlock(&sleeper_mutex);
172
173 /* Do Stuff */
174 switch (context->test)
175 {
176 case SET_TEST:
177 assert(context->execute_pairs);
178 execute_set(context->memc, context->execute_pairs, context->execute_number);
179 break;
180
181 case GET_TEST:
182 execute_get(context->memc, context->initial_pairs, context->initial_number);
183 break;
184
185 case MGET_TEST:
186 execute_mget(context->memc, (const char*const*)context->keys, context->key_lengths, context->initial_number);
187 break;
188 }
189
190 delete context;
191
192 pthread_exit(0);
193 }
194
195 }
196
197
198 int main(int argc, char *argv[])
199 {
200 conclusions_st conclusion;
201
202 srandom((unsigned int)time(NULL));
203 options_parse(argc, argv);
204
205 if (opt_servers == NULL)
206 {
207 char *temp;
208
209 if ((temp= getenv("MEMCACHED_SERVERS")))
210 {
211 opt_servers= strdup(temp);
212 }
213 else
214 {
215 fprintf(stderr, "No Servers provided\n");
216 return EXIT_FAILURE;
217 }
218 }
219
220 memcached_server_st *servers= memcached_servers_parse(opt_servers);
221
222 pthread_mutex_init(&sleeper_mutex, NULL);
223 pthread_cond_init(&sleep_threshhold, NULL);
224
225 scheduler(servers, &conclusion);
226
227 free(opt_servers);
228
229 (void)pthread_mutex_destroy(&sleeper_mutex);
230 (void)pthread_cond_destroy(&sleep_threshhold);
231 conclusions_print(&conclusion);
232 memcached_server_list_free(servers);
233
234 return EXIT_SUCCESS;
235 }
236
237 void scheduler(memcached_server_st *servers, conclusions_st *conclusion)
238 {
239 unsigned int actual_loaded= 0; /* Fix warning */
240
241 struct timeval start_time, end_time;
242 pairs_st *pairs= NULL;
243
244 memcached_st *memc= memcached_create(NULL);
245
246 /* We need to set udp behavior before adding servers to the client */
247 if (opt_udp_io)
248 {
249 memcached_behavior_set(memc, MEMCACHED_BEHAVIOR_USE_UDP,
250 (uint64_t)opt_udp_io);
251 for (uint32_t x= 0; x < memcached_server_list_count(servers); x++ )
252 {
253 servers[x].type= MEMCACHED_CONNECTION_UDP;
254 }
255 }
256 memcached_server_push(memc, servers);
257
258 memcached_behavior_set(memc, MEMCACHED_BEHAVIOR_BINARY_PROTOCOL,
259 (uint64_t)opt_binary);
260
261 if (opt_flush)
262 flush_all(memc);
263
264 if (opt_createial_load)
265 pairs= load_create_data(memc, opt_createial_load, &actual_loaded);
266
267 char **keys= static_cast<char **>(calloc(actual_loaded, sizeof(char*)));
268 size_t *key_lengths= static_cast<size_t *>(calloc(actual_loaded, sizeof(size_t)));
269
270 if (keys == NULL or key_lengths == NULL)
271 {
272 free(keys);
273 free(key_lengths);
274 keys= NULL;
275 key_lengths= NULL;
276 }
277 else
278 {
279 for (uint32_t x= 0; x < actual_loaded; ++x)
280 {
281 keys[x]= pairs[x].key;
282 key_lengths[x]= pairs[x].key_length;
283 }
284 }
285
286 /* We set this after we have loaded */
287 {
288 if (opt_non_blocking_io)
289 memcached_behavior_set(memc, MEMCACHED_BEHAVIOR_NO_BLOCK, 1);
290
291 if (opt_tcp_nodelay)
292 memcached_behavior_set(memc, MEMCACHED_BEHAVIOR_TCP_NODELAY, 1);
293 }
294
295 pthread_mutex_lock(&sleeper_mutex);
296 master_wakeup= 1;
297 pthread_mutex_unlock(&sleeper_mutex);
298
299 pthread_t *threads= new (std::nothrow) pthread_t[opt_concurrency];
300
301 if (not threads)
302 {
303 exit(EXIT_FAILURE);
304 }
305
306 for (uint32_t x= 0; x < opt_concurrency; x++)
307 {
308 thread_context_st *context= new thread_context_st(memc, opt_test);
309 context->test= opt_test;
310
311 context->initial_pairs= pairs;
312 context->initial_number= actual_loaded;
313 context->keys= keys;
314 context->key_lengths= key_lengths;
315
316 if (opt_test == SET_TEST)
317 {
318 context->execute_pairs= pairs_generate(opt_execute_number, 400);
319 context->execute_number= opt_execute_number;
320 }
321
322 /* now you create the thread */
323 if (pthread_create(threads +x, NULL, run_task, (void *)context) != 0)
324 {
325 fprintf(stderr,"Could not create thread\n");
326 exit(1);
327 }
328 }
329
330 pthread_mutex_lock(&sleeper_mutex);
331 master_wakeup= 0;
332 pthread_mutex_unlock(&sleeper_mutex);
333 pthread_cond_broadcast(&sleep_threshhold);
334 gettimeofday(&start_time, NULL);
335
336 for (uint32_t x= 0; x < opt_concurrency; x++)
337 {
338 void *retval;
339 pthread_join(threads[x], &retval);
340 }
341 delete [] threads;
342
343 gettimeofday(&end_time, NULL);
344
345 conclusion->load_time= timedif(end_time, start_time);
346 conclusion->read_time= timedif(end_time, start_time);
347 free(keys);
348 free(key_lengths);
349 pairs_free(pairs);
350 memcached_free(memc);
351 }
352
353 void options_parse(int argc, char *argv[])
354 {
355 memcached_programs_help_st help_options[]=
356 {
357 {0},
358 };
359
360 static struct option long_options[]=
361 {
362 {(OPTIONSTRING)"concurrency", required_argument, NULL, OPT_SLAP_CONCURRENCY},
363 {(OPTIONSTRING)"debug", no_argument, &opt_verbose, OPT_DEBUG},
364 {(OPTIONSTRING)"quiet", no_argument, NULL, OPT_QUIET},
365 {(OPTIONSTRING)"execute-number", required_argument, NULL, OPT_SLAP_EXECUTE_NUMBER},
366 {(OPTIONSTRING)"flag", no_argument, &opt_displayflag, OPT_FLAG},
367 {(OPTIONSTRING)"flush", no_argument, &opt_flush, OPT_FLUSH},
368 {(OPTIONSTRING)"help", no_argument, NULL, OPT_HELP},
369 {(OPTIONSTRING)"initial-load", required_argument, NULL, OPT_SLAP_INITIAL_LOAD}, /* Number to load initially */
370 {(OPTIONSTRING)"non-blocking", no_argument, &opt_non_blocking_io, OPT_SLAP_NON_BLOCK},
371 {(OPTIONSTRING)"servers", required_argument, NULL, OPT_SERVERS},
372 {(OPTIONSTRING)"tcp-nodelay", no_argument, &opt_tcp_nodelay, OPT_SLAP_TCP_NODELAY},
373 {(OPTIONSTRING)"test", required_argument, NULL, OPT_SLAP_TEST},
374 {(OPTIONSTRING)"verbose", no_argument, &opt_verbose, OPT_VERBOSE},
375 {(OPTIONSTRING)"version", no_argument, NULL, OPT_VERSION},
376 {(OPTIONSTRING)"binary", no_argument, NULL, OPT_BINARY},
377 {(OPTIONSTRING)"udp", no_argument, NULL, OPT_UDP},
378 {0, 0, 0, 0},
379 };
380
381 bool opt_help= false;
382 bool opt_version= false;
383 int option_index= 0;
384 while (1)
385 {
386 int option_rv= getopt_long(argc, argv, "Vhvds:", long_options, &option_index);
387
388 if (option_rv == -1) break;
389
390 switch (option_rv)
391 {
392 case 0:
393 break;
394
395 case OPT_UDP:
396 if (opt_test == GET_TEST)
397 {
398 fprintf(stderr, "You can not run a get test in UDP mode. UDP mode "
399 "does not currently support get ops.\n");
400 exit(1);
401 }
402 opt_udp_io= 1;
403 break;
404
405 case OPT_BINARY:
406 opt_binary= true;
407 break;
408
409 case OPT_VERBOSE: /* --verbose or -v */
410 opt_verbose= OPT_VERBOSE;
411 break;
412
413 case OPT_DEBUG: /* --debug or -d */
414 opt_verbose = OPT_DEBUG;
415 break;
416
417 case OPT_VERSION: /* --version or -V */
418 opt_version= true;
419 break;
420
421 case OPT_HELP: /* --help or -h */
422 opt_help= true;
423 break;
424
425 case OPT_SERVERS: /* --servers or -s */
426 opt_servers= strdup(optarg);
427 break;
428
429 case OPT_SLAP_TEST:
430 if (strcmp(optarg, "get") == 0)
431 {
432 if (opt_udp_io == 1)
433 {
434 fprintf(stderr, "You can not run a get test in UDP mode. UDP mode "
435 "does not currently support get ops.\n");
436 exit(EXIT_FAILURE);
437 }
438 opt_test= GET_TEST ;
439 }
440 else if (strcmp(optarg, "set") == 0)
441 {
442 opt_test= SET_TEST;
443 }
444 else if (strcmp(optarg, "mget") == 0)
445 {
446 opt_test= MGET_TEST;
447 }
448 else
449 {
450 fprintf(stderr, "Your test, %s, is not a known test\n", optarg);
451 exit(EXIT_FAILURE);
452 }
453 break;
454
455 case OPT_SLAP_CONCURRENCY:
456 opt_concurrency= (unsigned int)strtoul(optarg, (char **)NULL, 10);
457 break;
458
459 case OPT_SLAP_EXECUTE_NUMBER:
460 opt_execute_number= (unsigned int)strtoul(optarg, (char **)NULL, 10);
461 break;
462
463 case OPT_SLAP_INITIAL_LOAD:
464 opt_createial_load= (unsigned int)strtoul(optarg, (char **)NULL, 10);
465 break;
466
467 case OPT_QUIET:
468 close_stdio();
469 break;
470
471
472 case '?':
473 /* getopt_long already printed an error message. */
474 exit(EXIT_FAILURE);
475
476 default:
477 abort();
478 }
479 }
480
481 if (opt_version)
482 {
483 version_command(PROGRAM_NAME);
484 exit(EXIT_SUCCESS);
485 }
486
487 if (opt_help)
488 {
489 help_command(PROGRAM_NAME, PROGRAM_DESCRIPTION, long_options, help_options);
490 exit(EXIT_SUCCESS);
491 }
492
493 if ((opt_test == GET_TEST or opt_test == MGET_TEST) and opt_createial_load == 0)
494 opt_createial_load= DEFAULT_INITIAL_LOAD;
495
496 if (opt_execute_number == 0)
497 opt_execute_number= DEFAULT_EXECUTE_NUMBER;
498
499 if (opt_concurrency == 0)
500 opt_concurrency= DEFAULT_CONCURRENCY;
501 }
502
503 void conclusions_print(conclusions_st *conclusion)
504 {
505 printf("\tThreads connecting to servers %u\n", opt_concurrency);
506 #ifdef NOT_FINISHED
507 printf("\tLoaded %u rows\n", conclusion->rows_loaded);
508 printf("\tRead %u rows\n", conclusion->rows_read);
509 #endif
510 if (opt_test == SET_TEST)
511 printf("\tTook %ld.%03ld seconds to load data\n", conclusion->load_time / 1000,
512 conclusion->load_time % 1000);
513 else
514 printf("\tTook %ld.%03ld seconds to read data\n", conclusion->read_time / 1000,
515 conclusion->read_time % 1000);
516 }
517
518 void flush_all(memcached_st *memc)
519 {
520 memcached_flush(memc, 0);
521 }
522
523 pairs_st *load_create_data(memcached_st *memc, unsigned int number_of,
524 unsigned int *actual_loaded)
525 {
526 memcached_st *memc_clone= memcached_clone(NULL, memc);
527 /* We always used non-blocking IO for load since it is faster */
528 memcached_behavior_set(memc_clone, MEMCACHED_BEHAVIOR_NO_BLOCK, 0);
529
530 pairs_st *pairs= pairs_generate(number_of, 400);
531 *actual_loaded= execute_set(memc_clone, pairs, number_of);
532
533 memcached_free(memc_clone);
534
535 return pairs;
536 }