Update for mavericks
[m6w6/libmemcached] / clients / memslap.cc
1 /* vim:expandtab:shiftwidth=2:tabstop=2:smarttab:
2 *
3 * Libmemcached library
4 *
5 * Copyright (C) 2011-2012 Data Differential, http://datadifferential.com/
6 * Copyright (C) 2006-2009 Brian Aker All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions are
10 * met:
11 *
12 * * Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 *
15 * * Redistributions in binary form must reproduce the above
16 * copyright notice, this list of conditions and the following disclaimer
17 * in the documentation and/or other materials provided with the
18 * distribution.
19 *
20 * * The names of its contributors may not be used to endorse or
21 * promote products derived from this software without specific prior
22 * written permission.
23 *
24 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
25 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
26 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
27 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
28 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
29 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
30 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
31 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
32 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
33 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
34 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
35 *
36 */
37
38
39 #include <mem_config.h>
40
41 #include <cassert>
42 #include <cerrno>
43 #include <cstdio>
44 #include <cstdlib>
45 #include <cstring>
46 #include <fcntl.h>
47 #include <getopt.h>
48 #include <memory>
49 #include <pthread.h>
50 #include <sys/mman.h>
51 #include <sys/stat.h>
52 #include <sys/time.h>
53 #include <sys/types.h>
54 #include <unistd.h>
55
56 #include <iostream>
57
58 #include <libmemcached-1.0/memcached.h>
59
60 #include "client_options.h"
61 #include "utilities.h"
62 #include "generator.h"
63 #include "execute.h"
64
65 #define DEFAULT_INITIAL_LOAD 10000
66 #define DEFAULT_EXECUTE_NUMBER 10000
67 #define DEFAULT_CONCURRENCY 1
68
69 #define VALUE_BYTES 4096
70
71 #define PROGRAM_NAME "memslap"
72 #define PROGRAM_DESCRIPTION "Generates a load against a memcached custer of servers."
73
74 /* Global Thread counter */
75 volatile unsigned int master_wakeup;
76 pthread_mutex_t sleeper_mutex;
77 pthread_cond_t sleep_threshhold;
78
79 /* Types */
80 enum test_t {
81 SET_TEST,
82 GET_TEST,
83 MGET_TEST
84 };
85
86 struct thread_context_st {
87 unsigned int key_count;
88 pairs_st *initial_pairs;
89 unsigned int initial_number;
90 pairs_st *execute_pairs;
91 unsigned int execute_number;
92 char **keys;
93 size_t *key_lengths;
94 test_t test;
95 memcached_st *memc;
96 const memcached_st* root;
97
98 thread_context_st(const memcached_st* memc_arg, test_t test_arg) :
99 key_count(0),
100 initial_pairs(NULL),
101 initial_number(0),
102 execute_pairs(NULL),
103 execute_number(0),
104 keys(0),
105 key_lengths(NULL),
106 test(test_arg),
107 memc(NULL),
108 root(memc_arg)
109 {
110 }
111
112 void init()
113 {
114 memc= memcached_clone(NULL, root);
115 }
116
117 ~thread_context_st()
118 {
119 if (execute_pairs)
120 {
121 pairs_free(execute_pairs);
122 }
123 memcached_free(memc);
124 }
125 };
126
127 struct conclusions_st {
128 long int load_time;
129 long int read_time;
130 unsigned int rows_loaded;
131 unsigned int rows_read;
132
133 conclusions_st() :
134 load_time(0),
135 read_time(0),
136 rows_loaded(0),
137 rows_read()
138 { }
139 };
140
141 /* Prototypes */
142 void options_parse(int argc, char *argv[]);
143 void conclusions_print(conclusions_st *conclusion);
144 void scheduler(memcached_server_st *servers, conclusions_st *conclusion);
145 pairs_st *load_create_data(memcached_st *memc, unsigned int number_of,
146 unsigned int *actual_loaded);
147 void flush_all(memcached_st *memc);
148
149 static bool opt_binary= 0;
150 static int opt_verbose= 0;
151 static int opt_flush= 0;
152 static int opt_non_blocking_io= 0;
153 static int opt_tcp_nodelay= 0;
154 static unsigned int opt_execute_number= 0;
155 static unsigned int opt_createial_load= 0;
156 static unsigned int opt_concurrency= 0;
157 static int opt_displayflag= 0;
158 static char *opt_servers= NULL;
159 static bool opt_udp_io= false;
160 test_t opt_test= SET_TEST;
161
162 extern "C" {
163
164 static __attribute__((noreturn)) void *run_task(void *p)
165 {
166 thread_context_st *context= (thread_context_st *)p;
167
168 context->init();
169
170 pthread_mutex_lock(&sleeper_mutex);
171 while (master_wakeup)
172 {
173 pthread_cond_wait(&sleep_threshhold, &sleeper_mutex);
174 }
175 pthread_mutex_unlock(&sleeper_mutex);
176
177 /* Do Stuff */
178 switch (context->test)
179 {
180 case SET_TEST:
181 assert(context->execute_pairs);
182 execute_set(context->memc, context->execute_pairs, context->execute_number);
183 break;
184
185 case GET_TEST:
186 execute_get(context->memc, context->initial_pairs, context->initial_number);
187 break;
188
189 case MGET_TEST:
190 execute_mget(context->memc, (const char*const*)context->keys, context->key_lengths, context->initial_number);
191 break;
192 }
193
194 delete context;
195
196 pthread_exit(0);
197 }
198
199 }
200
201
202 int main(int argc, char *argv[])
203 {
204 conclusions_st conclusion;
205
206 srandom((unsigned int)time(NULL));
207 options_parse(argc, argv);
208
209 if (opt_servers == NULL)
210 {
211 char *temp;
212
213 if ((temp= getenv("MEMCACHED_SERVERS")))
214 {
215 opt_servers= strdup(temp);
216 }
217
218 if (opt_servers == NULL)
219 {
220 std::cerr << "No Servers provided" << std::endl;
221 exit(EXIT_FAILURE);
222 }
223 }
224
225 memcached_server_st *servers= memcached_servers_parse(opt_servers);
226 if (servers == NULL or memcached_server_list_count(servers) == 0)
227 {
228 std::cerr << "Invalid server list provided:" << opt_servers << std::endl;
229 return EXIT_FAILURE;
230 }
231
232 pthread_mutex_init(&sleeper_mutex, NULL);
233 pthread_cond_init(&sleep_threshhold, NULL);
234
235 int error_code= EXIT_SUCCESS;
236 try {
237 scheduler(servers, &conclusion);
238 }
239 catch(std::exception& e)
240 {
241 std::cerr << "Died with exception: " << e.what() << std::endl;
242 error_code= EXIT_FAILURE;
243 }
244
245 free(opt_servers);
246
247 (void)pthread_mutex_destroy(&sleeper_mutex);
248 (void)pthread_cond_destroy(&sleep_threshhold);
249 conclusions_print(&conclusion);
250 memcached_server_list_free(servers);
251
252 return error_code;
253 }
254
255 void scheduler(memcached_server_st *servers, conclusions_st *conclusion)
256 {
257 unsigned int actual_loaded= 0; /* Fix warning */
258
259 struct timeval start_time, end_time;
260 pairs_st *pairs= NULL;
261
262 memcached_st *memc= memcached_create(NULL);
263
264 memcached_server_push(memc, servers);
265
266 /* We need to set udp behavior before adding servers to the client */
267 if (opt_udp_io)
268 {
269 if (memcached_failed(memcached_behavior_set(memc, MEMCACHED_BEHAVIOR_USE_UDP, opt_udp_io)))
270 {
271 std::cerr << "Failed to enable UDP." << std::endl;
272 memcached_free(memc);
273 exit(EXIT_FAILURE);
274 }
275 }
276
277 memcached_behavior_set(memc, MEMCACHED_BEHAVIOR_BINARY_PROTOCOL,
278 (uint64_t)opt_binary);
279
280 if (opt_flush)
281 {
282 flush_all(memc);
283 }
284
285 if (opt_createial_load)
286 {
287 pairs= load_create_data(memc, opt_createial_load, &actual_loaded);
288 }
289
290 char **keys= static_cast<char **>(calloc(actual_loaded, sizeof(char*)));
291 size_t *key_lengths= static_cast<size_t *>(calloc(actual_loaded, sizeof(size_t)));
292
293 if (keys == NULL or key_lengths == NULL)
294 {
295 free(keys);
296 free(key_lengths);
297 keys= NULL;
298 key_lengths= NULL;
299 }
300 else
301 {
302 for (uint32_t x= 0; x < actual_loaded; ++x)
303 {
304 keys[x]= pairs[x].key;
305 key_lengths[x]= pairs[x].key_length;
306 }
307 }
308
309 /* We set this after we have loaded */
310 {
311 if (opt_non_blocking_io)
312 memcached_behavior_set(memc, MEMCACHED_BEHAVIOR_NO_BLOCK, 1);
313
314 if (opt_tcp_nodelay)
315 memcached_behavior_set(memc, MEMCACHED_BEHAVIOR_TCP_NODELAY, 1);
316 }
317
318 pthread_mutex_lock(&sleeper_mutex);
319 master_wakeup= 1;
320 pthread_mutex_unlock(&sleeper_mutex);
321
322 pthread_t *threads= new (std::nothrow) pthread_t[opt_concurrency];
323
324 if (threads == NULL)
325 {
326 exit(EXIT_FAILURE);
327 }
328
329 for (uint32_t x= 0; x < opt_concurrency; x++)
330 {
331 thread_context_st *context= new thread_context_st(memc, opt_test);
332 context->test= opt_test;
333
334 context->initial_pairs= pairs;
335 context->initial_number= actual_loaded;
336 context->keys= keys;
337 context->key_lengths= key_lengths;
338
339 if (opt_test == SET_TEST)
340 {
341 context->execute_pairs= pairs_generate(opt_execute_number, VALUE_BYTES);
342 context->execute_number= opt_execute_number;
343 }
344
345 /* now you create the thread */
346 if (pthread_create(threads +x, NULL, run_task, (void *)context) != 0)
347 {
348 fprintf(stderr,"Could not create thread\n");
349 exit(1);
350 }
351 }
352
353 pthread_mutex_lock(&sleeper_mutex);
354 master_wakeup= 0;
355 pthread_mutex_unlock(&sleeper_mutex);
356 pthread_cond_broadcast(&sleep_threshhold);
357 gettimeofday(&start_time, NULL);
358
359 for (uint32_t x= 0; x < opt_concurrency; x++)
360 {
361 void *retval;
362 pthread_join(threads[x], &retval);
363 }
364 delete [] threads;
365
366 gettimeofday(&end_time, NULL);
367
368 conclusion->load_time= timedif(end_time, start_time);
369 conclusion->read_time= timedif(end_time, start_time);
370 free(keys);
371 free(key_lengths);
372 pairs_free(pairs);
373 memcached_free(memc);
374 }
375
376 void options_parse(int argc, char *argv[])
377 {
378 memcached_programs_help_st help_options[]=
379 {
380 {0},
381 };
382
383 static struct option long_options[]=
384 {
385 {(OPTIONSTRING)"concurrency", required_argument, NULL, OPT_SLAP_CONCURRENCY},
386 {(OPTIONSTRING)"debug", no_argument, &opt_verbose, OPT_DEBUG},
387 {(OPTIONSTRING)"quiet", no_argument, NULL, OPT_QUIET},
388 {(OPTIONSTRING)"execute-number", required_argument, NULL, OPT_SLAP_EXECUTE_NUMBER},
389 {(OPTIONSTRING)"flag", no_argument, &opt_displayflag, OPT_FLAG},
390 {(OPTIONSTRING)"flush", no_argument, &opt_flush, OPT_FLUSH},
391 {(OPTIONSTRING)"help", no_argument, NULL, OPT_HELP},
392 {(OPTIONSTRING)"initial-load", required_argument, NULL, OPT_SLAP_INITIAL_LOAD}, /* Number to load initially */
393 {(OPTIONSTRING)"non-blocking", no_argument, &opt_non_blocking_io, OPT_SLAP_NON_BLOCK},
394 {(OPTIONSTRING)"servers", required_argument, NULL, OPT_SERVERS},
395 {(OPTIONSTRING)"tcp-nodelay", no_argument, &opt_tcp_nodelay, OPT_SLAP_TCP_NODELAY},
396 {(OPTIONSTRING)"test", required_argument, NULL, OPT_SLAP_TEST},
397 {(OPTIONSTRING)"verbose", no_argument, &opt_verbose, OPT_VERBOSE},
398 {(OPTIONSTRING)"version", no_argument, NULL, OPT_VERSION},
399 {(OPTIONSTRING)"binary", no_argument, NULL, OPT_BINARY},
400 {(OPTIONSTRING)"udp", no_argument, NULL, OPT_UDP},
401 {0, 0, 0, 0},
402 };
403
404 bool opt_help= false;
405 bool opt_version= false;
406 int option_index= 0;
407 while (1)
408 {
409 int option_rv= getopt_long(argc, argv, "Vhvds:", long_options, &option_index);
410
411 if (option_rv == -1) break;
412
413 switch (option_rv)
414 {
415 case 0:
416 break;
417
418 case OPT_UDP:
419 if (opt_test == GET_TEST)
420 {
421 fprintf(stderr, "You can not run a get test in UDP mode. UDP mode "
422 "does not currently support get ops.\n");
423 exit(1);
424 }
425 opt_udp_io= true;
426 break;
427
428 case OPT_BINARY:
429 opt_binary= true;
430 break;
431
432 case OPT_VERBOSE: /* --verbose or -v */
433 opt_verbose= OPT_VERBOSE;
434 break;
435
436 case OPT_DEBUG: /* --debug or -d */
437 opt_verbose= OPT_DEBUG;
438 break;
439
440 case OPT_VERSION: /* --version or -V */
441 opt_version= true;
442 break;
443
444 case OPT_HELP: /* --help or -h */
445 opt_help= true;
446 break;
447
448 case OPT_SERVERS: /* --servers or -s */
449 opt_servers= strdup(optarg);
450 break;
451
452 case OPT_SLAP_TEST:
453 if (strcmp(optarg, "get") == 0)
454 {
455 if (opt_udp_io == 1)
456 {
457 fprintf(stderr, "You can not run a get test in UDP mode. UDP mode "
458 "does not currently support get ops.\n");
459 exit(EXIT_FAILURE);
460 }
461 opt_test= GET_TEST ;
462 }
463 else if (strcmp(optarg, "set") == 0)
464 {
465 opt_test= SET_TEST;
466 }
467 else if (strcmp(optarg, "mget") == 0)
468 {
469 opt_test= MGET_TEST;
470 }
471 else
472 {
473 fprintf(stderr, "Your test, %s, is not a known test\n", optarg);
474 exit(EXIT_FAILURE);
475 }
476 break;
477
478 case OPT_SLAP_CONCURRENCY:
479 errno= 0;
480 opt_concurrency= (unsigned int)strtoul(optarg, (char **)NULL, 10);
481 if (errno != 0)
482 {
483 fprintf(stderr, "Invalid value for concurrency: %s\n", optarg);
484 exit(EXIT_FAILURE);
485 }
486 break;
487
488 case OPT_SLAP_EXECUTE_NUMBER:
489 errno= 0;
490 opt_execute_number= (unsigned int)strtoul(optarg, (char **)NULL, 10);
491 if (errno != 0)
492 {
493 fprintf(stderr, "Invalid value for execute: %s\n", optarg);
494 exit(EXIT_FAILURE);
495 }
496 break;
497
498 case OPT_SLAP_INITIAL_LOAD:
499 errno= 0;
500 opt_createial_load= (unsigned int)strtoul(optarg, (char **)NULL, 10);
501 if (errno != 0)
502 {
503 fprintf(stderr, "Invalid value for initial load: %s\n", optarg);
504 exit(EXIT_FAILURE);
505 }
506 break;
507
508 case OPT_QUIET:
509 close_stdio();
510 break;
511
512
513 case '?':
514 /* getopt_long already printed an error message. */
515 exit(EXIT_FAILURE);
516
517 default:
518 abort();
519 }
520 }
521
522 if (opt_version)
523 {
524 version_command(PROGRAM_NAME);
525 exit(EXIT_SUCCESS);
526 }
527
528 if (opt_help)
529 {
530 help_command(PROGRAM_NAME, PROGRAM_DESCRIPTION, long_options, help_options);
531 exit(EXIT_SUCCESS);
532 }
533
534 if ((opt_test == GET_TEST or opt_test == MGET_TEST) and opt_createial_load == 0)
535 opt_createial_load= DEFAULT_INITIAL_LOAD;
536
537 if (opt_execute_number == 0)
538 opt_execute_number= DEFAULT_EXECUTE_NUMBER;
539
540 if (opt_concurrency == 0)
541 opt_concurrency= DEFAULT_CONCURRENCY;
542 }
543
544 void conclusions_print(conclusions_st *conclusion)
545 {
546 printf("\tThreads connecting to servers %u\n", opt_concurrency);
547 #ifdef NOT_FINISHED
548 printf("\tLoaded %u rows\n", conclusion->rows_loaded);
549 printf("\tRead %u rows\n", conclusion->rows_read);
550 #endif
551 if (opt_test == SET_TEST)
552 printf("\tTook %ld.%03ld seconds to load data\n", conclusion->load_time / 1000,
553 conclusion->load_time % 1000);
554 else
555 printf("\tTook %ld.%03ld seconds to read data\n", conclusion->read_time / 1000,
556 conclusion->read_time % 1000);
557 }
558
559 void flush_all(memcached_st *memc)
560 {
561 memcached_flush(memc, 0);
562 }
563
564 pairs_st *load_create_data(memcached_st *memc, unsigned int number_of,
565 unsigned int *actual_loaded)
566 {
567 memcached_st *memc_clone= memcached_clone(NULL, memc);
568 /* We always used non-blocking IO for load since it is faster */
569 memcached_behavior_set(memc_clone, MEMCACHED_BEHAVIOR_NO_BLOCK, 0);
570
571 pairs_st *pairs= pairs_generate(number_of, VALUE_BYTES);
572 *actual_loaded= execute_set(memc_clone, pairs, number_of);
573
574 memcached_free(memc_clone);
575
576 return pairs;
577 }