Merge lp:~tangent-org/libmemcached/1.0-build/ Build: jenkins-Libmemcached-187
[m6w6/libmemcached] / clients / memslap.cc
1 /* vim:expandtab:shiftwidth=2:tabstop=2:smarttab:
2 *
3 * Libmemcached library
4 *
5 * Copyright (C) 2011-2012 Data Differential, http://datadifferential.com/
6 * Copyright (C) 2006-2009 Brian Aker All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions are
10 * met:
11 *
12 * * Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 *
15 * * Redistributions in binary form must reproduce the above
16 * copyright notice, this list of conditions and the following disclaimer
17 * in the documentation and/or other materials provided with the
18 * distribution.
19 *
20 * * The names of its contributors may not be used to endorse or
21 * promote products derived from this software without specific prior
22 * written permission.
23 *
24 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
25 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
26 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
27 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
28 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
29 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
30 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
31 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
32 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
33 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
34 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
35 *
36 */
37
38
39 #include <mem_config.h>
40
41 #include <cassert>
42 #include <cstdio>
43 #include <cstdlib>
44 #include <cstring>
45 #include <fcntl.h>
46 #include <getopt.h>
47 #include <memory>
48 #include <pthread.h>
49 #include <sys/mman.h>
50 #include <sys/stat.h>
51 #include <sys/time.h>
52 #include <sys/types.h>
53 #include <sys/types.h>
54 #include <unistd.h>
55
56 #include <iostream>
57
58 #include <libmemcached-1.0/memcached.h>
59
60 #include "client_options.h"
61 #include "utilities.h"
62 #include "generator.h"
63 #include "execute.h"
64
65 #define DEFAULT_INITIAL_LOAD 10000
66 #define DEFAULT_EXECUTE_NUMBER 10000
67 #define DEFAULT_CONCURRENCY 1
68
69 #define PROGRAM_NAME "memslap"
70 #define PROGRAM_DESCRIPTION "Generates a load against a memcached custer of servers."
71
72 /* Global Thread counter */
73 volatile unsigned int master_wakeup;
74 pthread_mutex_t sleeper_mutex;
75 pthread_cond_t sleep_threshhold;
76
77 /* Types */
78 enum test_t {
79 SET_TEST,
80 GET_TEST,
81 MGET_TEST
82 };
83
84 struct thread_context_st {
85 unsigned int key_count;
86 pairs_st *initial_pairs;
87 unsigned int initial_number;
88 pairs_st *execute_pairs;
89 unsigned int execute_number;
90 char **keys;
91 size_t *key_lengths;
92 test_t test;
93 memcached_st *memc;
94 const memcached_st* root;
95
96 thread_context_st(const memcached_st* memc_arg, test_t test_arg) :
97 key_count(0),
98 initial_pairs(NULL),
99 initial_number(0),
100 execute_pairs(NULL),
101 execute_number(0),
102 keys(0),
103 key_lengths(NULL),
104 test(test_arg),
105 memc(NULL),
106 root(memc_arg)
107 {
108 }
109
110 void init()
111 {
112 memc= memcached_clone(NULL, root);
113 }
114
115 ~thread_context_st()
116 {
117 if (execute_pairs)
118 {
119 pairs_free(execute_pairs);
120 }
121 memcached_free(memc);
122 }
123 };
124
125 struct conclusions_st {
126 long int load_time;
127 long int read_time;
128 unsigned int rows_loaded;
129 unsigned int rows_read;
130
131 conclusions_st() :
132 load_time(0),
133 read_time(0),
134 rows_loaded(0),
135 rows_read()
136 { }
137 };
138
139 /* Prototypes */
140 void options_parse(int argc, char *argv[]);
141 void conclusions_print(conclusions_st *conclusion);
142 void scheduler(memcached_server_st *servers, conclusions_st *conclusion);
143 pairs_st *load_create_data(memcached_st *memc, unsigned int number_of,
144 unsigned int *actual_loaded);
145 void flush_all(memcached_st *memc);
146
147 static bool opt_binary= 0;
148 static int opt_verbose= 0;
149 static int opt_flush= 0;
150 static int opt_non_blocking_io= 0;
151 static int opt_tcp_nodelay= 0;
152 static unsigned int opt_execute_number= 0;
153 static unsigned int opt_createial_load= 0;
154 static unsigned int opt_concurrency= 0;
155 static int opt_displayflag= 0;
156 static char *opt_servers= NULL;
157 static bool opt_udp_io= false;
158 test_t opt_test= SET_TEST;
159
160 extern "C" {
161
162 static __attribute__((noreturn)) void *run_task(void *p)
163 {
164 thread_context_st *context= (thread_context_st *)p;
165
166 context->init();
167
168 pthread_mutex_lock(&sleeper_mutex);
169 while (master_wakeup)
170 {
171 pthread_cond_wait(&sleep_threshhold, &sleeper_mutex);
172 }
173 pthread_mutex_unlock(&sleeper_mutex);
174
175 /* Do Stuff */
176 switch (context->test)
177 {
178 case SET_TEST:
179 assert(context->execute_pairs);
180 execute_set(context->memc, context->execute_pairs, context->execute_number);
181 break;
182
183 case GET_TEST:
184 execute_get(context->memc, context->initial_pairs, context->initial_number);
185 break;
186
187 case MGET_TEST:
188 execute_mget(context->memc, (const char*const*)context->keys, context->key_lengths, context->initial_number);
189 break;
190 }
191
192 delete context;
193
194 pthread_exit(0);
195 }
196
197 }
198
199
200 int main(int argc, char *argv[])
201 {
202 conclusions_st conclusion;
203
204 srandom((unsigned int)time(NULL));
205 options_parse(argc, argv);
206
207 if (opt_servers == NULL)
208 {
209 char *temp;
210
211 if ((temp= getenv("MEMCACHED_SERVERS")))
212 {
213 opt_servers= strdup(temp);
214 }
215 else
216 {
217 fprintf(stderr, "No Servers provided\n");
218 return EXIT_FAILURE;
219 }
220 }
221
222 memcached_server_st *servers= memcached_servers_parse(opt_servers);
223
224 pthread_mutex_init(&sleeper_mutex, NULL);
225 pthread_cond_init(&sleep_threshhold, NULL);
226
227 int error_code= EXIT_SUCCESS;
228 try {
229 scheduler(servers, &conclusion);
230 }
231 catch(std::exception& e)
232 {
233 std::cerr << "Died with exception: " << e.what() << std::endl;
234 error_code= EXIT_FAILURE;
235 }
236
237 free(opt_servers);
238
239 (void)pthread_mutex_destroy(&sleeper_mutex);
240 (void)pthread_cond_destroy(&sleep_threshhold);
241 conclusions_print(&conclusion);
242 memcached_server_list_free(servers);
243
244 return error_code;
245 }
246
247 void scheduler(memcached_server_st *servers, conclusions_st *conclusion)
248 {
249 unsigned int actual_loaded= 0; /* Fix warning */
250
251 struct timeval start_time, end_time;
252 pairs_st *pairs= NULL;
253
254 memcached_st *memc= memcached_create(NULL);
255
256 memcached_server_push(memc, servers);
257
258 /* We need to set udp behavior before adding servers to the client */
259 if (opt_udp_io)
260 {
261 if (memcached_failed(memcached_behavior_set(memc, MEMCACHED_BEHAVIOR_USE_UDP, opt_udp_io)))
262 {
263 std::cerr << "Failed to enable UDP." << std::endl;
264 memcached_free(memc);
265 exit(EXIT_FAILURE);
266 }
267 }
268
269 memcached_behavior_set(memc, MEMCACHED_BEHAVIOR_BINARY_PROTOCOL,
270 (uint64_t)opt_binary);
271
272 if (opt_flush)
273 {
274 flush_all(memc);
275 }
276
277 if (opt_createial_load)
278 {
279 pairs= load_create_data(memc, opt_createial_load, &actual_loaded);
280 }
281
282 char **keys= static_cast<char **>(calloc(actual_loaded, sizeof(char*)));
283 size_t *key_lengths= static_cast<size_t *>(calloc(actual_loaded, sizeof(size_t)));
284
285 if (keys == NULL or key_lengths == NULL)
286 {
287 free(keys);
288 free(key_lengths);
289 keys= NULL;
290 key_lengths= NULL;
291 }
292 else
293 {
294 for (uint32_t x= 0; x < actual_loaded; ++x)
295 {
296 keys[x]= pairs[x].key;
297 key_lengths[x]= pairs[x].key_length;
298 }
299 }
300
301 /* We set this after we have loaded */
302 {
303 if (opt_non_blocking_io)
304 memcached_behavior_set(memc, MEMCACHED_BEHAVIOR_NO_BLOCK, 1);
305
306 if (opt_tcp_nodelay)
307 memcached_behavior_set(memc, MEMCACHED_BEHAVIOR_TCP_NODELAY, 1);
308 }
309
310 pthread_mutex_lock(&sleeper_mutex);
311 master_wakeup= 1;
312 pthread_mutex_unlock(&sleeper_mutex);
313
314 pthread_t *threads= new (std::nothrow) pthread_t[opt_concurrency];
315
316 if (threads == NULL)
317 {
318 exit(EXIT_FAILURE);
319 }
320
321 for (uint32_t x= 0; x < opt_concurrency; x++)
322 {
323 thread_context_st *context= new thread_context_st(memc, opt_test);
324 context->test= opt_test;
325
326 context->initial_pairs= pairs;
327 context->initial_number= actual_loaded;
328 context->keys= keys;
329 context->key_lengths= key_lengths;
330
331 if (opt_test == SET_TEST)
332 {
333 context->execute_pairs= pairs_generate(opt_execute_number, 400);
334 context->execute_number= opt_execute_number;
335 }
336
337 /* now you create the thread */
338 if (pthread_create(threads +x, NULL, run_task, (void *)context) != 0)
339 {
340 fprintf(stderr,"Could not create thread\n");
341 exit(1);
342 }
343 }
344
345 pthread_mutex_lock(&sleeper_mutex);
346 master_wakeup= 0;
347 pthread_mutex_unlock(&sleeper_mutex);
348 pthread_cond_broadcast(&sleep_threshhold);
349 gettimeofday(&start_time, NULL);
350
351 for (uint32_t x= 0; x < opt_concurrency; x++)
352 {
353 void *retval;
354 pthread_join(threads[x], &retval);
355 }
356 delete [] threads;
357
358 gettimeofday(&end_time, NULL);
359
360 conclusion->load_time= timedif(end_time, start_time);
361 conclusion->read_time= timedif(end_time, start_time);
362 free(keys);
363 free(key_lengths);
364 pairs_free(pairs);
365 memcached_free(memc);
366 }
367
368 void options_parse(int argc, char *argv[])
369 {
370 memcached_programs_help_st help_options[]=
371 {
372 {0},
373 };
374
375 static struct option long_options[]=
376 {
377 {(OPTIONSTRING)"concurrency", required_argument, NULL, OPT_SLAP_CONCURRENCY},
378 {(OPTIONSTRING)"debug", no_argument, &opt_verbose, OPT_DEBUG},
379 {(OPTIONSTRING)"quiet", no_argument, NULL, OPT_QUIET},
380 {(OPTIONSTRING)"execute-number", required_argument, NULL, OPT_SLAP_EXECUTE_NUMBER},
381 {(OPTIONSTRING)"flag", no_argument, &opt_displayflag, OPT_FLAG},
382 {(OPTIONSTRING)"flush", no_argument, &opt_flush, OPT_FLUSH},
383 {(OPTIONSTRING)"help", no_argument, NULL, OPT_HELP},
384 {(OPTIONSTRING)"initial-load", required_argument, NULL, OPT_SLAP_INITIAL_LOAD}, /* Number to load initially */
385 {(OPTIONSTRING)"non-blocking", no_argument, &opt_non_blocking_io, OPT_SLAP_NON_BLOCK},
386 {(OPTIONSTRING)"servers", required_argument, NULL, OPT_SERVERS},
387 {(OPTIONSTRING)"tcp-nodelay", no_argument, &opt_tcp_nodelay, OPT_SLAP_TCP_NODELAY},
388 {(OPTIONSTRING)"test", required_argument, NULL, OPT_SLAP_TEST},
389 {(OPTIONSTRING)"verbose", no_argument, &opt_verbose, OPT_VERBOSE},
390 {(OPTIONSTRING)"version", no_argument, NULL, OPT_VERSION},
391 {(OPTIONSTRING)"binary", no_argument, NULL, OPT_BINARY},
392 {(OPTIONSTRING)"udp", no_argument, NULL, OPT_UDP},
393 {0, 0, 0, 0},
394 };
395
396 bool opt_help= false;
397 bool opt_version= false;
398 int option_index= 0;
399 while (1)
400 {
401 int option_rv= getopt_long(argc, argv, "Vhvds:", long_options, &option_index);
402
403 if (option_rv == -1) break;
404
405 switch (option_rv)
406 {
407 case 0:
408 break;
409
410 case OPT_UDP:
411 if (opt_test == GET_TEST)
412 {
413 fprintf(stderr, "You can not run a get test in UDP mode. UDP mode "
414 "does not currently support get ops.\n");
415 exit(1);
416 }
417 opt_udp_io= true;
418 break;
419
420 case OPT_BINARY:
421 opt_binary= true;
422 break;
423
424 case OPT_VERBOSE: /* --verbose or -v */
425 opt_verbose= OPT_VERBOSE;
426 break;
427
428 case OPT_DEBUG: /* --debug or -d */
429 opt_verbose = OPT_DEBUG;
430 break;
431
432 case OPT_VERSION: /* --version or -V */
433 opt_version= true;
434 break;
435
436 case OPT_HELP: /* --help or -h */
437 opt_help= true;
438 break;
439
440 case OPT_SERVERS: /* --servers or -s */
441 opt_servers= strdup(optarg);
442 break;
443
444 case OPT_SLAP_TEST:
445 if (strcmp(optarg, "get") == 0)
446 {
447 if (opt_udp_io == 1)
448 {
449 fprintf(stderr, "You can not run a get test in UDP mode. UDP mode "
450 "does not currently support get ops.\n");
451 exit(EXIT_FAILURE);
452 }
453 opt_test= GET_TEST ;
454 }
455 else if (strcmp(optarg, "set") == 0)
456 {
457 opt_test= SET_TEST;
458 }
459 else if (strcmp(optarg, "mget") == 0)
460 {
461 opt_test= MGET_TEST;
462 }
463 else
464 {
465 fprintf(stderr, "Your test, %s, is not a known test\n", optarg);
466 exit(EXIT_FAILURE);
467 }
468 break;
469
470 case OPT_SLAP_CONCURRENCY:
471 opt_concurrency= (unsigned int)strtoul(optarg, (char **)NULL, 10);
472 break;
473
474 case OPT_SLAP_EXECUTE_NUMBER:
475 opt_execute_number= (unsigned int)strtoul(optarg, (char **)NULL, 10);
476 break;
477
478 case OPT_SLAP_INITIAL_LOAD:
479 opt_createial_load= (unsigned int)strtoul(optarg, (char **)NULL, 10);
480 break;
481
482 case OPT_QUIET:
483 close_stdio();
484 break;
485
486
487 case '?':
488 /* getopt_long already printed an error message. */
489 exit(EXIT_FAILURE);
490
491 default:
492 abort();
493 }
494 }
495
496 if (opt_version)
497 {
498 version_command(PROGRAM_NAME);
499 exit(EXIT_SUCCESS);
500 }
501
502 if (opt_help)
503 {
504 help_command(PROGRAM_NAME, PROGRAM_DESCRIPTION, long_options, help_options);
505 exit(EXIT_SUCCESS);
506 }
507
508 if ((opt_test == GET_TEST or opt_test == MGET_TEST) and opt_createial_load == 0)
509 opt_createial_load= DEFAULT_INITIAL_LOAD;
510
511 if (opt_execute_number == 0)
512 opt_execute_number= DEFAULT_EXECUTE_NUMBER;
513
514 if (opt_concurrency == 0)
515 opt_concurrency= DEFAULT_CONCURRENCY;
516 }
517
518 void conclusions_print(conclusions_st *conclusion)
519 {
520 printf("\tThreads connecting to servers %u\n", opt_concurrency);
521 #ifdef NOT_FINISHED
522 printf("\tLoaded %u rows\n", conclusion->rows_loaded);
523 printf("\tRead %u rows\n", conclusion->rows_read);
524 #endif
525 if (opt_test == SET_TEST)
526 printf("\tTook %ld.%03ld seconds to load data\n", conclusion->load_time / 1000,
527 conclusion->load_time % 1000);
528 else
529 printf("\tTook %ld.%03ld seconds to read data\n", conclusion->read_time / 1000,
530 conclusion->read_time % 1000);
531 }
532
533 void flush_all(memcached_st *memc)
534 {
535 memcached_flush(memc, 0);
536 }
537
538 pairs_st *load_create_data(memcached_st *memc, unsigned int number_of,
539 unsigned int *actual_loaded)
540 {
541 memcached_st *memc_clone= memcached_clone(NULL, memc);
542 /* We always used non-blocking IO for load since it is faster */
543 memcached_behavior_set(memc_clone, MEMCACHED_BEHAVIOR_NO_BLOCK, 0);
544
545 pairs_st *pairs= pairs_generate(number_of, 400);
546 *actual_loaded= execute_set(memc_clone, pairs, number_of);
547
548 memcached_free(memc_clone);
549
550 return pairs;
551 }