memaslap.
clients/memflush \
clients/memparse \
clients/memrm \
+ clients/memslap \
clients/memstat
if HAVE_LIBEVENT
if !BUILD_WIN32_WRAPPERS
- bin_PROGRAMS+= clients/memslap
+ bin_PROGRAMS+= clients/memaslap
endif
endif
clients_memerror_SOURCES= clients/memerror.c
clients_memerror_LDADD= $(CLIENTS_LDADDS)
-clients_memslap_SOURCES= \
- clients/memslap.c \
+clients_memslap_SOURCES = clients/memslap.c
+clients_memslap_LDADD = $(PTHREAD_LIBS) clients/libgenexec.la $(CLIENTS_LDADDS)
+
+clients_memaslap_SOURCES= \
+ clients/memaslap.c \
clients/ms_conn.c \
clients/ms_setting.c \
clients/ms_sigsegv.c \
clients/ms_stats.c \
clients/ms_task.c \
clients/ms_thread.c
-clients_memslap_LDADD= $(LTLIBEVENT) clients/libgenexec.la $(CLIENTS_LDADDS)
+clients_memaslap_LDADD= $(LTLIBEVENT) clients/libgenexec.la $(CLIENTS_LDADDS)
clients_memcapable_SOURCES= clients/memcapable.c
clients_memcapable_LDADD= $(CLIENTS_LDADDS)
clients/memcat --servers=localhost /etc/services
clients/memrm --servers=localhost /etc/services
clients/memstat --servers=localhost
- clients/memslap --servers=localhost
- clients/memslap --servers=localhost --concurrency=10
- clients/memslap --servers=localhost --concurrency=10 --initial-load=1000
- clients/memslap --servers=localhost --concurrency=10 --initial-load=1000 --execute-number=10
- clients/memslap --servers=localhost --concurrency=10 --initial-load=1000 --execute-number=10 --test=get
- clients/memslap --servers=localhost --concurrency=10 --initial-load=1000 --execute-number=10 --test=set
- clients/memslap --servers=localhost --concurrency=10 --initial-load=1000 --execute-number=10 --test=set --non-blocking
+ clients/memaslap --servers=localhost
+ clients/memaslap --servers=localhost --concurrency=10
+ clients/memaslap --servers=localhost --concurrency=10 --initial-load=1000
+ clients/memaslap --servers=localhost --concurrency=10 --initial-load=1000 --execute-number=10
+ clients/memaslap --servers=localhost --concurrency=10 --initial-load=1000 --execute-number=10 --test=get
+ clients/memaslap --servers=localhost --concurrency=10 --initial-load=1000 --execute-number=10 --test=set
+ clients/memaslap --servers=localhost --concurrency=10 --initial-load=1000 --execute-number=10 --test=set --non-blocking
client-valgrind:
- libtool --mode=execute valgrind --leak-check=yes --show-reachable=yes clients/memslap --servers=localhost
- libtool --mode=execute valgrind --leak-check=yes --show-reachable=yes clients/memslap --servers=localhost --concurrency=10
- libtool --mode=execute valgrind --leak-check=yes --show-reachable=yes clients/memslap --servers=localhost --concurrency=10 --initial-load=1000
- libtool --mode=execute valgrind --leak-check=yes --show-reachable=yes clients/memslap --servers=localhost --concurrency=10 --initial-load=1000 --execute-number=10
- libtool --mode=execute valgrind --leak-check=yes --show-reachable=yes clients/memslap --servers=localhost --concurrency=10 --initial-load=1000 --execute-number=10 --test=get
- libtool --mode=execute valgrind --leak-check=yes --show-reachable=yes clients/memslap --servers=localhost --concurrency=10 --initial-load=1000 --execute-number=10 --test=set
- libtool --mode=execute valgrind --leak-check=yes --show-reachable=yes clients/memslap --servers=localhost --concurrency=10 --initial-load=1000 --execute-number=10 --test=set --non-blocking
+ libtool --mode=execute valgrind --leak-check=yes --show-reachable=yes clients/memaslap --servers=localhost
+ libtool --mode=execute valgrind --leak-check=yes --show-reachable=yes clients/memaslap --servers=localhost --concurrency=10
+ libtool --mode=execute valgrind --leak-check=yes --show-reachable=yes clients/memaslap --servers=localhost --concurrency=10 --initial-load=1000
+ libtool --mode=execute valgrind --leak-check=yes --show-reachable=yes clients/memaslap --servers=localhost --concurrency=10 --initial-load=1000 --execute-number=10
+ libtool --mode=execute valgrind --leak-check=yes --show-reachable=yes clients/memaslap --servers=localhost --concurrency=10 --initial-load=1000 --execute-number=10 --test=get
+ libtool --mode=execute valgrind --leak-check=yes --show-reachable=yes clients/memaslap --servers=localhost --concurrency=10 --initial-load=1000 --execute-number=10 --test=set
+ libtool --mode=execute valgrind --leak-check=yes --show-reachable=yes clients/memaslap --servers=localhost --concurrency=10 --initial-load=1000 --execute-number=10 --test=set --non-blocking
--- /dev/null
+/*
+ * memslap
+ *
+ * (c) Copyright 2009, Schooner Information Technology, Inc.
+ * All rights reserved.
+ * http://www.schoonerinfotech.com/
+ *
+ * Use and distribution licensed under the BSD license. See
+ * the COPYING file for full text.
+ *
+ * Authors:
+ * Brian Aker
+ * Mingqiang Zhuang <mingqiangzhuang@hengtiansoft.com>
+ *
+ */
+#include "config.h"
+
+#include <stdlib.h>
+#include <getopt.h>
+#include <limits.h>
+#if TIME_WITH_SYS_TIME
+# include <sys/time.h>
+# include <time.h>
+#else
+# if HAVE_SYS_TIME_H
+# include <sys/time.h>
+# else
+# include <time.h>
+# endif
+#endif
+
+
+#include "ms_sigsegv.h"
+#include "ms_setting.h"
+#include "ms_thread.h"
+
+#define PROGRAM_NAME "memslap"
+#define PROGRAM_DESCRIPTION \
+ "Generates workload against memcached servers."
+
+#ifdef __sun
+ /* For some odd reason the option struct on solaris defines the argument
+ * as char* and not const char*
+ */
+#define OPTIONSTRING char*
+#else
+#define OPTIONSTRING const char*
+#endif
+
+/* options */
+static struct option long_options[]=
+{
+ { (OPTIONSTRING)"servers", required_argument, NULL,
+ OPT_SERVERS },
+ { (OPTIONSTRING)"threads", required_argument, NULL,
+ OPT_THREAD_NUMBER },
+ { (OPTIONSTRING)"concurrency", required_argument, NULL,
+ OPT_CONCURRENCY },
+ { (OPTIONSTRING)"conn_sock", required_argument, NULL,
+ OPT_SOCK_PER_CONN },
+ { (OPTIONSTRING)"execute_number", required_argument, NULL,
+ OPT_EXECUTE_NUMBER },
+ { (OPTIONSTRING)"time", required_argument, NULL,
+ OPT_TIME },
+ { (OPTIONSTRING)"cfg_cmd", required_argument, NULL,
+ OPT_CONFIG_CMD },
+ { (OPTIONSTRING)"win_size", required_argument, NULL,
+ OPT_WINDOW_SIZE },
+ { (OPTIONSTRING)"fixed_size", required_argument, NULL,
+ OPT_FIXED_LTH },
+ { (OPTIONSTRING)"verify", required_argument, NULL,
+ OPT_VERIFY },
+ { (OPTIONSTRING)"division", required_argument, NULL,
+ OPT_GETS_DIVISION },
+ { (OPTIONSTRING)"stat_freq", required_argument, NULL,
+ OPT_STAT_FREQ },
+ { (OPTIONSTRING)"exp_verify", required_argument, NULL,
+ OPT_EXPIRE },
+ { (OPTIONSTRING)"overwrite", required_argument, NULL,
+ OPT_OVERWRITE },
+ { (OPTIONSTRING)"reconnect", no_argument, NULL,
+ OPT_RECONNECT },
+ { (OPTIONSTRING)"udp", no_argument, NULL,
+ OPT_UDP },
+ { (OPTIONSTRING)"facebook", no_argument, NULL,
+ OPT_FACEBOOK_TEST },
+ { (OPTIONSTRING)"binary", no_argument, NULL,
+ OPT_BINARY_PROTOCOL },
+ { (OPTIONSTRING)"tps", required_argument, NULL,
+ OPT_TPS },
+ { (OPTIONSTRING)"rep_write", required_argument, NULL,
+ OPT_REP_WRITE_SRV },
+ { (OPTIONSTRING)"verbose", no_argument, NULL,
+ OPT_VERBOSE },
+ { (OPTIONSTRING)"help", no_argument, NULL,
+ OPT_HELP },
+ { (OPTIONSTRING)"version", no_argument, NULL,
+ OPT_VERSION },
+ { 0, 0, 0, 0 },
+};
+
+/* Prototypes */
+static void ms_sync_lock_init(void);
+static void ms_sync_lock_destroy(void);
+static void ms_global_struct_init(void);
+static void ms_global_struct_destroy(void);
+static void ms_version_command(const char *command_name);
+static const char *ms_lookup_help(ms_options_t option);
+static int64_t ms_parse_time(void);
+static int64_t ms_parse_size(void);
+static void ms_options_parse(int argc, char *argv[]);
+static int ms_check_para(void);
+static void ms_statistic_init(void);
+static void ms_stats_init(void);
+static void ms_print_statistics(int in_time);
+static void ms_print_memslap_stats(struct timeval *start_time,
+ struct timeval *end_time);
+static void ms_monitor_slap_mode(void);
+void ms_help_command(const char *command_name, const char *description);
+
+
+/* initialize the global locks */
+static void ms_sync_lock_init()
+{
+ ms_global.init_lock.count= 0;
+ pthread_mutex_init(&ms_global.init_lock.lock, NULL);
+ pthread_cond_init(&ms_global.init_lock.cond, NULL);
+
+ ms_global.warmup_lock.count = 0;
+ pthread_mutex_init(&ms_global.warmup_lock.lock, NULL);
+ pthread_cond_init(&ms_global.warmup_lock.cond, NULL);
+
+ ms_global.run_lock.count= 0;
+ pthread_mutex_init(&ms_global.run_lock.lock, NULL);
+ pthread_cond_init(&ms_global.run_lock.cond, NULL);
+
+ pthread_mutex_init(&ms_global.quit_mutex, NULL);
+ pthread_mutex_init(&ms_global.seq_mutex, NULL);
+} /* ms_sync_lock_init */
+
+
+/* destroy the global locks */
+static void ms_sync_lock_destroy()
+{
+ pthread_mutex_destroy(&ms_global.init_lock.lock);
+ pthread_cond_destroy(&ms_global.init_lock.cond);
+
+ pthread_mutex_destroy(&ms_global.warmup_lock.lock);
+ pthread_cond_destroy(&ms_global.warmup_lock.cond);
+
+ pthread_mutex_destroy(&ms_global.run_lock.lock);
+ pthread_cond_destroy(&ms_global.run_lock.cond);
+
+ pthread_mutex_destroy(&ms_global.quit_mutex);
+ pthread_mutex_destroy(&ms_global.seq_mutex);
+
+ if (ms_setting.stat_freq > 0)
+ {
+ pthread_mutex_destroy(&ms_statistic.stat_mutex);
+ }
+} /* ms_sync_lock_destroy */
+
+
+/* initialize the global structure */
+static void ms_global_struct_init()
+{
+ ms_sync_lock_init();
+ ms_global.finish_warmup= false;
+ ms_global.time_out= false;
+}
+
+
+/* destroy the global structure */
+static void ms_global_struct_destroy()
+{
+ ms_sync_lock_destroy();
+}
+
+
+/**
+ * output the version information
+ *
+ * @param command_name, the string of this process
+ */
+static void ms_version_command(const char *command_name)
+{
+ printf("%s v%u.%u\n", command_name, 1U, 0U);
+ exit(0);
+}
+
+
+/**
+ * get the description of the option
+ *
+ * @param option, option of command line
+ *
+ * @return char*, description of the command option
+ */
+static const char *ms_lookup_help(ms_options_t option)
+{
+ switch (option)
+ {
+ case OPT_SERVERS:
+ return
+ "List one or more servers to connect. Servers count must be less than\n"
+ " threads count. e.g.: --servers=localhost:1234,localhost:11211";
+
+ case OPT_VERSION:
+ return "Display the version of the application and then exit.";
+
+ case OPT_HELP:
+ return "Display this message and then exit.";
+
+ case OPT_EXECUTE_NUMBER:
+ return "Number of operations(get and set) to execute for the\n"
+ " given test. Default 1000000.";
+
+ case OPT_THREAD_NUMBER:
+ return
+ "Number of threads to startup, better equal to CPU numbers. Default 8.";
+
+ case OPT_CONCURRENCY:
+ return "Number of concurrency to simulate with load. Default 128.";
+
+ case OPT_FIXED_LTH:
+ return "Fixed length of value.";
+
+ case OPT_VERIFY:
+ return "The proportion of date verification, e.g.: --verify=0.01";
+
+ case OPT_GETS_DIVISION:
+ return "Number of keys to multi-get once. Default 1, means single get.";
+
+ case OPT_TIME:
+ return
+ "How long the test to run, suffix: s-seconds, m-minutes, h-hours,\n"
+ " d-days e.g.: --time=2h.";
+
+ case OPT_CONFIG_CMD:
+ return
+ "Load the configure file to get command,key and value distribution list.";
+
+ case OPT_WINDOW_SIZE:
+ return
+ "Task window size of each concurrency, suffix: K, M e.g.: --win_size=10k.\n"
+ " Default 10k.";
+
+ case OPT_UDP:
+ return
+ "UDP support, default memslap uses TCP, TCP port and UDP port of\n"
+ " server must be same.";
+
+ case OPT_EXPIRE:
+ return
+ "The proportion of objects with expire time, e.g.: --exp_verify=0.01.\n"
+ " Default no object with expire time";
+
+ case OPT_OVERWRITE:
+ return
+ "The proportion of objects need overwrite, e.g.: --overwrite=0.01.\n"
+ " Default never overwrite object.";
+
+ case OPT_STAT_FREQ:
+ return
+ "Frequency of dumping statistic information. suffix: s-seconds,\n"
+ " m-minutes, e.g.: --resp_freq=10s.";
+
+ case OPT_SOCK_PER_CONN:
+ return "Number of TCP socks per concurrency. Default 1.";
+
+ case OPT_RECONNECT:
+ return
+ "Reconnect support, when connection is closed it will be reconnected.";
+
+ case OPT_VERBOSE:
+ return
+ "Whether it outputs detailed information when verification fails.";
+
+ case OPT_FACEBOOK_TEST:
+ return
+ "Whether it enables facebook test feature, set with TCP and multi-get with UDP.";
+
+ case OPT_BINARY_PROTOCOL:
+ return
+ "Whether it enables binary protocol. Default with ASCII protocol.";
+
+ case OPT_TPS:
+ return "Expected throughput, suffix: K, e.g.: --tps=10k.";
+
+ case OPT_REP_WRITE_SRV:
+ return "The first nth servers can write data, e.g.: --rep_write=2.";
+
+ default:
+ return "Forgot to document this option :)";
+ } /* switch */
+} /* ms_lookup_help */
+
+
+/**
+ * output the help information
+ *
+ * @param command_name, the string of this process
+ * @param description, description of this process
+ * @param long_options, global options array
+ */
+void ms_help_command(const char *command_name, const char *description)
+{
+ char *help_message= NULL;
+
+ printf("%s v%u.%u\n", command_name, 1U, 0U);
+ printf(" %s\n\n", description);
+ printf(
+ "Usage:\n"
+ " memslap -hV | -s servers [-F config_file] [-t time | -x exe_num] [...]\n\n"
+ "Options:\n");
+
+ for (int x= 0; long_options[x].name; x++)
+ {
+ printf(" -%c, --%s%c\n", long_options[x].val, long_options[x].name,
+ long_options[x].has_arg ? '=' : ' ');
+
+ if ((help_message= (char *)ms_lookup_help(long_options[x].val)) != NULL)
+ {
+ printf(" %s\n", help_message);
+ }
+ }
+
+ printf(
+ "\nExamples:\n"
+ " memslap -s 127.0.0.1:11211 -S 5s\n"
+ " memslap -s 127.0.0.1:11211 -t 2m -v 0.2 -e 0.05 -b\n"
+ " memslap -s 127.0.0.1:11211 -F config -t 2m -w 40k -S 20s -o 0.2\n"
+ " memslap -s 127.0.0.1:11211 -F config -t 2m -T 4 -c 128 -d 20 -P 40k\n"
+ " memslap -s 127.0.0.1:11211 -F config -t 2m -d 50 -a -n 40\n"
+ " memslap -s 127.0.0.1:11211,127.0.0.1:11212 -F config -t 2m\n"
+ " memslap -s 127.0.0.1:11211,127.0.0.1:11212 -F config -t 2m -p 2\n\n");
+
+ exit(0);
+} /* ms_help_command */
+
+
+/* used to parse the time string */
+static int64_t ms_parse_time()
+{
+ int64_t ret= 0;
+ char unit= optarg[strlen(optarg) - 1];
+
+ optarg[strlen(optarg) - 1]= '\0';
+ ret= atoi(optarg);
+
+ switch (unit)
+ {
+ case 'd':
+ case 'D':
+ ret*= 24;
+
+ case 'h':
+ case 'H':
+ ret*= 60;
+
+ case 'm':
+ case 'M':
+ ret*= 60;
+
+ case 's':
+ case 'S':
+ break;
+
+ default:
+ ret= -1;
+ break;
+ } /* switch */
+
+ return ret;
+} /* ms_parse_time */
+
+
+/* used to parse the size string */
+static int64_t ms_parse_size()
+{
+ int64_t ret= -1;
+ char unit= optarg[strlen(optarg) - 1];
+
+ optarg[strlen(optarg) - 1]= '\0';
+ ret= strtoll(optarg, (char **)NULL, 10);
+
+ switch (unit)
+ {
+ case 'k':
+ case 'K':
+ ret*= 1024;
+ break;
+
+ case 'm':
+ case 'M':
+ ret*= 1024 * 1024;
+ break;
+
+ case 'g':
+ case 'G':
+ ret*= 1024 * 1024 * 1024;
+ break;
+
+ default:
+ ret= -1;
+ break;
+ } /* switch */
+
+ return ret;
+} /* ms_parse_size */
+
+
+/* used to parse the options of command line */
+static void ms_options_parse(int argc, char *argv[])
+{
+ int option_index= 0;
+ int option_rv;
+
+ while ((option_rv= getopt_long(argc, argv, "VhURbaBs:x:T:c:X:v:d:"
+ "t:S:F:w:e:o:n:P:p:",
+ long_options, &option_index)) != -1)
+ {
+ switch (option_rv)
+ {
+ case 0:
+ break;
+
+ case OPT_VERSION: /* --version or -V */
+ ms_version_command(PROGRAM_NAME);
+ break;
+
+ case OPT_HELP: /* --help or -h */
+ ms_help_command(PROGRAM_NAME, PROGRAM_DESCRIPTION);
+ break;
+
+ case OPT_SERVERS: /* --servers or -s */
+ ms_setting.srv_str= strdup(optarg);
+ break;
+
+ case OPT_CONCURRENCY: /* --concurrency or -c */
+ ms_setting.nconns= (uint32_t)strtoul(optarg, (char **) NULL, 10);
+ if (ms_setting.nconns <= 0)
+ {
+ fprintf(stderr, "Concurrency must be greater than 0.:-)\n");
+ exit(1);
+ }
+ break;
+
+ case OPT_EXECUTE_NUMBER: /* --execute_number or -x */
+ ms_setting.exec_num= (int)strtol(optarg, (char **) NULL, 10);
+ if (ms_setting.exec_num <= 0)
+ {
+ fprintf(stderr, "Execute number must be greater than 0.:-)\n");
+ exit(1);
+ }
+ break;
+
+ case OPT_THREAD_NUMBER: /* --threads or -T */
+ ms_setting.nthreads= (uint32_t)strtoul(optarg, (char **) NULL, 10);
+ if (ms_setting.nthreads <= 0)
+ {
+ fprintf(stderr, "Threads number must be greater than 0.:-)\n");
+ exit(1);
+ }
+ break;
+
+ case OPT_FIXED_LTH: /* --fixed_size or -X */
+ ms_setting.fixed_value_size= (size_t)strtoull(optarg, (char **) NULL, 10);
+ if ((ms_setting.fixed_value_size <= 0)
+ || (ms_setting.fixed_value_size > MAX_VALUE_SIZE))
+ {
+ fprintf(stderr, "Value size must be between 0 and 1M.:-)\n");
+ exit(1);
+ }
+ break;
+
+ case OPT_VERIFY: /* --verify or -v */
+ ms_setting.verify_percent= atof(optarg);
+ if ((ms_setting.verify_percent <= 0)
+ || (ms_setting.verify_percent > 1.0))
+ {
+ fprintf(stderr, "Data verification rate must be "
+ "greater than 0 and less than 1.0. :-)\n");
+ exit(1);
+ }
+ break;
+
+ case OPT_GETS_DIVISION: /* --division or -d */
+ ms_setting.mult_key_num= (int)strtol(optarg, (char **) NULL, 10);
+ if (ms_setting.mult_key_num <= 0)
+ {
+ fprintf(stderr, "Multi-get key number must be greater than 0.:-)\n");
+ exit(1);
+ }
+ break;
+
+ case OPT_TIME: /* --time or -t */
+ ms_setting.run_time= (int)ms_parse_time();
+ if (ms_setting.run_time == -1)
+ {
+ fprintf(stderr, "Please specify the run time. :-)\n"
+ "'s' for second, 'm' for minute, 'h' for hour, "
+ "'d' for day. e.g.: --time=24h (means 24 hours).\n");
+ exit(1);
+ }
+
+ if (ms_setting.run_time == 0)
+ {
+ fprintf(stderr, "Running time can not be 0. :-)\n");
+ exit(1);
+ }
+ break;
+
+ case OPT_CONFIG_CMD: /* --cfg_cmd or -F */
+ ms_setting.cfg_file= strdup(optarg);
+ break;
+
+ case OPT_WINDOW_SIZE: /* --win_size or -w */
+ ms_setting.win_size= (size_t)ms_parse_size();
+ if (ms_setting.win_size == (size_t)-1)
+ {
+ fprintf(
+ stderr,
+ "Please specify the item window size. :-)\n"
+ "e.g.: --win_size=10k (means 10k task window size).\n");
+ exit(1);
+ }
+ break;
+
+ case OPT_UDP: /* --udp or -U*/
+ ms_setting.udp= true;
+ break;
+
+ case OPT_EXPIRE: /* --exp_verify or -e */
+ ms_setting.exp_ver_per= atof(optarg);
+ if ((ms_setting.exp_ver_per <= 0) || (ms_setting.exp_ver_per > 1.0))
+ {
+ fprintf(stderr, "Expire time verification rate must be "
+ "greater than 0 and less than 1.0. :-)\n");
+ exit(1);
+ }
+ break;
+
+ case OPT_OVERWRITE: /* --overwrite or -o */
+ ms_setting.overwrite_percent= atof(optarg);
+ if ((ms_setting.overwrite_percent <= 0)
+ || (ms_setting.overwrite_percent > 1.0))
+ {
+ fprintf(stderr, "Objects overwrite rate must be "
+ "greater than 0 and less than 1.0. :-)\n");
+ exit(1);
+ }
+ break;
+
+ case OPT_STAT_FREQ: /* --stat_freq or -S */
+ ms_setting.stat_freq= (int)ms_parse_time();
+ if (ms_setting.stat_freq == -1)
+ {
+ fprintf(stderr, "Please specify the frequency of dumping "
+ "statistic information. :-)\n"
+ "'s' for second, 'm' for minute, 'h' for hour, "
+ "'d' for day. e.g.: --time=24h (means 24 hours).\n");
+ exit(1);
+ }
+
+ if (ms_setting.stat_freq == 0)
+ {
+ fprintf(stderr, "The frequency of dumping statistic information "
+ "can not be 0. :-)\n");
+ exit(1);
+ }
+ break;
+
+ case OPT_SOCK_PER_CONN: /* --conn_sock or -n */
+ ms_setting.sock_per_conn= (uint32_t)strtoul(optarg, (char **) NULL, 10);
+ if (ms_setting.sock_per_conn <= 0)
+ {
+ fprintf(stderr, "Number of socks of each concurrency "
+ "must be greater than 0.:-)\n");
+ exit(1);
+ }
+ break;
+
+ case OPT_RECONNECT: /* --reconnect or -R */
+ ms_setting.reconnect= true;
+ break;
+
+ case OPT_VERBOSE: /* --verbose or -b */
+ ms_setting.verbose= true;
+ break;
+
+ case OPT_FACEBOOK_TEST: /* --facebook or -a */
+ ms_setting.facebook_test= true;
+ break;
+
+ case OPT_BINARY_PROTOCOL: /* --binary or -B */
+ ms_setting.binary_prot= true;
+ break;
+
+ case OPT_TPS: /* --tps or -P */
+ ms_setting.expected_tps= (int)ms_parse_size();
+ if (ms_setting.expected_tps == -1)
+ {
+ fprintf(stderr,
+ "Please specify the item expected throughput. :-)\n"
+ "e.g.: --tps=10k (means 10k throughput).\n");
+ exit(1);
+ }
+ break;
+
+ case OPT_REP_WRITE_SRV: /* --rep_write or -p */
+ ms_setting.rep_write_srv= (uint32_t)strtoul(optarg, (char **) NULL, 10);
+ if (ms_setting.rep_write_srv <= 0)
+ {
+ fprintf(stderr,
+ "Number of replication writing server must be greater "
+ "than 0.:-)\n");
+ exit(1);
+ }
+ break;
+
+ case '?':
+ /* getopt_long already printed an error message. */
+ exit(1);
+
+ default:
+ abort();
+ } /* switch */
+ }
+} /* ms_options_parse */
+
+
+static int ms_check_para()
+{
+ if (ms_setting.srv_str == NULL)
+ {
+ char *temp;
+
+ if ((temp= getenv("MEMCACHED_SERVERS")))
+ {
+ ms_setting.srv_str= strdup(temp);
+ }
+ else
+ {
+ fprintf(stderr, "No Servers provided\n\n");
+ return -1;
+ }
+ }
+
+ if (ms_setting.nconns % (uint32_t)ms_setting.nthreads != 0)
+ {
+ fprintf(stderr, "Concurrency must be the multiples of threads count.\n");
+ return -1;
+ }
+
+ if (ms_setting.win_size % UNIT_ITEMS_COUNT != 0)
+ {
+ fprintf(stderr, "Window size must be the multiples of 1024.\n\n");
+ return -1;
+ }
+
+ return EXIT_SUCCESS;
+} /* ms_check_para */
+
+
+/* initialize the statistic structure */
+static void ms_statistic_init()
+{
+ pthread_mutex_init(&ms_statistic.stat_mutex, NULL);
+ ms_init_stats(&ms_statistic.get_stat, "Get");
+ ms_init_stats(&ms_statistic.set_stat, "Set");
+ ms_init_stats(&ms_statistic.total_stat, "Total");
+} /* ms_statistic_init */
+
+
+/* initialize the global state structure */
+static void ms_stats_init()
+{
+ memset(&ms_stats, 0, sizeof(ms_stats_t));
+ if (ms_setting.stat_freq > 0)
+ {
+ ms_statistic_init();
+ }
+} /* ms_stats_init */
+
+
+/* use to output the statistic */
+static void ms_print_statistics(int in_time)
+{
+ int obj_size= (int)(ms_setting.avg_key_size + ms_setting.avg_val_size);
+
+ printf("\033[1;1H\033[2J\n");
+ ms_dump_format_stats(&ms_statistic.get_stat, in_time,
+ ms_setting.stat_freq, obj_size);
+ ms_dump_format_stats(&ms_statistic.set_stat, in_time,
+ ms_setting.stat_freq, obj_size);
+ ms_dump_format_stats(&ms_statistic.total_stat, in_time,
+ ms_setting.stat_freq, obj_size);
+} /* ms_print_statistics */
+
+
+/* used to print the states of memslap */
+static void ms_print_memslap_stats(struct timeval *start_time,
+ struct timeval *end_time)
+{
+ char buf[1024];
+ char *pos= buf;
+
+ pos+= snprintf(pos,
+ sizeof(buf), "cmd_get: %lu\n",
+ (unsigned long) ms_stats.cmd_get);
+ pos+= snprintf(pos,
+ sizeof(buf) - (size_t)(pos -buf),
+ "cmd_set: %lu\n",
+ (unsigned long) ms_stats.cmd_set);
+ pos+= snprintf(pos,
+ sizeof(buf) - (size_t)(pos -buf),
+ "get_misses: %lu\n",
+ (unsigned long) ms_stats.get_misses);
+
+ if (ms_setting.verify_percent > 0)
+ {
+ pos+= snprintf(pos,
+ sizeof(buf) - (size_t)(pos -buf),
+ "verify_misses: %lu\n",
+ (unsigned long) ms_stats.vef_miss);
+ pos+= snprintf(pos,
+ sizeof(buf) - (size_t)(pos -buf),
+ "verify_failed: %lu\n",
+ (unsigned long) ms_stats.vef_failed);
+ }
+
+ if (ms_setting.exp_ver_per > 0)
+ {
+ pos+= snprintf(pos,
+ sizeof(buf) - (size_t)(pos -buf),
+ "expired_get: %lu\n",
+ (unsigned long) ms_stats.exp_get);
+ pos+= snprintf(pos,
+ sizeof(buf) - (size_t)(pos -buf),
+ "unexpired_unget: %lu\n",
+ (unsigned long) ms_stats.unexp_unget);
+ }
+
+ pos+= snprintf(pos,
+ sizeof(buf) - (size_t)(pos -buf),
+ "written_bytes: %lu\n",
+ (unsigned long) ms_stats.bytes_written);
+ pos+= snprintf(pos,
+ sizeof(buf) - (size_t)(pos -buf),
+ "read_bytes: %lu\n",
+ (unsigned long) ms_stats.bytes_read);
+ pos+= snprintf(pos,
+ sizeof(buf) - (size_t)(pos -buf),
+ "object_bytes: %lu\n",
+ (unsigned long) ms_stats.obj_bytes);
+
+ if (ms_setting.udp || ms_setting.facebook_test)
+ {
+ pos+= snprintf(pos,
+ sizeof(buf) - (size_t)(pos -buf),
+ "packet_disorder: %lu\n",
+ (unsigned long) ms_stats.pkt_disorder);
+ pos+= snprintf(pos,
+ sizeof(buf) - (size_t)(pos -buf),
+ "packet_drop: %lu\n",
+ (unsigned long)ms_stats.pkt_drop);
+ pos+= snprintf(pos,
+ sizeof(buf) - (size_t)(pos -buf),
+ "udp_timeout: %lu\n",
+ (unsigned long)ms_stats.udp_timeout);
+ }
+
+ if (ms_setting.stat_freq > 0)
+ {
+ ms_dump_stats(&ms_statistic.get_stat);
+ ms_dump_stats(&ms_statistic.set_stat);
+ ms_dump_stats(&ms_statistic.total_stat);
+ }
+
+ int64_t time_diff= ms_time_diff(start_time, end_time);
+ pos+= snprintf(pos,
+ sizeof(buf) - (size_t)(pos -buf),
+ "\nRun time: %.1fs Ops: %llu TPS: %.0Lf Net_rate: %.1fM/s\n",
+ (double)time_diff / 1000000,
+ (unsigned long long)(ms_stats.cmd_get + ms_stats.cmd_set),
+ (ms_stats.cmd_get
+ + ms_stats.cmd_set) / ((long double)time_diff / 1000000),
+ (double)(
+ ms_stats.bytes_written
+ + ms_stats.bytes_read) / 1024 / 1024
+ / ((double)time_diff / 1000000));
+ assert(pos <= buf);
+
+ fprintf(stdout, "%s", buf);
+ fflush(stdout);
+} /* ms_print_memslap_stats */
+
+
+/* the loop of the main thread, wait the work threads to complete */
+static void ms_monitor_slap_mode()
+{
+ int second= 0;
+ struct timeval start_time, end_time;
+
+ /* Wait all the threads complete initialization. */
+ pthread_mutex_lock(&ms_global.init_lock.lock);
+ while (ms_global.init_lock.count < ms_setting.nthreads)
+ {
+ pthread_cond_wait(&ms_global.init_lock.cond,
+ &ms_global.init_lock.lock);
+ }
+ pthread_mutex_unlock(&ms_global.init_lock.lock);
+
+ /* only when there is no set operation it need warm up */
+ if (ms_setting.cmd_distr[CMD_SET].cmd_prop < PROP_ERROR)
+ {
+ /* Wait all the connects complete warm up. */
+ pthread_mutex_lock(&ms_global.warmup_lock.lock);
+ while (ms_global.warmup_lock.count < ms_setting.nconns)
+ {
+ pthread_cond_wait(&ms_global.warmup_lock.cond, &ms_global.warmup_lock.lock);
+ }
+ pthread_mutex_unlock(&ms_global.warmup_lock.lock);
+ }
+ ms_global.finish_warmup= true;
+
+ /* running in "run time" mode, user specify run time */
+ if (ms_setting.run_time > 0)
+ {
+ gettimeofday(&start_time, NULL);
+ while (1)
+ {
+ sleep(1);
+ second++;
+
+ if ((ms_setting.stat_freq > 0) && (second % ms_setting.stat_freq == 0)
+ && (ms_stats.active_conns >= ms_setting.nconns)
+ && (ms_stats.active_conns <= INT_MAX))
+ {
+ ms_print_statistics(second);
+ }
+
+ if (ms_setting.run_time <= second)
+ {
+ ms_global.time_out= true;
+ break;
+ }
+
+ /* all connections disconnect */
+ if ((second > 5) && (ms_stats.active_conns == 0))
+ {
+ break;
+ }
+ }
+ gettimeofday(&end_time, NULL);
+ sleep(1); /* wait all threads clean up */
+ }
+ else
+ {
+ /* running in "execute number" mode, user specify execute number */
+ gettimeofday(&start_time, NULL);
+
+ /*
+ * We loop until we know that all connects have cleaned up.
+ */
+ pthread_mutex_lock(&ms_global.run_lock.lock);
+ while (ms_global.run_lock.count < ms_setting.nconns)
+ {
+ pthread_cond_wait(&ms_global.run_lock.cond, &ms_global.run_lock.lock);
+ }
+ pthread_mutex_unlock(&ms_global.run_lock.lock);
+
+ gettimeofday(&end_time, NULL);
+ }
+
+ ms_print_memslap_stats(&start_time, &end_time);
+} /* ms_monitor_slap_mode */
+
+
+/* the main function */
+int main(int argc, char *argv[])
+{
+ srandom((unsigned int)time(NULL));
+ ms_global_struct_init();
+
+ /* initialization */
+ ms_setting_init_pre();
+ ms_options_parse(argc, argv);
+ if (ms_check_para())
+ {
+ ms_help_command(PROGRAM_NAME, PROGRAM_DESCRIPTION);
+ exit(1);
+ }
+ ms_setting_init_post();
+ ms_stats_init();
+ ms_thread_init();
+
+ /* waiting work thread complete its task */
+ ms_monitor_slap_mode();
+
+ /* clean up */
+ ms_thread_cleanup();
+ ms_global_struct_destroy();
+ ms_setting_cleanup();
+
+ return EXIT_SUCCESS;
+} /* main */
-/*
- * memslap
+/* vim:expandtab:shiftwidth=2:tabstop=2:smarttab:
+ *
+ * Libmemcached library
*
- * (c) Copyright 2009, Schooner Information Technology, Inc.
- * All rights reserved.
- * http://www.schoonerinfotech.com/
+ * Copyright (C) 2011 Data Differential, http://datadifferential.com/
+ * Copyright (C) 2006-2009 Brian Aker All rights reserved.
*
- * Use and distribution licensed under the BSD license. See
- * the COPYING file for full text.
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
*
- * Authors:
- * Brian Aker
- * Mingqiang Zhuang <mingqiangzhuang@hengtiansoft.com>
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *
+ * * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *
+ * * The names of its contributors may not be used to endorse or
+ * promote products derived from this software without specific prior
+ * written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
*/
-#include "config.h"
+
+#include <config.h>
+#include <stdio.h>
#include <stdlib.h>
+#include <string.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <sys/mman.h>
+#include <fcntl.h>
+#include <sys/time.h>
#include <getopt.h>
-#include <limits.h>
-#if TIME_WITH_SYS_TIME
-# include <sys/time.h>
-# include <time.h>
-#else
-# if HAVE_SYS_TIME_H
-# include <sys/time.h>
-# else
-# include <time.h>
-# endif
-#endif
-
-
-#include "ms_sigsegv.h"
-#include "ms_setting.h"
-#include "ms_thread.h"
-
-#define PROGRAM_NAME "memslap"
-#define PROGRAM_DESCRIPTION \
- "Generates workload against memcached servers."
-
-#ifdef __sun
- /* For some odd reason the option struct on solaris defines the argument
- * as char* and not const char*
- */
-#define OPTIONSTRING char*
-#else
-#define OPTIONSTRING const char*
-#endif
+#include <pthread.h>
+#include <assert.h>
+
+#include <libmemcached/memcached.h>
+
+#include "client_options.h"
+#include "utilities.h"
+#include "generator.h"
+#include "execute.h"
+
+#define DEFAULT_INITIAL_LOAD 10000
+#define DEFAULT_EXECUTE_NUMBER 10000
+#define DEFAULT_CONCURRENCY 1
+
+#define PROGRAM_NAME "memslap"
+#define PROGRAM_DESCRIPTION "Generates a load against a memcached custer of servers."
+
+/* Global Thread counter */
+volatile unsigned int thread_counter;
+pthread_mutex_t counter_mutex;
+pthread_cond_t count_threshhold;
+volatile unsigned int master_wakeup;
+pthread_mutex_t sleeper_mutex;
+pthread_cond_t sleep_threshhold;
+
+void *run_task(void *p);
+
+/* Types */
+typedef struct conclusions_st conclusions_st;
+typedef struct thread_context_st thread_context_st;
+typedef enum {
+ SET_TEST,
+ GET_TEST,
+ MGET_TEST
+} test_type;
+
+struct thread_context_st {
+ unsigned int key_count;
+ pairs_st *initial_pairs;
+ unsigned int initial_number;
+ pairs_st *execute_pairs;
+ unsigned int execute_number;
+ char **keys;
+ size_t *key_lengths;
+ test_type test;
+ memcached_st *memc;
+};
-/* options */
-static struct option long_options[]=
-{
- { (OPTIONSTRING)"servers", required_argument, NULL,
- OPT_SERVERS },
- { (OPTIONSTRING)"threads", required_argument, NULL,
- OPT_THREAD_NUMBER },
- { (OPTIONSTRING)"concurrency", required_argument, NULL,
- OPT_CONCURRENCY },
- { (OPTIONSTRING)"conn_sock", required_argument, NULL,
- OPT_SOCK_PER_CONN },
- { (OPTIONSTRING)"execute_number", required_argument, NULL,
- OPT_EXECUTE_NUMBER },
- { (OPTIONSTRING)"time", required_argument, NULL,
- OPT_TIME },
- { (OPTIONSTRING)"cfg_cmd", required_argument, NULL,
- OPT_CONFIG_CMD },
- { (OPTIONSTRING)"win_size", required_argument, NULL,
- OPT_WINDOW_SIZE },
- { (OPTIONSTRING)"fixed_size", required_argument, NULL,
- OPT_FIXED_LTH },
- { (OPTIONSTRING)"verify", required_argument, NULL,
- OPT_VERIFY },
- { (OPTIONSTRING)"division", required_argument, NULL,
- OPT_GETS_DIVISION },
- { (OPTIONSTRING)"stat_freq", required_argument, NULL,
- OPT_STAT_FREQ },
- { (OPTIONSTRING)"exp_verify", required_argument, NULL,
- OPT_EXPIRE },
- { (OPTIONSTRING)"overwrite", required_argument, NULL,
- OPT_OVERWRITE },
- { (OPTIONSTRING)"reconnect", no_argument, NULL,
- OPT_RECONNECT },
- { (OPTIONSTRING)"udp", no_argument, NULL,
- OPT_UDP },
- { (OPTIONSTRING)"facebook", no_argument, NULL,
- OPT_FACEBOOK_TEST },
- { (OPTIONSTRING)"binary", no_argument, NULL,
- OPT_BINARY_PROTOCOL },
- { (OPTIONSTRING)"tps", required_argument, NULL,
- OPT_TPS },
- { (OPTIONSTRING)"rep_write", required_argument, NULL,
- OPT_REP_WRITE_SRV },
- { (OPTIONSTRING)"verbose", no_argument, NULL,
- OPT_VERBOSE },
- { (OPTIONSTRING)"help", no_argument, NULL,
- OPT_HELP },
- { (OPTIONSTRING)"version", no_argument, NULL,
- OPT_VERSION },
- { 0, 0, 0, 0 },
+struct conclusions_st {
+ long int load_time;
+ long int read_time;
+ unsigned int rows_loaded;
+ unsigned int rows_read;
};
/* Prototypes */
-static void ms_sync_lock_init(void);
-static void ms_sync_lock_destroy(void);
-static void ms_global_struct_init(void);
-static void ms_global_struct_destroy(void);
-static void ms_version_command(const char *command_name);
-static const char *ms_lookup_help(ms_options_t option);
-static int64_t ms_parse_time(void);
-static int64_t ms_parse_size(void);
-static void ms_options_parse(int argc, char *argv[]);
-static int ms_check_para(void);
-static void ms_statistic_init(void);
-static void ms_stats_init(void);
-static void ms_print_statistics(int in_time);
-static void ms_print_memslap_stats(struct timeval *start_time,
- struct timeval *end_time);
-static void ms_monitor_slap_mode(void);
-void ms_help_command(const char *command_name, const char *description);
-
-
-/* initialize the global locks */
-static void ms_sync_lock_init()
-{
- ms_global.init_lock.count= 0;
- pthread_mutex_init(&ms_global.init_lock.lock, NULL);
- pthread_cond_init(&ms_global.init_lock.cond, NULL);
-
- ms_global.warmup_lock.count = 0;
- pthread_mutex_init(&ms_global.warmup_lock.lock, NULL);
- pthread_cond_init(&ms_global.warmup_lock.cond, NULL);
-
- ms_global.run_lock.count= 0;
- pthread_mutex_init(&ms_global.run_lock.lock, NULL);
- pthread_cond_init(&ms_global.run_lock.cond, NULL);
-
- pthread_mutex_init(&ms_global.quit_mutex, NULL);
- pthread_mutex_init(&ms_global.seq_mutex, NULL);
-} /* ms_sync_lock_init */
+void options_parse(int argc, char *argv[]);
+void conclusions_print(conclusions_st *conclusion);
+void scheduler(memcached_server_st *servers, conclusions_st *conclusion);
+pairs_st *load_create_data(memcached_st *memc, unsigned int number_of,
+ unsigned int *actual_loaded);
+void flush_all(memcached_st *memc);
+
+static int opt_binary= 0;
+static int opt_verbose= 0;
+static int opt_flush= 0;
+static int opt_non_blocking_io= 0;
+static int opt_tcp_nodelay= 0;
+static unsigned int opt_execute_number= 0;
+static unsigned int opt_createial_load= 0;
+static unsigned int opt_concurrency= 0;
+static int opt_displayflag= 0;
+static char *opt_servers= NULL;
+static int opt_udp_io= 0;
+test_type opt_test= SET_TEST;
-
-/* destroy the global locks */
-static void ms_sync_lock_destroy()
+int main(int argc, char *argv[])
{
- pthread_mutex_destroy(&ms_global.init_lock.lock);
- pthread_cond_destroy(&ms_global.init_lock.cond);
+ conclusions_st conclusion;
+ memcached_server_st *servers;
- pthread_mutex_destroy(&ms_global.warmup_lock.lock);
- pthread_cond_destroy(&ms_global.warmup_lock.cond);
+ memset(&conclusion, 0, sizeof(conclusions_st));
- pthread_mutex_destroy(&ms_global.run_lock.lock);
- pthread_cond_destroy(&ms_global.run_lock.cond);
-
- pthread_mutex_destroy(&ms_global.quit_mutex);
- pthread_mutex_destroy(&ms_global.seq_mutex);
+ srandom((unsigned int)time(NULL));
+ options_parse(argc, argv);
- if (ms_setting.stat_freq > 0)
+ if (!opt_servers)
{
- pthread_mutex_destroy(&ms_statistic.stat_mutex);
+ char *temp;
+
+ if ((temp= getenv("MEMCACHED_SERVERS")))
+ opt_servers= strdup(temp);
+ else
+ {
+ fprintf(stderr, "No Servers provided\n");
+ exit(1);
+ }
}
-} /* ms_sync_lock_destroy */
+ servers= memcached_servers_parse(opt_servers);
-/* initialize the global structure */
-static void ms_global_struct_init()
-{
- ms_sync_lock_init();
- ms_global.finish_warmup= false;
- ms_global.time_out= false;
-}
+ pthread_mutex_init(&counter_mutex, NULL);
+ pthread_cond_init(&count_threshhold, NULL);
+ pthread_mutex_init(&sleeper_mutex, NULL);
+ pthread_cond_init(&sleep_threshhold, NULL);
+ scheduler(servers, &conclusion);
-/* destroy the global structure */
-static void ms_global_struct_destroy()
-{
- ms_sync_lock_destroy();
-}
+ free(opt_servers);
+ (void)pthread_mutex_destroy(&counter_mutex);
+ (void)pthread_cond_destroy(&count_threshhold);
+ (void)pthread_mutex_destroy(&sleeper_mutex);
+ (void)pthread_cond_destroy(&sleep_threshhold);
+ conclusions_print(&conclusion);
+ memcached_server_list_free(servers);
-/**
- * output the version information
- *
- * @param command_name, the string of this process
- */
-static void ms_version_command(const char *command_name)
-{
- printf("%s v%u.%u\n", command_name, 1U, 0U);
- exit(0);
+ return 0;
}
-
-/**
- * get the description of the option
- *
- * @param option, option of command line
- *
- * @return char*, description of the command option
- */
-static const char *ms_lookup_help(ms_options_t option)
+void scheduler(memcached_server_st *servers, conclusions_st *conclusion)
{
- switch (option)
- {
- case OPT_SERVERS:
- return
- "List one or more servers to connect. Servers count must be less than\n"
- " threads count. e.g.: --servers=localhost:1234,localhost:11211";
-
- case OPT_VERSION:
- return "Display the version of the application and then exit.";
-
- case OPT_HELP:
- return "Display this message and then exit.";
-
- case OPT_EXECUTE_NUMBER:
- return "Number of operations(get and set) to execute for the\n"
- " given test. Default 1000000.";
-
- case OPT_THREAD_NUMBER:
- return
- "Number of threads to startup, better equal to CPU numbers. Default 8.";
-
- case OPT_CONCURRENCY:
- return "Number of concurrency to simulate with load. Default 128.";
-
- case OPT_FIXED_LTH:
- return "Fixed length of value.";
-
- case OPT_VERIFY:
- return "The proportion of date verification, e.g.: --verify=0.01";
-
- case OPT_GETS_DIVISION:
- return "Number of keys to multi-get once. Default 1, means single get.";
-
- case OPT_TIME:
- return
- "How long the test to run, suffix: s-seconds, m-minutes, h-hours,\n"
- " d-days e.g.: --time=2h.";
+ unsigned int actual_loaded= 0; /* Fix warning */
+ memcached_st *memc;
- case OPT_CONFIG_CMD:
- return
- "Load the configure file to get command,key and value distribution list.";
-
- case OPT_WINDOW_SIZE:
- return
- "Task window size of each concurrency, suffix: K, M e.g.: --win_size=10k.\n"
- " Default 10k.";
-
- case OPT_UDP:
- return
- "UDP support, default memslap uses TCP, TCP port and UDP port of\n"
- " server must be same.";
-
- case OPT_EXPIRE:
- return
- "The proportion of objects with expire time, e.g.: --exp_verify=0.01.\n"
- " Default no object with expire time";
-
- case OPT_OVERWRITE:
- return
- "The proportion of objects need overwrite, e.g.: --overwrite=0.01.\n"
- " Default never overwrite object.";
-
- case OPT_STAT_FREQ:
- return
- "Frequency of dumping statistic information. suffix: s-seconds,\n"
- " m-minutes, e.g.: --resp_freq=10s.";
-
- case OPT_SOCK_PER_CONN:
- return "Number of TCP socks per concurrency. Default 1.";
-
- case OPT_RECONNECT:
- return
- "Reconnect support, when connection is closed it will be reconnected.";
-
- case OPT_VERBOSE:
- return
- "Whether it outputs detailed information when verification fails.";
-
- case OPT_FACEBOOK_TEST:
- return
- "Whether it enables facebook test feature, set with TCP and multi-get with UDP.";
-
- case OPT_BINARY_PROTOCOL:
- return
- "Whether it enables binary protocol. Default with ASCII protocol.";
+ struct timeval start_time, end_time;
+ pthread_t mainthread; /* Thread descriptor */
+ pthread_attr_t attr; /* Thread attributes */
+ pairs_st *pairs= NULL;
- case OPT_TPS:
- return "Expected throughput, suffix: K, e.g.: --tps=10k.";
+ pthread_attr_init(&attr);
+ pthread_attr_setdetachstate(&attr,
+ PTHREAD_CREATE_DETACHED);
- case OPT_REP_WRITE_SRV:
- return "The first nth servers can write data, e.g.: --rep_write=2.";
+ memc= memcached_create(NULL);
- default:
- return "Forgot to document this option :)";
- } /* switch */
-} /* ms_lookup_help */
+ /* We need to set udp behavior before adding servers to the client */
+ if (opt_udp_io)
+ {
+ memcached_behavior_set(memc, MEMCACHED_BEHAVIOR_USE_UDP,
+ (uint64_t)opt_udp_io);
+ for (uint32_t x= 0; x < memcached_server_list_count(servers); x++ )
+ {
+ servers[x].type= MEMCACHED_CONNECTION_UDP;
+ }
+ }
+ memcached_server_push(memc, servers);
+ memcached_behavior_set(memc, MEMCACHED_BEHAVIOR_BINARY_PROTOCOL,
+ (uint64_t)opt_binary);
-/**
- * output the help information
- *
- * @param command_name, the string of this process
- * @param description, description of this process
- * @param long_options, global options array
- */
-void ms_help_command(const char *command_name, const char *description)
-{
- char *help_message= NULL;
+ if (opt_flush)
+ flush_all(memc);
+ if (opt_createial_load)
+ pairs= load_create_data(memc, opt_createial_load, &actual_loaded);
- printf("%s v%u.%u\n", command_name, 1U, 0U);
- printf(" %s\n\n", description);
- printf(
- "Usage:\n"
- " memslap -hV | -s servers [-F config_file] [-t time | -x exe_num] [...]\n\n"
- "Options:\n");
+ char **keys= calloc(actual_loaded, sizeof(char*));
+ size_t *key_lengths= calloc(actual_loaded, sizeof(size_t));
- for (int x= 0; long_options[x].name; x++)
+ if (keys == NULL || key_lengths == NULL)
{
- printf(" -%c, --%s%c\n", long_options[x].val, long_options[x].name,
- long_options[x].has_arg ? '=' : ' ');
-
- if ((help_message= (char *)ms_lookup_help(long_options[x].val)) != NULL)
+ free(keys);
+ free(key_lengths);
+ keys= NULL;
+ key_lengths= NULL;
+ }
+ else
+ {
+ for (uint32_t x= 0; x < actual_loaded; ++x)
{
- printf(" %s\n", help_message);
+ keys[x]= pairs[x].key;
+ key_lengths[x]= pairs[x].key_length;
}
}
- printf(
- "\nExamples:\n"
- " memslap -s 127.0.0.1:11211 -S 5s\n"
- " memslap -s 127.0.0.1:11211 -t 2m -v 0.2 -e 0.05 -b\n"
- " memslap -s 127.0.0.1:11211 -F config -t 2m -w 40k -S 20s -o 0.2\n"
- " memslap -s 127.0.0.1:11211 -F config -t 2m -T 4 -c 128 -d 20 -P 40k\n"
- " memslap -s 127.0.0.1:11211 -F config -t 2m -d 50 -a -n 40\n"
- " memslap -s 127.0.0.1:11211,127.0.0.1:11212 -F config -t 2m\n"
- " memslap -s 127.0.0.1:11211,127.0.0.1:11212 -F config -t 2m -p 2\n\n");
-
- exit(0);
-} /* ms_help_command */
-
+ /* We set this after we have loaded */
+ {
+ if (opt_non_blocking_io)
+ memcached_behavior_set(memc, MEMCACHED_BEHAVIOR_NO_BLOCK, 1);
+ if (opt_tcp_nodelay)
+ memcached_behavior_set(memc, MEMCACHED_BEHAVIOR_TCP_NODELAY, 1);
+ }
-/* used to parse the time string */
-static int64_t ms_parse_time()
-{
- int64_t ret= 0;
- char unit= optarg[strlen(optarg) - 1];
+ pthread_mutex_lock(&counter_mutex);
+ thread_counter= 0;
- optarg[strlen(optarg) - 1]= '\0';
- ret= atoi(optarg);
+ pthread_mutex_lock(&sleeper_mutex);
+ master_wakeup= 1;
+ pthread_mutex_unlock(&sleeper_mutex);
- switch (unit)
+ for (uint32_t x= 0; x < opt_concurrency; x++)
{
- case 'd':
- case 'D':
- ret*= 24;
-
- case 'h':
- case 'H':
- ret*= 60;
+ thread_context_st *context;
+ context= (thread_context_st *)calloc(1, sizeof(thread_context_st));
- case 'm':
- case 'M':
- ret*= 60;
+ context->memc= memcached_clone(NULL, memc);
+ context->test= opt_test;
- case 's':
- case 'S':
- break;
+ context->initial_pairs= pairs;
+ context->initial_number= actual_loaded;
+ context->keys= keys;
+ context->key_lengths= key_lengths;
- default:
- ret= -1;
- break;
- } /* switch */
+ if (opt_test == SET_TEST)
+ {
+ context->execute_pairs= pairs_generate(opt_execute_number, 400);
+ context->execute_number= opt_execute_number;
+ }
- return ret;
-} /* ms_parse_time */
+ /* now you create the thread */
+ if (pthread_create(&mainthread, &attr, run_task,
+ (void *)context) != 0)
+ {
+ fprintf(stderr,"Could not create thread\n");
+ exit(1);
+ }
+ thread_counter++;
+ }
+ pthread_mutex_unlock(&counter_mutex);
+ pthread_attr_destroy(&attr);
+
+ pthread_mutex_lock(&sleeper_mutex);
+ master_wakeup= 0;
+ pthread_mutex_unlock(&sleeper_mutex);
+ pthread_cond_broadcast(&sleep_threshhold);
+
+ gettimeofday(&start_time, NULL);
+ /*
+ We loop until we know that all children have cleaned up.
+ */
+ pthread_mutex_lock(&counter_mutex);
+ while (thread_counter)
+ pthread_cond_wait(&count_threshhold, &counter_mutex);
+ pthread_mutex_unlock(&counter_mutex);
+
+ gettimeofday(&end_time, NULL);
+
+ conclusion->load_time= timedif(end_time, start_time);
+ conclusion->read_time= timedif(end_time, start_time);
+ free(keys);
+ free(key_lengths);
+ pairs_free(pairs);
+ memcached_free(memc);
+}
-/* used to parse the size string */
-static int64_t ms_parse_size()
+void options_parse(int argc, char *argv[])
{
- int64_t ret= -1;
- char unit= optarg[strlen(optarg) - 1];
-
- optarg[strlen(optarg) - 1]= '\0';
- ret= strtoll(optarg, (char **)NULL, 10);
-
- switch (unit)
+ memcached_programs_help_st help_options[]=
{
- case 'k':
- case 'K':
- ret*= 1024;
- break;
-
- case 'm':
- case 'M':
- ret*= 1024 * 1024;
- break;
-
- case 'g':
- case 'G':
- ret*= 1024 * 1024 * 1024;
- break;
-
- default:
- ret= -1;
- break;
- } /* switch */
-
- return ret;
-} /* ms_parse_size */
+ {0},
+ };
+ static struct option long_options[]=
+ {
+ {(OPTIONSTRING)"concurrency", required_argument, NULL, OPT_SLAP_CONCURRENCY},
+ {(OPTIONSTRING)"debug", no_argument, &opt_verbose, OPT_DEBUG},
+ {(OPTIONSTRING)"execute-number", required_argument, NULL, OPT_SLAP_EXECUTE_NUMBER},
+ {(OPTIONSTRING)"flag", no_argument, &opt_displayflag, OPT_FLAG},
+ {(OPTIONSTRING)"flush", no_argument, &opt_flush, OPT_FLUSH},
+ {(OPTIONSTRING)"help", no_argument, NULL, OPT_HELP},
+ {(OPTIONSTRING)"initial-load", required_argument, NULL, OPT_SLAP_INITIAL_LOAD}, /* Number to load initially */
+ {(OPTIONSTRING)"non-blocking", no_argument, &opt_non_blocking_io, OPT_SLAP_NON_BLOCK},
+ {(OPTIONSTRING)"servers", required_argument, NULL, OPT_SERVERS},
+ {(OPTIONSTRING)"tcp-nodelay", no_argument, &opt_tcp_nodelay, OPT_SLAP_TCP_NODELAY},
+ {(OPTIONSTRING)"test", required_argument, NULL, OPT_SLAP_TEST},
+ {(OPTIONSTRING)"verbose", no_argument, &opt_verbose, OPT_VERBOSE},
+ {(OPTIONSTRING)"version", no_argument, NULL, OPT_VERSION},
+ {(OPTIONSTRING)"binary", no_argument, NULL, OPT_BINARY},
+ {(OPTIONSTRING)"udp", no_argument, NULL, OPT_UDP},
+ {0, 0, 0, 0},
+ };
-/* used to parse the options of command line */
-static void ms_options_parse(int argc, char *argv[])
-{
int option_index= 0;
int option_rv;
- while ((option_rv= getopt_long(argc, argv, "VhURbaBs:x:T:c:X:v:d:"
- "t:S:F:w:e:o:n:P:p:",
- long_options, &option_index)) != -1)
+ while (1)
{
+ option_rv= getopt_long(argc, argv, "Vhvds:", long_options, &option_index);
+ if (option_rv == -1) break;
switch (option_rv)
{
case 0:
break;
-
- case OPT_VERSION: /* --version or -V */
- ms_version_command(PROGRAM_NAME);
- break;
-
- case OPT_HELP: /* --help or -h */
- ms_help_command(PROGRAM_NAME, PROGRAM_DESCRIPTION);
- break;
-
- case OPT_SERVERS: /* --servers or -s */
- ms_setting.srv_str= strdup(optarg);
- break;
-
- case OPT_CONCURRENCY: /* --concurrency or -c */
- ms_setting.nconns= (uint32_t)strtoul(optarg, (char **) NULL, 10);
- if (ms_setting.nconns <= 0)
- {
- fprintf(stderr, "Concurrency must be greater than 0.:-)\n");
- exit(1);
- }
- break;
-
- case OPT_EXECUTE_NUMBER: /* --execute_number or -x */
- ms_setting.exec_num= (int)strtol(optarg, (char **) NULL, 10);
- if (ms_setting.exec_num <= 0)
- {
- fprintf(stderr, "Execute number must be greater than 0.:-)\n");
- exit(1);
- }
- break;
-
- case OPT_THREAD_NUMBER: /* --threads or -T */
- ms_setting.nthreads= (uint32_t)strtoul(optarg, (char **) NULL, 10);
- if (ms_setting.nthreads <= 0)
- {
- fprintf(stderr, "Threads number must be greater than 0.:-)\n");
- exit(1);
- }
- break;
-
- case OPT_FIXED_LTH: /* --fixed_size or -X */
- ms_setting.fixed_value_size= (size_t)strtoull(optarg, (char **) NULL, 10);
- if ((ms_setting.fixed_value_size <= 0)
- || (ms_setting.fixed_value_size > MAX_VALUE_SIZE))
- {
- fprintf(stderr, "Value size must be between 0 and 1M.:-)\n");
- exit(1);
- }
- break;
-
- case OPT_VERIFY: /* --verify or -v */
- ms_setting.verify_percent= atof(optarg);
- if ((ms_setting.verify_percent <= 0)
- || (ms_setting.verify_percent > 1.0))
+ case OPT_UDP:
+ if (opt_test == GET_TEST)
{
- fprintf(stderr, "Data verification rate must be "
- "greater than 0 and less than 1.0. :-)\n");
+ fprintf(stderr, "You can not run a get test in UDP mode. UDP mode "
+ "does not currently support get ops.\n");
exit(1);
}
+ opt_udp_io= 1;
break;
-
- case OPT_GETS_DIVISION: /* --division or -d */
- ms_setting.mult_key_num= (int)strtol(optarg, (char **) NULL, 10);
- if (ms_setting.mult_key_num <= 0)
- {
- fprintf(stderr, "Multi-get key number must be greater than 0.:-)\n");
- exit(1);
- }
+ case OPT_BINARY:
+ opt_binary = 1;
break;
-
- case OPT_TIME: /* --time or -t */
- ms_setting.run_time= (int)ms_parse_time();
- if (ms_setting.run_time == -1)
- {
- fprintf(stderr, "Please specify the run time. :-)\n"
- "'s' for second, 'm' for minute, 'h' for hour, "
- "'d' for day. e.g.: --time=24h (means 24 hours).\n");
- exit(1);
- }
-
- if (ms_setting.run_time == 0)
- {
- fprintf(stderr, "Running time can not be 0. :-)\n");
- exit(1);
- }
+ case OPT_VERBOSE: /* --verbose or -v */
+ opt_verbose = OPT_VERBOSE;
break;
-
- case OPT_CONFIG_CMD: /* --cfg_cmd or -F */
- ms_setting.cfg_file= strdup(optarg);
+ case OPT_DEBUG: /* --debug or -d */
+ opt_verbose = OPT_DEBUG;
break;
-
- case OPT_WINDOW_SIZE: /* --win_size or -w */
- ms_setting.win_size= (size_t)ms_parse_size();
- if (ms_setting.win_size == (size_t)-1)
- {
- fprintf(
- stderr,
- "Please specify the item window size. :-)\n"
- "e.g.: --win_size=10k (means 10k task window size).\n");
- exit(1);
- }
+ case OPT_VERSION: /* --version or -V */
+ version_command(PROGRAM_NAME);
break;
-
- case OPT_UDP: /* --udp or -U*/
- ms_setting.udp= true;
+ case OPT_HELP: /* --help or -h */
+ help_command(PROGRAM_NAME, PROGRAM_DESCRIPTION, long_options, help_options);
break;
-
- case OPT_EXPIRE: /* --exp_verify or -e */
- ms_setting.exp_ver_per= atof(optarg);
- if ((ms_setting.exp_ver_per <= 0) || (ms_setting.exp_ver_per > 1.0))
- {
- fprintf(stderr, "Expire time verification rate must be "
- "greater than 0 and less than 1.0. :-)\n");
- exit(1);
- }
+ case OPT_SERVERS: /* --servers or -s */
+ opt_servers= strdup(optarg);
break;
-
- case OPT_OVERWRITE: /* --overwrite or -o */
- ms_setting.overwrite_percent= atof(optarg);
- if ((ms_setting.overwrite_percent <= 0)
- || (ms_setting.overwrite_percent > 1.0))
+ case OPT_SLAP_TEST:
+ if (!strcmp(optarg, "get"))
{
- fprintf(stderr, "Objects overwrite rate must be "
- "greater than 0 and less than 1.0. :-)\n");
- exit(1);
+ if (opt_udp_io == 1)
+ {
+ fprintf(stderr, "You can not run a get test in UDP mode. UDP mode "
+ "does not currently support get ops.\n");
+ exit(1);
+ }
+ opt_test= GET_TEST ;
}
- break;
-
- case OPT_STAT_FREQ: /* --stat_freq or -S */
- ms_setting.stat_freq= (int)ms_parse_time();
- if (ms_setting.stat_freq == -1)
+ else if (!strcmp(optarg, "set"))
+ opt_test= SET_TEST;
+ else if (!strcmp(optarg, "mget"))
{
- fprintf(stderr, "Please specify the frequency of dumping "
- "statistic information. :-)\n"
- "'s' for second, 'm' for minute, 'h' for hour, "
- "'d' for day. e.g.: --time=24h (means 24 hours).\n");
- exit(1);
+ opt_test= MGET_TEST;
}
-
- if (ms_setting.stat_freq == 0)
+ else
{
- fprintf(stderr, "The frequency of dumping statistic information "
- "can not be 0. :-)\n");
+ fprintf(stderr, "Your test, %s, is not a known test\n", optarg);
exit(1);
}
break;
-
- case OPT_SOCK_PER_CONN: /* --conn_sock or -n */
- ms_setting.sock_per_conn= (uint32_t)strtoul(optarg, (char **) NULL, 10);
- if (ms_setting.sock_per_conn <= 0)
- {
- fprintf(stderr, "Number of socks of each concurrency "
- "must be greater than 0.:-)\n");
- exit(1);
- }
- break;
-
- case OPT_RECONNECT: /* --reconnect or -R */
- ms_setting.reconnect= true;
- break;
-
- case OPT_VERBOSE: /* --verbose or -b */
- ms_setting.verbose= true;
- break;
-
- case OPT_FACEBOOK_TEST: /* --facebook or -a */
- ms_setting.facebook_test= true;
+ case OPT_SLAP_CONCURRENCY:
+ opt_concurrency= (unsigned int)strtoul(optarg, (char **)NULL, 10);
break;
-
- case OPT_BINARY_PROTOCOL: /* --binary or -B */
- ms_setting.binary_prot= true;
+ case OPT_SLAP_EXECUTE_NUMBER:
+ opt_execute_number= (unsigned int)strtoul(optarg, (char **)NULL, 10);
break;
-
- case OPT_TPS: /* --tps or -P */
- ms_setting.expected_tps= (int)ms_parse_size();
- if (ms_setting.expected_tps == -1)
- {
- fprintf(stderr,
- "Please specify the item expected throughput. :-)\n"
- "e.g.: --tps=10k (means 10k throughput).\n");
- exit(1);
- }
+ case OPT_SLAP_INITIAL_LOAD:
+ opt_createial_load= (unsigned int)strtoul(optarg, (char **)NULL, 10);
break;
-
- case OPT_REP_WRITE_SRV: /* --rep_write or -p */
- ms_setting.rep_write_srv= (uint32_t)strtoul(optarg, (char **) NULL, 10);
- if (ms_setting.rep_write_srv <= 0)
- {
- fprintf(stderr,
- "Number of replication writing server must be greater "
- "than 0.:-)\n");
- exit(1);
- }
- break;
-
case '?':
/* getopt_long already printed an error message. */
exit(1);
-
default:
abort();
- } /* switch */
- }
-} /* ms_options_parse */
-
-
-static int ms_check_para()
-{
- if (ms_setting.srv_str == NULL)
- {
- char *temp;
-
- if ((temp= getenv("MEMCACHED_SERVERS")))
- {
- ms_setting.srv_str= strdup(temp);
- }
- else
- {
- fprintf(stderr, "No Servers provided\n\n");
- return -1;
}
}
- if (ms_setting.nconns % (uint32_t)ms_setting.nthreads != 0)
- {
- fprintf(stderr, "Concurrency must be the multiples of threads count.\n");
- return -1;
- }
-
- if (ms_setting.win_size % UNIT_ITEMS_COUNT != 0)
- {
- fprintf(stderr, "Window size must be the multiples of 1024.\n\n");
- return -1;
- }
-
- return EXIT_SUCCESS;
-} /* ms_check_para */
-
-
-/* initialize the statistic structure */
-static void ms_statistic_init()
-{
- pthread_mutex_init(&ms_statistic.stat_mutex, NULL);
- ms_init_stats(&ms_statistic.get_stat, "Get");
- ms_init_stats(&ms_statistic.set_stat, "Set");
- ms_init_stats(&ms_statistic.total_stat, "Total");
-} /* ms_statistic_init */
-
+ if ((opt_test == GET_TEST || opt_test == MGET_TEST) && opt_createial_load == 0)
+ opt_createial_load= DEFAULT_INITIAL_LOAD;
-/* initialize the global state structure */
-static void ms_stats_init()
-{
- memset(&ms_stats, 0, sizeof(ms_stats_t));
- if (ms_setting.stat_freq > 0)
- {
- ms_statistic_init();
- }
-} /* ms_stats_init */
+ if (opt_execute_number == 0)
+ opt_execute_number= DEFAULT_EXECUTE_NUMBER;
+ if (opt_concurrency == 0)
+ opt_concurrency= DEFAULT_CONCURRENCY;
+}
-/* use to output the statistic */
-static void ms_print_statistics(int in_time)
+void conclusions_print(conclusions_st *conclusion)
{
- int obj_size= (int)(ms_setting.avg_key_size + ms_setting.avg_val_size);
-
- printf("\033[1;1H\033[2J\n");
- ms_dump_format_stats(&ms_statistic.get_stat, in_time,
- ms_setting.stat_freq, obj_size);
- ms_dump_format_stats(&ms_statistic.set_stat, in_time,
- ms_setting.stat_freq, obj_size);
- ms_dump_format_stats(&ms_statistic.total_stat, in_time,
- ms_setting.stat_freq, obj_size);
-} /* ms_print_statistics */
-
+ printf("\tThreads connecting to servers %u\n", opt_concurrency);
+#ifdef NOT_FINISHED
+ printf("\tLoaded %u rows\n", conclusion->rows_loaded);
+ printf("\tRead %u rows\n", conclusion->rows_read);
+#endif
+ if (opt_test == SET_TEST)
+ printf("\tTook %ld.%03ld seconds to load data\n", conclusion->load_time / 1000,
+ conclusion->load_time % 1000);
+ else
+ printf("\tTook %ld.%03ld seconds to read data\n", conclusion->read_time / 1000,
+ conclusion->read_time % 1000);
+}
-/* used to print the states of memslap */
-static void ms_print_memslap_stats(struct timeval *start_time,
- struct timeval *end_time)
+void *run_task(void *p)
{
- char buf[1024];
- char *pos= buf;
-
- pos+= snprintf(pos,
- sizeof(buf), "cmd_get: %lu\n",
- (unsigned long) ms_stats.cmd_get);
- pos+= snprintf(pos,
- sizeof(buf) - (size_t)(pos -buf),
- "cmd_set: %lu\n",
- (unsigned long) ms_stats.cmd_set);
- pos+= snprintf(pos,
- sizeof(buf) - (size_t)(pos -buf),
- "get_misses: %lu\n",
- (unsigned long) ms_stats.get_misses);
-
- if (ms_setting.verify_percent > 0)
- {
- pos+= snprintf(pos,
- sizeof(buf) - (size_t)(pos -buf),
- "verify_misses: %lu\n",
- (unsigned long) ms_stats.vef_miss);
- pos+= snprintf(pos,
- sizeof(buf) - (size_t)(pos -buf),
- "verify_failed: %lu\n",
- (unsigned long) ms_stats.vef_failed);
- }
-
- if (ms_setting.exp_ver_per > 0)
- {
- pos+= snprintf(pos,
- sizeof(buf) - (size_t)(pos -buf),
- "expired_get: %lu\n",
- (unsigned long) ms_stats.exp_get);
- pos+= snprintf(pos,
- sizeof(buf) - (size_t)(pos -buf),
- "unexpired_unget: %lu\n",
- (unsigned long) ms_stats.unexp_unget);
- }
+ thread_context_st *context= (thread_context_st *)p;
+ memcached_st *memc;
- pos+= snprintf(pos,
- sizeof(buf) - (size_t)(pos -buf),
- "written_bytes: %lu\n",
- (unsigned long) ms_stats.bytes_written);
- pos+= snprintf(pos,
- sizeof(buf) - (size_t)(pos -buf),
- "read_bytes: %lu\n",
- (unsigned long) ms_stats.bytes_read);
- pos+= snprintf(pos,
- sizeof(buf) - (size_t)(pos -buf),
- "object_bytes: %lu\n",
- (unsigned long) ms_stats.obj_bytes);
-
- if (ms_setting.udp || ms_setting.facebook_test)
- {
- pos+= snprintf(pos,
- sizeof(buf) - (size_t)(pos -buf),
- "packet_disorder: %lu\n",
- (unsigned long) ms_stats.pkt_disorder);
- pos+= snprintf(pos,
- sizeof(buf) - (size_t)(pos -buf),
- "packet_drop: %lu\n",
- (unsigned long)ms_stats.pkt_drop);
- pos+= snprintf(pos,
- sizeof(buf) - (size_t)(pos -buf),
- "udp_timeout: %lu\n",
- (unsigned long)ms_stats.udp_timeout);
- }
-
- if (ms_setting.stat_freq > 0)
- {
- ms_dump_stats(&ms_statistic.get_stat);
- ms_dump_stats(&ms_statistic.set_stat);
- ms_dump_stats(&ms_statistic.total_stat);
- }
-
- int64_t time_diff= ms_time_diff(start_time, end_time);
- pos+= snprintf(pos,
- sizeof(buf) - (size_t)(pos -buf),
- "\nRun time: %.1fs Ops: %llu TPS: %.0Lf Net_rate: %.1fM/s\n",
- (double)time_diff / 1000000,
- (unsigned long long)(ms_stats.cmd_get + ms_stats.cmd_set),
- (ms_stats.cmd_get
- + ms_stats.cmd_set) / ((long double)time_diff / 1000000),
- (double)(
- ms_stats.bytes_written
- + ms_stats.bytes_read) / 1024 / 1024
- / ((double)time_diff / 1000000));
- assert(pos <= buf);
-
- fprintf(stdout, "%s", buf);
- fflush(stdout);
-} /* ms_print_memslap_stats */
-
-
-/* the loop of the main thread, wait the work threads to complete */
-static void ms_monitor_slap_mode()
-{
- int second= 0;
- struct timeval start_time, end_time;
+ memc= context->memc;
- /* Wait all the threads complete initialization. */
- pthread_mutex_lock(&ms_global.init_lock.lock);
- while (ms_global.init_lock.count < ms_setting.nthreads)
+ pthread_mutex_lock(&sleeper_mutex);
+ while (master_wakeup)
{
- pthread_cond_wait(&ms_global.init_lock.cond,
- &ms_global.init_lock.lock);
+ pthread_cond_wait(&sleep_threshhold, &sleeper_mutex);
}
- pthread_mutex_unlock(&ms_global.init_lock.lock);
+ pthread_mutex_unlock(&sleeper_mutex);
- /* only when there is no set operation it need warm up */
- if (ms_setting.cmd_distr[CMD_SET].cmd_prop < PROP_ERROR)
+ /* Do Stuff */
+ switch (context->test)
{
- /* Wait all the connects complete warm up. */
- pthread_mutex_lock(&ms_global.warmup_lock.lock);
- while (ms_global.warmup_lock.count < ms_setting.nconns)
- {
- pthread_cond_wait(&ms_global.warmup_lock.cond, &ms_global.warmup_lock.lock);
- }
- pthread_mutex_unlock(&ms_global.warmup_lock.lock);
+ case SET_TEST:
+ assert(context->execute_pairs);
+ execute_set(memc, context->execute_pairs, context->execute_number);
+ break;
+ case GET_TEST:
+ execute_get(memc, context->initial_pairs, context->initial_number);
+ break;
+ case MGET_TEST:
+ execute_mget(memc, (const char*const*)context->keys, context->key_lengths,
+ context->initial_number);
+ break;
+ default:
+ WATCHPOINT_ASSERT(context->test);
+ break;
}
- ms_global.finish_warmup= true;
- /* running in "run time" mode, user specify run time */
- if (ms_setting.run_time > 0)
- {
- gettimeofday(&start_time, NULL);
- while (1)
- {
- sleep(1);
- second++;
+ memcached_free(memc);
- if ((ms_setting.stat_freq > 0) && (second % ms_setting.stat_freq == 0)
- && (ms_stats.active_conns >= ms_setting.nconns)
- && (ms_stats.active_conns <= INT_MAX))
- {
- ms_print_statistics(second);
- }
+ if (context->execute_pairs)
+ pairs_free(context->execute_pairs);
- if (ms_setting.run_time <= second)
- {
- ms_global.time_out= true;
- break;
- }
+ free(context);
- /* all connections disconnect */
- if ((second > 5) && (ms_stats.active_conns == 0))
- {
- break;
- }
- }
- gettimeofday(&end_time, NULL);
- sleep(1); /* wait all threads clean up */
- }
- else
- {
- /* running in "execute number" mode, user specify execute number */
- gettimeofday(&start_time, NULL);
-
- /*
- * We loop until we know that all connects have cleaned up.
- */
- pthread_mutex_lock(&ms_global.run_lock.lock);
- while (ms_global.run_lock.count < ms_setting.nconns)
- {
- pthread_cond_wait(&ms_global.run_lock.cond, &ms_global.run_lock.lock);
- }
- pthread_mutex_unlock(&ms_global.run_lock.lock);
-
- gettimeofday(&end_time, NULL);
- }
+ pthread_mutex_lock(&counter_mutex);
+ thread_counter--;
+ pthread_cond_signal(&count_threshhold);
+ pthread_mutex_unlock(&counter_mutex);
- ms_print_memslap_stats(&start_time, &end_time);
-} /* ms_monitor_slap_mode */
+ return NULL;
+}
+void flush_all(memcached_st *memc)
+{
+ memcached_flush(memc, 0);
+}
-/* the main function */
-int main(int argc, char *argv[])
+pairs_st *load_create_data(memcached_st *memc, unsigned int number_of,
+ unsigned int *actual_loaded)
{
- srandom((unsigned int)time(NULL));
- ms_global_struct_init();
+ memcached_st *memc_clone;
+ pairs_st *pairs;
- /* initialization */
- ms_setting_init_pre();
- ms_options_parse(argc, argv);
- if (ms_check_para())
- {
- ms_help_command(PROGRAM_NAME, PROGRAM_DESCRIPTION);
- exit(1);
- }
- ms_setting_init_post();
- ms_stats_init();
- ms_thread_init();
+ memc_clone= memcached_clone(NULL, memc);
+ /* We always used non-blocking IO for load since it is faster */
+ memcached_behavior_set(memc_clone, MEMCACHED_BEHAVIOR_NO_BLOCK, 0);
- /* waiting work thread complete its task */
- ms_monitor_slap_mode();
+ pairs= pairs_generate(number_of, 400);
+ *actual_loaded= execute_set(memc_clone, pairs, number_of);
- /* clean up */
- ms_thread_cleanup();
- ms_global_struct_destroy();
- ms_setting_cleanup();
+ memcached_free(memc_clone);
- return EXIT_SUCCESS;
-} /* main */
+ return pairs;
+}
memerror.pop \
memflush.pop \
memrm.pop \
+ memaslap.pop \
memslap.pop \
memstat.pop
BUILT_SOURCES += ${GENERIC_PAGES}
memerror.html \
memflush.html \
memrm.html \
+ memaslap.html \
memslap.html \
memstat.html
memerror.pod \
memflush.pod \
memrm.pod \
+ memaslap.pod \
memslap.pod \
memstat.pod
EXTRA_DIST+= $(POD_FILES)
memerror.1 \
memflush.1 \
memrm.1 \
+ memaslap.1 \
memslap.1 \
memstat.1
--- /dev/null
+=head1 NAME
+
+memaslap - Load testing and benchmarking tool for memcached
+
+=head1 SYNOPSIS
+
+ memaslap [options]
+
+=head1 DESCRIPTION
+
+B<memaslap> is a load generation and benchmark tool for memcached(1)
+servers. It generates configurable workload such as threads, concurrencies, connections,
+run time, overwrite, miss rate, key size, value size, get/set proportion,
+expected throughput, and so on. Furthermore, it also supports data
+verification, expire-time verification, UDP, binary protocol, facebook test,
+replication test, multi-get and reconnection, etc.
+
+Memslap manages network connections like memcached with
+libevent. Each thread of memaslap is bound with a CPU core, all
+the threads don't communicate with each other, and there are several socket
+connections in each thread. Each connection keeps key size distribution,
+value size distribution, and command distribution by itself.
+
+You can specify servers via the B<--servers> option or via the
+environment variable C<MEMCACHED_SERVERS>.
+
+
+=head1 FEATURES
+
+Memslap is developed to for the following purposes:
+
+=over
+
+=item Manages network connections with libevent asynchronously.
+
+=item Set both TCP and UDP up to use non-blocking IO.
+
+=item Improves parallelism: higher performance in multi-threads environments.
+
+=item Improves time efficiency: faster processing speed.
+
+=item Generates key and value more efficiently; key size distribution and value size distribution are configurable.
+
+=item Supports get, multi-get, and set commands; command distribution is configurable.
+
+=item Supports controllable miss rate and overwrite rate.
+
+=item Supports data and expire-time verification.
+
+=item Supports dumping statistic information periodically.
+
+=item Supports thousands of TCP connections.
+
+=item Supports binary protocol.
+
+=item Supports facebook test (set with TCP and multi-get with UDP) and replication test.
+
+=back
+
+=head1 DETAILS
+
+=head2 Effective implementation of network.
+
+For memaslap, both TCP and UDP use non-blocking network IO. All
+the network events are managed by libevent as memcached. The network module
+of memaslap is similar to memcached. Libevent can ensure
+memaslap can handle network very efficiently.
+
+=head2 Effective implementation of multi-threads and concurrency
+
+Memslap has the similar implementation of multi-threads to
+memcached. Memslap creates one or more self-governed threads;
+each thread is bound with one CPU core if the system supports setting CPU
+core affinity.
+
+In addition, each thread has a libevent to manage the events of the network;
+each thread has one or more self-governed concurrencies; and each
+concurrency has one or more socket connections. All the concurrencies don’t
+communicate with each other even though they are in the same thread.
+
+Memslap can create thousands of socket connections, and each
+concurrency has tens of socket connections. Each concurrency randomly or
+sequentially selects one socket connection from its socket connection pool
+to run, so memaslap can ensure each concurrency handles one
+socket connection at any given time. Users can specify the number of
+concurrency and socket connections of each concurrency according to their
+expected workload.
+
+=head2 Effective implementation of generating key and value
+
+In order to improve time efficiency and space efficiency,
+memaslap creates a random characters table with 10M characters. All the
+suffixes of keys and values are generated from this random characters table.
+
+Memslap uses the offset in the character table and the length
+of the string to identify a string. It can save much memory.
+Each key contains two parts, a prefix and a suffix. The prefix is an
+uint64_t, 8 bytes. In order to verify the data set before,
+memaslap need to ensure each key is unique, so it uses the prefix to identify
+a key. The prefix cannot include illegal characters, such as ‘\r’, ‘\n’,
+‘\0’ and ‘ ‘. And memaslap has an algorithm to ensure that.
+
+Memslap doesn’t generate all the objects (key-value pairs) at
+the beginning. It only generates enough objects to fill the task window
+(default 10K objects) of each concurrency. Each object has the following
+basic information, key prefix, key suffix offset in the character table, key
+length, value offset in the character table, and value length.
+
+In the work process, each concurrency sequentially or randomly selects an
+object from the window to do set operation or get operation. At the same
+time, each concurrency kicks objects out of its window and adds new object
+into it.
+
+=head2 Simple but useful task scheduling
+
+Memslap uses libevent to schedule all the concurrencies of
+threads, and each concurrency schedules tasks based on the local task
+window. Memslap assumes that if each concurrency keeps the same
+key distribution, value distribution and commands distribution, from
+outside, memaslap keeps all the distribution as a whole.
+Each task window includes a lot of objects, each object stores its basic
+information, such as key, value, expire time, and so on. At any time, all
+the objects in the window keep the same and fixed key and value
+distribution. If an object is overwritten, the value of the object will be
+updated. Memslap verifies the data or expire-time according to
+the object information stored in the task window.
+
+Libevent selects which concurrency to handle based on a specific network
+event. Then the concurrency selects which command (get or set) to operate
+based on the command distribution. If it needs to kick out an old object and
+add a new object, in order to keep the same key and value distribution, the
+new object must have the same key length and value length.
+
+If memcached server has two cache layers (memory and SSD), running
+memaslap with different window sizes can get different cache
+miss rates. If memaslap adds enough objects into the windows at
+the beginning, and the cache of memcached cannot store all the objects
+initialized, then memaslap will get some objects from the second
+cache layer. It causes the first cache layer to miss. So the user can
+specify the window size to get the expected miss rate of the first cache
+layer.
+
+=head2 Useful implementation of multi-servers , UDP, TCP, multi-get and binary protocol
+
+Because each thread is self-governed, memaslap can assign
+different threads to handle different memcached servers. This is just one of
+the ways in which memaslap supports multiple servers. The only
+limitation is that the number of servers cannot be greater than the number
+of threads. The other way to support multiple servers is for replication
+test. Each concurrency has one socket connection to each memcached server.
+For the implementation, memaslap can set some objects to one
+memcached server, and get these objects from the other servers.
+
+By default, Memslap does single get. If the user specifies
+multi-get option, memaslap will collect enough get commands and
+pack and send the commands together.
+
+Memslap supports both the ASCII protocol and binary protocol,
+but it runs on the ASCII protocol by default.
+Memslap by default runs on the TCP protocol, but it also
+supports UDP. Because UDP is unreliable, dropped packages and out-of-order
+packages may occur. Memslap creates a memory buffer to handle
+these problems. Memslap tries to read all the response data of
+one command from the server and reorders the response data. If some packages
+get lost, the waiting timeout mechanism can ensure half-baked packages will
+be discarded and the next command will be sent.
+
+
+=head1 USAGE
+
+Below are some usage samples:
+
+=over 4
+
+=item memaslap -s 127.0.0.1:11211 -S 5s
+
+=item memaslap -s 127.0.0.1:11211 -t 2m -v 0.2 -e 0.05 -b
+
+=item memaslap -s 127.0.0.1:11211 -F config -t 2m -w 40k -S 20s -o 0.2
+
+=item memaslap -s 127.0.0.1:11211 -F config -t 2m -T 4 -c 128 -d 20 -P 40k
+
+=item memaslap -s 127.0.0.1:11211 -F config -t 2m -d 50 -a -n 40
+
+=item memaslap -s 127.0.0.1:11211,127.0.0.1:11212 -F config -t 2m
+
+=item memaslap -s 127.0.0.1:11211,127.0.0.1:11212 -F config -t 2m -p 2
+
+=back
+
+The user must specify one server at least to run memaslap. The
+rest of the parameters have default values, as shown below:
+
+Thread number = 1 Concurrency = 16
+
+Run time = 600 seconds Configuration file = NULL
+
+Key size = 64 Value size = 1024
+
+Get/set = 9:1 Window size = 10k
+
+Execute number = 0 Single get = true
+
+Multi-get = false Number of sockets of each concurrency = 1
+
+Reconnect = false Data verification = false
+
+Expire-time verification = false ASCII protocol = true
+
+Binary protocol = false Dumping statistic information
+
+periodically = false
+
+Overwrite proportion = 0% UDP = false
+
+TCP = true Limit throughput = false
+
+Facebook test = false Replication test = false
+
+=head2 Key size, value size and command distribution.
+
+All the distributions are read from the configuration file specified by user
+with “—cfg_cmd” option. If the user does not specify a configuration file,
+memaslap will run with the default distribution (key size = 64,
+value size = 1024, get/set = 9:1). For information on how to edit the
+configuration file, refer to the “Configuration File” section.
+
+The minimum key size is 16 bytes; the maximum key size is 250 bytes. The
+precision of proportion is 0.001. The proportion of distribution will be
+rounded to 3 decimal places.
+
+The minimum value size is 1 bytes; the maximum value size is 1M bytes. The
+precision of proportion is 0.001. The proportion of distribution will be
+rounded to 3 decimal places.
+Currently, memaslap only supports set and get commands. And it
+supports 100% set and 100% get. For 100% get, it will preset some objects to
+the server.
+
+=head2 Multi-thread and concurrency
+
+The high performance of memaslap benefits from the special
+schedule of thread and concurrency. It’s important to specify the proper
+number of them. The default number of threads is 1; the default number of
+concurrency is 16. The user can use “—threads” and “--concurrency” to
+specify these variables.
+
+If the system supports setting CPU affinity and the number of threads
+specified by the user is greater than 1, memaslap will try to
+bind each thread to a different CPU core. So if you want to get the best
+performance memaslap, it is better to specify the number of
+thread equal to the number of CPU cores. The number of threads specified by
+the user can also be less or greater than the number of CPU cores. Because
+of the limitation of implementation, the number of concurrencies could be
+the multiple of the number of threads.
+
+1. For 8 CPU cores system
+
+For example:
+
+--threads=2 --concurrency=128
+
+--threads=8 --concurrency=128
+
+--threads=8 --concurrency=256
+
+--threads=12 --concurrency=144
+
+2. For 16 CPU cores system
+
+For example:
+
+--threads=8 --concurrency=128
+
+--threads=16 --concurrency=256
+
+--threads=16 --concurrency=512
+
+--threads=24 --concurrency=288
+
+The memaslap performs very well, when
+used to test the performance of memcached servers.
+Most of the time, the bottleneck is the network or
+the server. If for some reason the user wants to
+limit the performance of memaslap, there
+are two ways to do this:
+
+Decrease the number of threads and concurrencies.
+Use the option “--tps” that memaslap
+provides to limit the throughput. This option allows
+the user to get the expected throughput. For
+example, assume that the maximum throughput is 50
+kops/s for a specific configuration, you can specify
+the throughput equal to or less than the maximum
+throughput using “--tps” option.
+
+=head2 Window size
+
+Most of the time, the user does not need to specify the window size. The
+default window size is 10k. For Schooner Memcached, the user can specify
+different window sizes to get different cache miss rates based on the test
+case. Memslap supports cache miss rate between 0% and 100%.
+If you use this utility to test the performance of Schooner Memcached, you
+can specify a proper window size to get the expected cache miss rate. The
+formula for calculating window size is as follows:
+
+Assume that the key size is 128 bytes, and the value size is 2048 bytes, and
+concurrency=128.
+
+1. Small cache cache_size=1M, 100% cache miss (all data get from SSD).
+win_size=10k
+
+2. cache_size=4G
+
+(1). cache miss rate 0%
+
+win_size=8k
+
+(2). cache miss rate 5%
+
+win_size=11k
+
+3. cache_size=16G
+
+(1). cache miss rate 0%
+
+win_size=32k
+
+(2). cache miss
+
+rate 5%
+
+win_size=46k
+
+The formula for calculating window size for cache miss rate 0%:
+
+cache_size / concurrency / (key_size + value_size) * 0.5
+
+The formula for calculating window size for cache miss rate 5%:
+
+cache_size / concurrency / (key_size + value_size) * 0.7
+
+=head2 Verification
+
+Memslap supports both data verification and expire-time
+verification. The user can use "--verify=" or "-v" to specify the proportion
+of data verification. In theory, it supports 100% data verification. The
+user can use "--exp_verify=" or "-e" to specify the proportion of
+expire-time verification. In theory, it supports 100% expire-time
+verification. Specify the "--verbose" options to get more detailed error
+information.
+
+For example: --exp_verify=0.01 –verify=0.1 , it means that 1% of the objects
+set with expire-time, 10% of the objects gotten will be verified. If the
+objects are gotten, memaslap will verify the expire-time and
+value.
+
+=head2 multi-servers and multi-clients
+
+Memslap supports multi-servers based on self-governed thread.
+There is a limitation that the number of servers cannot be greater than the
+number of threads. Memslap assigns one thread to handle one
+server at least. The user can use the "--servers=" or "-s" option to specify
+multi-servers.
+
+For example:
+
+--servers=10.1.1.1:11211,10.1.1.2:11212,10.1.1.3:11213 --threads=6 --concurrency=36
+
+The above command means that there are 6 threads, with each thread having 6
+concurrencies and that threads 0 and 3 handle server 0 (10.1.1.1); threads 1
+and 4 handle server 1 (10.1.1.2); and thread 2 and 5 handle server 2
+(10.1.1.3).
+
+All the threads and concurrencies in memaslap are self-governed.
+
+So is memaslap. The user can start up several
+memaslap instances. The user can run memaslap on different client
+machines to communicate with the same memcached server at the same. It is
+recommended that the user start different memaslap on different
+machines using the same configuration.
+
+=head2 Run with execute number mode or time mode
+
+The default memaslap runs with time mode. The default run time
+is 10 minutes. If it times out, memaslap will exit. Do not
+specify both execute number mode and time mode at the same time; just
+specify one instead.
+
+For example:
+
+--time=30s (It means the test will run 30 seconds.)
+
+--execute_number=100000 (It means that after running 100000 commands, the test will exit.)
+
+=head2 Dump statistic information periodically.
+
+The user can use "--stat_freq=" or "-S" to specify the frequency.
+
+For example:
+
+--stat_freq=20s
+
+Memslap will dump the statistics of the commands (get and set) at the frequency of every 20
+seconds.
+
+For more information on the format of dumping statistic information, refer to “Format of Output” section.
+
+=head2 Multi-get
+
+The user can use "--division=" or "-d" to specify multi-get keys count.
+Memslap by default does single get with TCP. Memslap also supports data
+verification and expire-time verification for multi-get.
+
+Memslap supports multi-get with both TCP and UDP. Because of
+the different implementation of the ASCII protocol and binary protocol,
+there are some differences between the two. For the ASCII protocol,
+memaslap sends one “multi-get” to the server once. For the
+binary protocol, memaslap sends several single get commands
+together as “multi-get” to the server.
+
+=head2 UDP and TCP
+
+Memslap supports both UDP and TCP. For TCP,
+memaslap does not reconnect the memcached server if socket connections are
+lost. If all the socket connections are lost or memcached server crashes,
+memaslap will exit. If the user specifies the “--reconnect”
+option when socket connections are lost, it will reconnect them.
+
+User can use “--udp” to enable the UDP feature, but UDP comes with some
+limitations:
+
+UDP cannot set data more than 1400 bytes.
+
+UDP is not supported by the binary protocol because the binary protocol of
+memcached does not support that.
+
+UDP doesn’t support reconnection.
+
+=head2 Facebook test
+
+Set data with TCP and multi-get with UDP. Specify the following options:
+
+"--facebook --division=50"
+
+If you want to create thousands of TCP connections, specify the
+
+"--conn_sock=" option.
+
+For example: --facebook --division=50 --conn_sock=200
+
+The above command means that memaslap will do facebook test,
+each concurrency has 200 socket TCP connections and one UDP socket.
+
+Memslap sets objects with the TCP socket, and multi-gets 50
+objects once with the UDP socket.
+
+If you specify "--division=50", the key size must be less that 25 bytes
+because the UDP packet size is 1400 bytes.
+
+=head2 Replication test
+
+For replication test, the user must specify at least two memcached servers.
+The user can use “—rep_write=” option to enable feature.
+
+For example:
+
+--servers=10.1.1.1:11211,10.1.1.2:11212 –rep_write=2
+
+The above command means that there are 2 replication memcached servers,
+memaslap will set objects to both server 0 and server 1, get
+objects which are set to server 0 before from server 1, and also get objects
+which are set to server 1 before from server 0. If server 0 crashes,
+memaslap will only get objects from server 1. If server 0 comes
+back to life again, memaslap will reconnect server 0. If both
+server 0 and server 1 crash, memaslap will exit.
+
+=head2 Supports thousands of TCP connections
+
+Start memaslap with "--conn_sock=" or "-n" to enable this
+feature. Make sure that your system can support opening thousands of files
+and creating thousands of sockets. However, this feature does not support
+reconnection if sockets disconnect.
+
+For example:
+
+--threads=8 --concurrency=128 --conn_sock=128
+
+The above command means that memaslap starts up 8 threads, each
+thread has 16 concurrencies, each concurrency has 128 TCP socket
+connections, and the total number of TCP socket connections is 128 * 128 =
+16384.
+
+=head2 Supports binary protocol
+
+Start memaslap with "--binary" or "-B" options to enable this
+feature. It supports all the above features except UDP, because the latest
+memcached 1.3.3 does not implement binary UDP protocol.
+
+For example:
+
+--binary
+
+Since memcached 1.3.3 doesn't implement binary UDP protocol,
+memaslap does not support UDP. In addition, memcached 1.3.3 does not support
+multi-get. If you specify "--division=50" option, it just sends 50 get
+commands together as “mulit-get” to the server.
+
+=head1 Configuration file
+
+This section describes the format of the configuration file. By default
+when no configuration file is specified memaslap reads the default
+one located at ~/.memaslap.cnf.
+
+Below is a sample configuration file:
+
+ ***************************************************************************
+ #comments should start with '#'
+ #key
+ #start_len end_len proportion
+ #
+ #key length range from start_len to end_len
+ #start_len must be equal to or greater than 16
+ #end_len must be equal to or less than 250
+ #start_len must be equal to or greater than end_len
+ #memaslap will generate keys according to the key range
+ #proportion: indicates keys generated from one range accounts for the total
+ generated keys
+ #
+ #example1: key range 16~100 accounts for 80%
+ # key range 101~200 accounts for 10%
+ # key range 201~250 accounts for 10%
+ # total should be 1 (0.8+0.1+0.1 = 1)
+ #
+ # 16 100 0.8
+ # 101 200 0.1
+ # 201 249 0.1
+ #
+ #example2: all keys length are 128 bytes
+ #
+ # 128 128 1
+ key
+ 128 128 1
+ #value
+ #start_len end_len proportion
+ #
+ #value length range from start_len to end_len
+ #start_len must be equal to or greater than 1
+ #end_len must be equal to or less than 1M
+ #start_len must be equal to or greater than end_len
+ #memaslap will generate values according to the value range
+ #proportion: indicates values generated from one range accounts for the
+ total generated values
+ #
+ #example1: value range 1~1000 accounts for 80%
+ # value range 1001~10000 accounts for 10%
+ # value range 10001~100000 accounts for 10%
+ # total should be 1 (0.8+0.1+0.1 = 1)
+ #
+ # 1 1000 0.8
+ # 1001 10000 0.1
+ # 10001 100000 0.1
+ #
+ #example2: all value length are 128 bytes
+ #
+ # 128 128 1
+ value
+ 2048 2048 1
+ #cmd
+ #cmd_type cmd_proportion
+ #
+ #currently memaslap only supports get and set command.
+ #
+ #cmd_type
+ #set 0
+ #get 1
+ #
+ #example: set command accounts for 50%
+ # get command accounts for 50%
+ # total should be 1 (0.5+0.5 = 1)
+ #
+ # cmd
+ # 0 0.5
+ # 1 0.5
+ cmd
+ 0 0.1
+ 1.0 0.9
+
+
+
+=head1 Format of output
+
+At the beginning, memaslap displays some configuration information as follows:
+
+=over 4
+
+=item servers : 127.0.0.1:11211
+
+=item threads count: 1
+
+=item concurrency: 16
+
+=item run time: 20s
+
+=item windows size: 10k
+
+=item set proportion: set_prop=0.10
+
+=item get proportion: get_prop=0.90
+
+=back
+
+=head2 Where
+
+=over 4
+
+=item servers : "servers"
+
+The servers used by memaslap.
+
+=item threads count
+
+The number of threads memaslap runs with.
+
+=item concurrency
+
+The number of concurrencies memaslap runs with.
+
+=item run time
+
+How long to run memaslap.
+
+=item windows size
+
+The task window size of each concurrency.
+
+=item set proportion
+
+The proportion of set command.
+
+=item get proportion
+
+The proportion of get command.
+
+=back
+
+The output of dynamic statistics is something like this:
+
+ ---------------------------------------------------------------------------------------------------------------------------------
+ Get Statistics
+ Type Time(s) Ops TPS(ops/s) Net(M/s) Get_miss Min(us) Max(us)
+ Avg(us) Std_dev Geo_dist
+ Period 5 345826 69165 65.3 0 27 2198 203
+ 95.43 177.29
+ Global 20 1257935 62896 71.8 0 26 3791 224
+ 117.79 192.60
+
+
+ Set Statistics
+ Type Time(s) Ops TPS(ops/s) Net(M/s) Get_miss Min(us) Max(us)
+ Avg(us) Std_dev Geo_dist
+ Period 5 38425 7685 7.3 0 42 628 240
+ 88.05 220.21
+ Global 20 139780 6989 8.0 0 37 3790 253
+ 117.93 224.83
+
+
+ Total Statistics
+ Type Time(s) Ops TPS(ops/s) Net(M/s) Get_miss Min(us) Max(us)
+ Avg(us) Std_dev Geo_dist
+ Period 5 384252 76850 72.5 0 27 2198 207
+ 94.72 181.18
+ Global 20 1397720 69886 79.7 0 26 3791 227
+ 117.93 195.60
+ ---------------------------------------------------------------------------------------------------------------------------------
+
+=head2 Where
+
+=over 4
+
+=item Get Statistics
+
+Statistics information of get command
+
+=item Set Statistics
+
+Statistics information of set command
+
+=item Total Statistics
+
+Statistics information of both get and set command
+
+=item Period
+
+Result within a period
+
+=item Global
+
+Accumulated results
+
+=item Ops
+
+Total operations
+
+=item TPS
+
+Throughput, operations/second
+
+=item Net
+
+The rate of network
+
+=item Get_miss
+
+How many objects can’t be gotten
+
+=item Min
+
+The minimum response time
+
+=item Max
+
+The maximum response time
+
+=item Avg:
+
+The average response time
+
+=item Std_dev
+
+Standard deviation of response time
+
+=item Geo_dist
+
+Geometric distribution based on natural exponential function
+
+=back
+
+At the end, memaslap will output something like this:
+
+ ---------------------------------------------------------------------------------------------------------------------------------
+ Get Statistics (1257956 events)
+ Min: 26
+ Max: 3791
+ Avg: 224
+ Geo: 192.60
+ Std: 116.23
+ Log2 Dist:
+ 4: 0 10 84490 215345
+ 8: 484890 459823 12543 824
+ 12: 31
+
+ Set Statistics (139782 events)
+ Min: 37
+ Max: 3790
+ Avg: 253
+ Geo: 224.84
+ Std: 116.83
+ Log2 Dist:
+ 4: 0 0 4200 16988
+ 8: 50784 65574 2064 167
+ 12: 5
+
+ Total Statistics (1397738 events)
+ Min: 26
+ Max: 3791
+ Avg: 227
+ Geo: 195.60
+ Std: 116.60
+ Log2 Dist:
+ 4: 0 10 88690 232333
+ 8: 535674 525397 14607 991
+ 12: 36
+
+ cmd_get: 1257969
+ cmd_set: 139785
+ get_misses: 0
+ verify_misses: 0
+ verify_failed: 0
+ expired_get: 0
+ unexpired_unget: 0
+ written_bytes: 242516030
+ read_bytes: 1003702556
+ object_bytes: 152086080
+ packet_disorder: 0
+ packet_drop: 0
+ udp_timeout: 0
+
+ Run time: 20.0s Ops: 1397754 TPS: 69817 Net_rate: 59.4M/s
+ ---------------------------------------------------------------------------------------------------------------------------------
+
+=head2 Where
+
+=over 4
+
+=item Get Statistics
+
+Get statistics of response time
+
+=item Set Statistics
+
+Set statistics of response time
+
+=item Total Statistics
+
+Both get and set statistics of response time
+
+=item Min
+
+The accumulated and minimum response time
+
+=item Max
+
+The accumulated and maximum response time
+
+=item Avg
+
+The accumulated and average response time
+
+=item Std
+
+Standard deviation of response time
+
+=item Log2 Dist
+
+Geometric distribution based on logarithm 2
+
+=item cmd_get
+
+Total get commands done
+
+=item cmd_set
+
+Total set commands done
+
+=item get_misses
+
+How many objects can’t be gotten from server
+
+=item verify_misses
+
+How many objects need to verify but can’t get them
+
+=item verify_failed
+
+How many objects with insistent value
+
+=item expired_get
+
+How many objects are expired but we get them
+
+=item unexpired_unget
+
+How many objects are unexpired but we can’t get them
+
+=item written_bytes
+
+Total written bytes
+
+=item read_bytes
+
+Total read bytes
+
+=item object_bytes
+
+Total object bytes
+
+=item packet_disorder
+
+How many UDP packages are disorder
+
+=item packet_drop
+
+How many UDP packages are lost
+
+=item udp_timeout
+
+How many times UDP time out happen
+
+=item Run time
+
+Total run time
+
+=item Ops
+
+Total operations
+
+=item TPS
+
+Throughput, operations/second
+
+=item Net_rate
+
+The average rate of network
+
+=back
+
+=head1 OPTIONS
+
+-s, --servers=
+ List one or more servers to connect. Servers count must be less than
+ threads count. e.g.: --servers=localhost:1234,localhost:11211
+
+-T, --threads=
+ Number of threads to startup, better equal to CPU numbers. Default 8.
+
+-c, --concurrency=
+ Number of concurrency to simulate with load. Default 128.
+
+-n, --conn_sock=
+ Number of TCP socks per concurrency. Default 1.
+
+-x, --execute_number=
+ Number of operations(get and set) to execute for the
+ given test. Default 1000000.
+
+-t, --time=
+ How long the test to run, suffix: s-seconds, m-minutes, h-hours,
+ d-days e.g.: --time=2h.
+
+-F, --cfg_cmd=
+ Load the configure file to get command,key and value distribution list.
+
+-w, --win_size=
+ Task window size of each concurrency, suffix: K, M e.g.: --win_size=10k.
+ Default 10k.
+
+-X, --fixed_size=
+ Fixed length of value.
+
+-v, --verify=
+ The proportion of date verification, e.g.: --verify=0.01
+
+-d, --division=
+ Number of keys to multi-get once. Default 1, means single get.
+
+-S, --stat_freq=
+ Frequency of dumping statistic information. suffix: s-seconds,
+ m-minutes, e.g.: --resp_freq=10s.
+
+-e, --exp_verify=
+ The proportion of objects with expire time, e.g.: --exp_verify=0.01.
+ Default no object with expire time
+
+-o, --overwrite=
+ The proportion of objects need overwrite, e.g.: --overwrite=0.01.
+ Default never overwrite object.
+
+-R, --reconnect
+ Reconnect support, when connection is closed it will be reconnected.
+
+-U, --udp
+ UDP support, default memaslap uses TCP, TCP port and UDP port of
+ server must be same.
+
+-a, --facebook
+ Whether it enables facebook test feature, set with TCP and multi-get with UDP.
+
+-B, --binary
+ Whether it enables binary protocol. Default with ASCII protocol.
+
+-P, --tps=
+ Expected throughput, suffix: K, e.g.: --tps=10k.
+
+-p, --rep_write=
+ The first nth servers can write data, e.g.: --rep_write=2.
+
+-b, --verbose
+ Whether it outputs detailed information when verification fails.
+
+-h, --help
+ Display this message and then exit.
+
+-V, --version
+ Display the version of the application and then exit.
+
+=head1 EXAMPLES
+
+memaslap -s 127.0.0.1:11211 -S 5s
+
+memaslap -s 127.0.0.1:11211 -t 2m -v 0.2 -e 0.05 -b
+
+memaslap -s 127.0.0.1:11211 -F config -t 2m -w 40k -S 20s -o 0.2
+
+memaslap -s 127.0.0.1:11211 -F config -t 2m -T 4 -c 128 -d 20 -P 40k
+
+memaslap -s 127.0.0.1:11211 -F config -t 2m -d 50 -a -n 40
+
+memaslap -s 127.0.0.1:11211,127.0.0.1:11212 -F config -t 2m
+
+memaslap -s 127.0.0.1:11211,127.0.0.1:11212 -F config -t 2m -p 2
+
+=head1 HOME
+
+To find out more information please check:
+L<http://launchpad.org/libmemcached>
+
+=head1 AUTHORS
+
+Mingqiang Zhuang E<lt>mingqiangzhuang@hengtiansoft.comE<gt> (Schooner Technolgy)
+Brian Aker, E<lt>brian@tangent.orgE<gt>
+
+=head1 SEE ALSO
+
+memcached(1) libmemcached(3)
+
+=cut
+
=head1 DESCRIPTION
B<memslap> is a load generation and benchmark tool for memcached(1)
-servers. It generates configurable workload such as threads, concurrencies, connections,
-run time, overwrite, miss rate, key size, value size, get/set proportion,
-expected throughput, and so on. Furthermore, it also supports data
-verification, expire-time verification, UDP, binary protocol, facebook test,
-replication test, multi-get and reconnection, etc.
-
-Memslap manages network connections like memcached with
-libevent. Each thread of memslap is bound with a CPU core, all
-the threads don't communicate with each other, and there are several socket
-connections in each thread. Each connection keeps key size distribution,
-value size distribution, and command distribution by itself.
+servers. It simulates loads on memcached server clusters.
You can specify servers via the B<--servers> option or via the
environment variable C<MEMCACHED_SERVERS>.
-
-=head1 FEATURES
-
-Memslap is developed to for the following purposes:
-
-=over
-
-=item Manages network connections with libevent asynchronously.
-
-=item Set both TCP and UDP up to use non-blocking IO.
-
-=item Improves parallelism: higher performance in multi-threads environments.
-
-=item Improves time efficiency: faster processing speed.
-
-=item Generates key and value more efficiently; key size distribution and value size distribution are configurable.
-
-=item Supports get, multi-get, and set commands; command distribution is configurable.
-
-=item Supports controllable miss rate and overwrite rate.
-
-=item Supports data and expire-time verification.
-
-=item Supports dumping statistic information periodically.
-
-=item Supports thousands of TCP connections.
-
-=item Supports binary protocol.
-
-=item Supports facebook test (set with TCP and multi-get with UDP) and replication test.
-
-=back
-
-=head1 DETAILS
-
-=head2 Effective implementation of network.
-
-For memslap, both TCP and UDP use non-blocking network IO. All
-the network events are managed by libevent as memcached. The network module
-of memslap is similar to memcached. Libevent can ensure
-memslap can handle network very efficiently.
-
-=head2 Effective implementation of multi-threads and concurrency
-
-Memslap has the similar implementation of multi-threads to
-memcached. Memslap creates one or more self-governed threads;
-each thread is bound with one CPU core if the system supports setting CPU
-core affinity.
-
-In addition, each thread has a libevent to manage the events of the network;
-each thread has one or more self-governed concurrencies; and each
-concurrency has one or more socket connections. All the concurrencies don’t
-communicate with each other even though they are in the same thread.
-
-Memslap can create thousands of socket connections, and each
-concurrency has tens of socket connections. Each concurrency randomly or
-sequentially selects one socket connection from its socket connection pool
-to run, so memslap can ensure each concurrency handles one
-socket connection at any given time. Users can specify the number of
-concurrency and socket connections of each concurrency according to their
-expected workload.
-
-=head2 Effective implementation of generating key and value
-
-In order to improve time efficiency and space efficiency,
-memslap creates a random characters table with 10M characters. All the
-suffixes of keys and values are generated from this random characters table.
-
-Memslap uses the offset in the character table and the length
-of the string to identify a string. It can save much memory.
-Each key contains two parts, a prefix and a suffix. The prefix is an
-uint64_t, 8 bytes. In order to verify the data set before,
-memslap need to ensure each key is unique, so it uses the prefix to identify
-a key. The prefix cannot include illegal characters, such as ‘\r’, ‘\n’,
-‘\0’ and ‘ ‘. And memslap has an algorithm to ensure that.
-
-Memslap doesn’t generate all the objects (key-value pairs) at
-the beginning. It only generates enough objects to fill the task window
-(default 10K objects) of each concurrency. Each object has the following
-basic information, key prefix, key suffix offset in the character table, key
-length, value offset in the character table, and value length.
-
-In the work process, each concurrency sequentially or randomly selects an
-object from the window to do set operation or get operation. At the same
-time, each concurrency kicks objects out of its window and adds new object
-into it.
-
-=head2 Simple but useful task scheduling
-
-Memslap uses libevent to schedule all the concurrencies of
-threads, and each concurrency schedules tasks based on the local task
-window. Memslap assumes that if each concurrency keeps the same
-key distribution, value distribution and commands distribution, from
-outside, memslap keeps all the distribution as a whole.
-Each task window includes a lot of objects, each object stores its basic
-information, such as key, value, expire time, and so on. At any time, all
-the objects in the window keep the same and fixed key and value
-distribution. If an object is overwritten, the value of the object will be
-updated. Memslap verifies the data or expire-time according to
-the object information stored in the task window.
-
-Libevent selects which concurrency to handle based on a specific network
-event. Then the concurrency selects which command (get or set) to operate
-based on the command distribution. If it needs to kick out an old object and
-add a new object, in order to keep the same key and value distribution, the
-new object must have the same key length and value length.
-
-If memcached server has two cache layers (memory and SSD), running
-memslap with different window sizes can get different cache
-miss rates. If memslap adds enough objects into the windows at
-the beginning, and the cache of memcached cannot store all the objects
-initialized, then memslap will get some objects from the second
-cache layer. It causes the first cache layer to miss. So the user can
-specify the window size to get the expected miss rate of the first cache
-layer.
-
-=head2 Useful implementation of multi-servers , UDP, TCP, multi-get and binary protocol
-
-Because each thread is self-governed, memslap can assign
-different threads to handle different memcached servers. This is just one of
-the ways in which memslap supports multiple servers. The only
-limitation is that the number of servers cannot be greater than the number
-of threads. The other way to support multiple servers is for replication
-test. Each concurrency has one socket connection to each memcached server.
-For the implementation, memslap can set some objects to one
-memcached server, and get these objects from the other servers.
-
-By default, Memslap does single get. If the user specifies
-multi-get option, memslap will collect enough get commands and
-pack and send the commands together.
-
-Memslap supports both the ASCII protocol and binary protocol,
-but it runs on the ASCII protocol by default.
-Memslap by default runs on the TCP protocol, but it also
-supports UDP. Because UDP is unreliable, dropped packages and out-of-order
-packages may occur. Memslap creates a memory buffer to handle
-these problems. Memslap tries to read all the response data of
-one command from the server and reorders the response data. If some packages
-get lost, the waiting timeout mechanism can ensure half-baked packages will
-be discarded and the next command will be sent.
-
-
-=head1 USAGE
-
-Below are some usage samples:
-
-=over 4
-
-=item memslap -s 127.0.0.1:11211 -S 5s
-
-=item memslap -s 127.0.0.1:11211 -t 2m -v 0.2 -e 0.05 -b
-
-=item memslap -s 127.0.0.1:11211 -F config -t 2m -w 40k -S 20s -o 0.2
-
-=item memslap -s 127.0.0.1:11211 -F config -t 2m -T 4 -c 128 -d 20 -P 40k
-
-=item memslap -s 127.0.0.1:11211 -F config -t 2m -d 50 -a -n 40
-
-=item memslap -s 127.0.0.1:11211,127.0.0.1:11212 -F config -t 2m
-
-=item memslap -s 127.0.0.1:11211,127.0.0.1:11212 -F config -t 2m -p 2
-
-=back
-
-The user must specify one server at least to run memslap. The
-rest of the parameters have default values, as shown below:
-
-Thread number = 1 Concurrency = 16
-
-Run time = 600 seconds Configuration file = NULL
-
-Key size = 64 Value size = 1024
-
-Get/set = 9:1 Window size = 10k
-
-Execute number = 0 Single get = true
-
-Multi-get = false Number of sockets of each concurrency = 1
-
-Reconnect = false Data verification = false
-
-Expire-time verification = false ASCII protocol = true
-
-Binary protocol = false Dumping statistic information
-
-periodically = false
-
-Overwrite proportion = 0% UDP = false
-
-TCP = true Limit throughput = false
-
-Facebook test = false Replication test = false
-
-=head2 Key size, value size and command distribution.
-
-All the distributions are read from the configuration file specified by user
-with “—cfg_cmd” option. If the user does not specify a configuration file,
-memslap will run with the default distribution (key size = 64,
-value size = 1024, get/set = 9:1). For information on how to edit the
-configuration file, refer to the “Configuration File” section.
-
-The minimum key size is 16 bytes; the maximum key size is 250 bytes. The
-precision of proportion is 0.001. The proportion of distribution will be
-rounded to 3 decimal places.
-
-The minimum value size is 1 bytes; the maximum value size is 1M bytes. The
-precision of proportion is 0.001. The proportion of distribution will be
-rounded to 3 decimal places.
-Currently, memslap only supports set and get commands. And it
-supports 100% set and 100% get. For 100% get, it will preset some objects to
-the server.
-
-=head2 Multi-thread and concurrency
-
-The high performance of memslap benefits from the special
-schedule of thread and concurrency. It’s important to specify the proper
-number of them. The default number of threads is 1; the default number of
-concurrency is 16. The user can use “—threads” and “--concurrency” to
-specify these variables.
-
-If the system supports setting CPU affinity and the number of threads
-specified by the user is greater than 1, memslap will try to
-bind each thread to a different CPU core. So if you want to get the best
-performance memslap, it is better to specify the number of
-thread equal to the number of CPU cores. The number of threads specified by
-the user can also be less or greater than the number of CPU cores. Because
-of the limitation of implementation, the number of concurrencies could be
-the multiple of the number of threads.
-
-1. For 8 CPU cores system
-
-For example:
-
---threads=2 --concurrency=128
-
---threads=8 --concurrency=128
-
---threads=8 --concurrency=256
-
---threads=12 --concurrency=144
-
-2. For 16 CPU cores system
-
-For example:
-
---threads=8 --concurrency=128
-
---threads=16 --concurrency=256
-
---threads=16 --concurrency=512
-
---threads=24 --concurrency=288
-
-The memslap performs very well, when
-used to test the performance of memcached servers.
-Most of the time, the bottleneck is the network or
-the server. If for some reason the user wants to
-limit the performance of memslap, there
-are two ways to do this:
-
-Decrease the number of threads and concurrencies.
-Use the option “--tps” that memslap
-provides to limit the throughput. This option allows
-the user to get the expected throughput. For
-example, assume that the maximum throughput is 50
-kops/s for a specific configuration, you can specify
-the throughput equal to or less than the maximum
-throughput using “--tps” option.
-
-=head2 Window size
-
-Most of the time, the user does not need to specify the window size. The
-default window size is 10k. For Schooner Memcached, the user can specify
-different window sizes to get different cache miss rates based on the test
-case. Memslap supports cache miss rate between 0% and 100%.
-If you use this utility to test the performance of Schooner Memcached, you
-can specify a proper window size to get the expected cache miss rate. The
-formula for calculating window size is as follows:
-
-Assume that the key size is 128 bytes, and the value size is 2048 bytes, and
-concurrency=128.
-
-1. Small cache cache_size=1M, 100% cache miss (all data get from SSD).
-win_size=10k
-
-2. cache_size=4G
-
-(1). cache miss rate 0%
-
-win_size=8k
-
-(2). cache miss rate 5%
-
-win_size=11k
-
-3. cache_size=16G
-
-(1). cache miss rate 0%
-
-win_size=32k
-
-(2). cache miss
-
-rate 5%
-
-win_size=46k
-
-The formula for calculating window size for cache miss rate 0%:
-
-cache_size / concurrency / (key_size + value_size) * 0.5
-
-The formula for calculating window size for cache miss rate 5%:
-
-cache_size / concurrency / (key_size + value_size) * 0.7
-
-=head2 Verification
-
-Memslap supports both data verification and expire-time
-verification. The user can use "--verify=" or "-v" to specify the proportion
-of data verification. In theory, it supports 100% data verification. The
-user can use "--exp_verify=" or "-e" to specify the proportion of
-expire-time verification. In theory, it supports 100% expire-time
-verification. Specify the "--verbose" options to get more detailed error
-information.
-
-For example: --exp_verify=0.01 –verify=0.1 , it means that 1% of the objects
-set with expire-time, 10% of the objects gotten will be verified. If the
-objects are gotten, memslap will verify the expire-time and
-value.
-
-=head2 multi-servers and multi-clients
-
-Memslap supports multi-servers based on self-governed thread.
-There is a limitation that the number of servers cannot be greater than the
-number of threads. Memslap assigns one thread to handle one
-server at least. The user can use the "--servers=" or "-s" option to specify
-multi-servers.
-
-For example:
-
---servers=10.1.1.1:11211,10.1.1.2:11212,10.1.1.3:11213 --threads=6 --concurrency=36
-
-The above command means that there are 6 threads, with each thread having 6
-concurrencies and that threads 0 and 3 handle server 0 (10.1.1.1); threads 1
-and 4 handle server 1 (10.1.1.2); and thread 2 and 5 handle server 2
-(10.1.1.3).
-
-All the threads and concurrencies in memslap are self-governed.
-
-So is memslap. The user can start up several
-memslap instances. The user can run memslap on different client
-machines to communicate with the same memcached server at the same. It is
-recommended that the user start different memslap on different
-machines using the same configuration.
-
-=head2 Run with execute number mode or time mode
-
-The default memslap runs with time mode. The default run time
-is 10 minutes. If it times out, memslap will exit. Do not
-specify both execute number mode and time mode at the same time; just
-specify one instead.
-
-For example:
-
---time=30s (It means the test will run 30 seconds.)
-
---execute_number=100000 (It means that after running 100000 commands, the test will exit.)
-
-=head2 Dump statistic information periodically.
-
-The user can use "--stat_freq=" or "-S" to specify the frequency.
-
-For example:
-
---stat_freq=20s
-
-Memslap will dump the statistics of the commands (get and set) at the frequency of every 20
-seconds.
-
-For more information on the format of dumping statistic information, refer to “Format of Output” section.
-
-=head2 Multi-get
-
-The user can use "--division=" or "-d" to specify multi-get keys count.
-Memslap by default does single get with TCP. Memslap also supports data
-verification and expire-time verification for multi-get.
-
-Memslap supports multi-get with both TCP and UDP. Because of
-the different implementation of the ASCII protocol and binary protocol,
-there are some differences between the two. For the ASCII protocol,
-memslap sends one “multi-get” to the server once. For the
-binary protocol, memslap sends several single get commands
-together as “multi-get” to the server.
-
-=head2 UDP and TCP
-
-Memslap supports both UDP and TCP. For TCP,
-memslap does not reconnect the memcached server if socket connections are
-lost. If all the socket connections are lost or memcached server crashes,
-memslap will exit. If the user specifies the “--reconnect”
-option when socket connections are lost, it will reconnect them.
-
-User can use “--udp” to enable the UDP feature, but UDP comes with some
-limitations:
-
-UDP cannot set data more than 1400 bytes.
-
-UDP is not supported by the binary protocol because the binary protocol of
-memcached does not support that.
-
-UDP doesn’t support reconnection.
-
-=head2 Facebook test
-
-Set data with TCP and multi-get with UDP. Specify the following options:
-
-"--facebook --division=50"
-
-If you want to create thousands of TCP connections, specify the
-
-"--conn_sock=" option.
-
-For example: --facebook --division=50 --conn_sock=200
-
-The above command means that memslap will do facebook test,
-each concurrency has 200 socket TCP connections and one UDP socket.
-
-Memslap sets objects with the TCP socket, and multi-gets 50
-objects once with the UDP socket.
-
-If you specify "--division=50", the key size must be less that 25 bytes
-because the UDP packet size is 1400 bytes.
-
-=head2 Replication test
-
-For replication test, the user must specify at least two memcached servers.
-The user can use “—rep_write=” option to enable feature.
-
-For example:
-
---servers=10.1.1.1:11211,10.1.1.2:11212 –rep_write=2
-
-The above command means that there are 2 replication memcached servers,
-memslap will set objects to both server 0 and server 1, get
-objects which are set to server 0 before from server 1, and also get objects
-which are set to server 1 before from server 0. If server 0 crashes,
-memslap will only get objects from server 1. If server 0 comes
-back to life again, memslap will reconnect server 0. If both
-server 0 and server 1 crash, memslap will exit.
-
-=head2 Supports thousands of TCP connections
-
-Start memslap with "--conn_sock=" or "-n" to enable this
-feature. Make sure that your system can support opening thousands of files
-and creating thousands of sockets. However, this feature does not support
-reconnection if sockets disconnect.
-
-For example:
-
---threads=8 --concurrency=128 --conn_sock=128
-
-The above command means that memslap starts up 8 threads, each
-thread has 16 concurrencies, each concurrency has 128 TCP socket
-connections, and the total number of TCP socket connections is 128 * 128 =
-16384.
-
-=head2 Supports binary protocol
-
-Start memslap with "--binary" or "-B" options to enable this
-feature. It supports all the above features except UDP, because the latest
-memcached 1.3.3 does not implement binary UDP protocol.
-
-For example:
-
---binary
-
-Since memcached 1.3.3 doesn't implement binary UDP protocol,
-memslap does not support UDP. In addition, memcached 1.3.3 does not support
-multi-get. If you specify "--division=50" option, it just sends 50 get
-commands together as “mulit-get” to the server.
-
-=head1 Configuration file
-
-This section describes the format of the configuration file. By default
-when no configuration file is specified memslap reads the default
-one located at ~/.memslap.cnf.
-
-Below is a sample configuration file:
-
- ***************************************************************************
- #comments should start with '#'
- #key
- #start_len end_len proportion
- #
- #key length range from start_len to end_len
- #start_len must be equal to or greater than 16
- #end_len must be equal to or less than 250
- #start_len must be equal to or greater than end_len
- #memslap will generate keys according to the key range
- #proportion: indicates keys generated from one range accounts for the total
- generated keys
- #
- #example1: key range 16~100 accounts for 80%
- # key range 101~200 accounts for 10%
- # key range 201~250 accounts for 10%
- # total should be 1 (0.8+0.1+0.1 = 1)
- #
- # 16 100 0.8
- # 101 200 0.1
- # 201 249 0.1
- #
- #example2: all keys length are 128 bytes
- #
- # 128 128 1
- key
- 128 128 1
- #value
- #start_len end_len proportion
- #
- #value length range from start_len to end_len
- #start_len must be equal to or greater than 1
- #end_len must be equal to or less than 1M
- #start_len must be equal to or greater than end_len
- #memslap will generate values according to the value range
- #proportion: indicates values generated from one range accounts for the
- total generated values
- #
- #example1: value range 1~1000 accounts for 80%
- # value range 1001~10000 accounts for 10%
- # value range 10001~100000 accounts for 10%
- # total should be 1 (0.8+0.1+0.1 = 1)
- #
- # 1 1000 0.8
- # 1001 10000 0.1
- # 10001 100000 0.1
- #
- #example2: all value length are 128 bytes
- #
- # 128 128 1
- value
- 2048 2048 1
- #cmd
- #cmd_type cmd_proportion
- #
- #currently memslap only supports get and set command.
- #
- #cmd_type
- #set 0
- #get 1
- #
- #example: set command accounts for 50%
- # get command accounts for 50%
- # total should be 1 (0.5+0.5 = 1)
- #
- # cmd
- # 0 0.5
- # 1 0.5
- cmd
- 0 0.1
- 1.0 0.9
-
-
-
-=head1 Format of output
-
-At the beginning, memslap displays some configuration information as follows:
-
-=over 4
-
-=item servers : 127.0.0.1:11211
-
-=item threads count: 1
-
-=item concurrency: 16
-
-=item run time: 20s
-
-=item windows size: 10k
-
-=item set proportion: set_prop=0.10
-
-=item get proportion: get_prop=0.90
-
-=back
-
-=head2 Where
-
-=over 4
-
-=item servers : "servers"
-
-The servers used by memslap.
-
-=item threads count
-
-The number of threads memslap runs with.
-
-=item concurrency
-
-The number of concurrencies memslap runs with.
-
-=item run time
-
-How long to run memslap.
-
-=item windows size
-
-The task window size of each concurrency.
-
-=item set proportion
-
-The proportion of set command.
-
-=item get proportion
-
-The proportion of get command.
-
-=back
-
-The output of dynamic statistics is something like this:
-
- ---------------------------------------------------------------------------------------------------------------------------------
- Get Statistics
- Type Time(s) Ops TPS(ops/s) Net(M/s) Get_miss Min(us) Max(us)
- Avg(us) Std_dev Geo_dist
- Period 5 345826 69165 65.3 0 27 2198 203
- 95.43 177.29
- Global 20 1257935 62896 71.8 0 26 3791 224
- 117.79 192.60
-
-
- Set Statistics
- Type Time(s) Ops TPS(ops/s) Net(M/s) Get_miss Min(us) Max(us)
- Avg(us) Std_dev Geo_dist
- Period 5 38425 7685 7.3 0 42 628 240
- 88.05 220.21
- Global 20 139780 6989 8.0 0 37 3790 253
- 117.93 224.83
-
-
- Total Statistics
- Type Time(s) Ops TPS(ops/s) Net(M/s) Get_miss Min(us) Max(us)
- Avg(us) Std_dev Geo_dist
- Period 5 384252 76850 72.5 0 27 2198 207
- 94.72 181.18
- Global 20 1397720 69886 79.7 0 26 3791 227
- 117.93 195.60
- ---------------------------------------------------------------------------------------------------------------------------------
-
-=head2 Where
-
-=over 4
-
-=item Get Statistics
-
-Statistics information of get command
-
-=item Set Statistics
-
-Statistics information of set command
-
-=item Total Statistics
-
-Statistics information of both get and set command
-
-=item Period
-
-Result within a period
-
-=item Global
-
-Accumulated results
-
-=item Ops
-
-Total operations
-
-=item TPS
-
-Throughput, operations/second
-
-=item Net
-
-The rate of network
-
-=item Get_miss
-
-How many objects can’t be gotten
-
-=item Min
-
-The minimum response time
-
-=item Max
-
-The maximum response time
-
-=item Avg:
-
-The average response time
-
-=item Std_dev
-
-Standard deviation of response time
-
-=item Geo_dist
-
-Geometric distribution based on natural exponential function
-
-=back
-
-At the end, memslap will output something like this:
-
- ---------------------------------------------------------------------------------------------------------------------------------
- Get Statistics (1257956 events)
- Min: 26
- Max: 3791
- Avg: 224
- Geo: 192.60
- Std: 116.23
- Log2 Dist:
- 4: 0 10 84490 215345
- 8: 484890 459823 12543 824
- 12: 31
-
- Set Statistics (139782 events)
- Min: 37
- Max: 3790
- Avg: 253
- Geo: 224.84
- Std: 116.83
- Log2 Dist:
- 4: 0 0 4200 16988
- 8: 50784 65574 2064 167
- 12: 5
-
- Total Statistics (1397738 events)
- Min: 26
- Max: 3791
- Avg: 227
- Geo: 195.60
- Std: 116.60
- Log2 Dist:
- 4: 0 10 88690 232333
- 8: 535674 525397 14607 991
- 12: 36
-
- cmd_get: 1257969
- cmd_set: 139785
- get_misses: 0
- verify_misses: 0
- verify_failed: 0
- expired_get: 0
- unexpired_unget: 0
- written_bytes: 242516030
- read_bytes: 1003702556
- object_bytes: 152086080
- packet_disorder: 0
- packet_drop: 0
- udp_timeout: 0
-
- Run time: 20.0s Ops: 1397754 TPS: 69817 Net_rate: 59.4M/s
- ---------------------------------------------------------------------------------------------------------------------------------
-
-=head2 Where
-
-=over 4
-
-=item Get Statistics
-
-Get statistics of response time
-
-=item Set Statistics
-
-Set statistics of response time
-
-=item Total Statistics
-
-Both get and set statistics of response time
-
-=item Min
-
-The accumulated and minimum response time
-
-=item Max
-
-The accumulated and maximum response time
-
-=item Avg
-
-The accumulated and average response time
-
-=item Std
-
-Standard deviation of response time
-
-=item Log2 Dist
-
-Geometric distribution based on logarithm 2
-
-=item cmd_get
-
-Total get commands done
-
-=item cmd_set
-
-Total set commands done
-
-=item get_misses
-
-How many objects can’t be gotten from server
-
-=item verify_misses
-
-How many objects need to verify but can’t get them
-
-=item verify_failed
-
-How many objects with insistent value
-
-=item expired_get
-
-How many objects are expired but we get them
-
-=item unexpired_unget
-
-How many objects are unexpired but we can’t get them
-
-=item written_bytes
-
-Total written bytes
-
-=item read_bytes
-
-Total read bytes
-
-=item object_bytes
-
-Total object bytes
-
-=item packet_disorder
-
-How many UDP packages are disorder
-
-=item packet_drop
-
-How many UDP packages are lost
-
-=item udp_timeout
-
-How many times UDP time out happen
-
-=item Run time
-
-Total run time
-
-=item Ops
-
-Total operations
-
-=item TPS
-
-Throughput, operations/second
-
-=item Net_rate
-
-The average rate of network
-
-=back
-
-=head1 OPTIONS
-
--s, --servers=
- List one or more servers to connect. Servers count must be less than
- threads count. e.g.: --servers=localhost:1234,localhost:11211
-
--T, --threads=
- Number of threads to startup, better equal to CPU numbers. Default 8.
-
--c, --concurrency=
- Number of concurrency to simulate with load. Default 128.
-
--n, --conn_sock=
- Number of TCP socks per concurrency. Default 1.
-
--x, --execute_number=
- Number of operations(get and set) to execute for the
- given test. Default 1000000.
-
--t, --time=
- How long the test to run, suffix: s-seconds, m-minutes, h-hours,
- d-days e.g.: --time=2h.
-
--F, --cfg_cmd=
- Load the configure file to get command,key and value distribution list.
-
--w, --win_size=
- Task window size of each concurrency, suffix: K, M e.g.: --win_size=10k.
- Default 10k.
-
--X, --fixed_size=
- Fixed length of value.
-
--v, --verify=
- The proportion of date verification, e.g.: --verify=0.01
-
--d, --division=
- Number of keys to multi-get once. Default 1, means single get.
-
--S, --stat_freq=
- Frequency of dumping statistic information. suffix: s-seconds,
- m-minutes, e.g.: --resp_freq=10s.
-
--e, --exp_verify=
- The proportion of objects with expire time, e.g.: --exp_verify=0.01.
- Default no object with expire time
-
--o, --overwrite=
- The proportion of objects need overwrite, e.g.: --overwrite=0.01.
- Default never overwrite object.
-
--R, --reconnect
- Reconnect support, when connection is closed it will be reconnected.
-
--U, --udp
- UDP support, default memslap uses TCP, TCP port and UDP port of
- server must be same.
-
--a, --facebook
- Whether it enables facebook test feature, set with TCP and multi-get with UDP.
-
--B, --binary
- Whether it enables binary protocol. Default with ASCII protocol.
-
--P, --tps=
- Expected throughput, suffix: K, e.g.: --tps=10k.
-
--p, --rep_write=
- The first nth servers can write data, e.g.: --rep_write=2.
-
--b, --verbose
- Whether it outputs detailed information when verification fails.
-
--h, --help
- Display this message and then exit.
-
--V, --version
- Display the version of the application and then exit.
-
-=head1 EXAMPLES
-
-memslap -s 127.0.0.1:11211 -S 5s
-
-memslap -s 127.0.0.1:11211 -t 2m -v 0.2 -e 0.05 -b
-
-memslap -s 127.0.0.1:11211 -F config -t 2m -w 40k -S 20s -o 0.2
-
-memslap -s 127.0.0.1:11211 -F config -t 2m -T 4 -c 128 -d 20 -P 40k
-
-memslap -s 127.0.0.1:11211 -F config -t 2m -d 50 -a -n 40
-
-memslap -s 127.0.0.1:11211,127.0.0.1:11212 -F config -t 2m
-
-memslap -s 127.0.0.1:11211,127.0.0.1:11212 -F config -t 2m -p 2
+For a full list of operations run the tool with the B<--help> option.
=head1 HOME
To find out more information please check:
-L<http://launchpad.org/libmemcached>
+L<http://libmemcached.org/>
-=head1 AUTHORS
+=head1 AUTHOR
-Mingqiang Zhuang E<lt>mingqiangzhuang@hengtiansoft.comE<gt> (Schooner Technolgy)
Brian Aker, E<lt>brian@tangent.orgE<gt>
=head1 SEE ALSO
projects / awesomized / libmemcached / commitdiff
