From 7bfd925c54662cf1f67b651559dd91f77bfe8817 Mon Sep 17 00:00:00 2001 From: Brian Aker Date: Mon, 1 Oct 2007 06:15:52 -0700 Subject: [PATCH] Updates to tools for help and version. memslap can now do useful things :) An example: ./memslap --server=localhost --initial-load=50 --execute-number=100 --concurrency=30 Threads connecting to servers 30 Took 1.340 seconds to load data Took 1.340 seconds to read data --- configure.in | 2 +- src/Makefile.am | 7 +- src/client_options.h | 5 +- src/generator.h | 5 ++ src/memcat.c | 14 ++-- src/memcp.c | 16 +++-- src/memflush.c | 14 ++-- src/memrm.c | 14 ++-- src/memslap.c | 167 ++++++++++++++++++++++++++++--------------- src/memstat.c | 14 ++-- src/utilities.c | 26 ++++++- src/utilities.h | 12 ++++ 12 files changed, 215 insertions(+), 81 deletions(-) diff --git a/configure.in b/configure.in index 84b756ed..991f1ada 100644 --- a/configure.in +++ b/configure.in @@ -1,6 +1,6 @@ AC_INIT(src/memcat.c) AC_CONFIG_AUX_DIR(config) -AM_CONFIG_HEADER(libmemcached_config.h) +AM_CONFIG_HEADER(include/libmemcached_config.h) AM_INIT_AUTOMAKE("libmemcached", 0.2 ) AC_PROG_CC diff --git a/src/Makefile.am b/src/Makefile.am index 67a8612f..435376bd 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -3,7 +3,10 @@ LDADDS = ../lib/libmemcached.la bin_PROGRAMS = memcat memcp memstat memrm memflush memslap -noinst_HEADERS = client_options.h utilities.h +noinst_HEADERS = client_options.h \ + utilities.h \ + generator.h \ + execute.h memcat_SOURCES = memcat.c utilities.c memcat_LDADD = $(LDADDS) @@ -20,5 +23,5 @@ memrm_LDADD = $(LDADDS) memflush_SOURCES = memflush.c utilities.c memflush_LDADD = $(LDADDS) -memslap_SOURCES = memslap.c utilities.c generator.c +memslap_SOURCES = memslap.c utilities.c generator.c execute.c memslap_LDADD = $(LDADDS) -lpthread diff --git a/src/client_options.h b/src/client_options.h index cc427213..57c87db6 100644 --- a/src/client_options.h +++ b/src/client_options.h @@ -9,5 +9,8 @@ typedef enum { OPT_SET, OPT_REPLACE, OPT_ADD, - OPT_SLAP_DEFAULT_PAIRS, + OPT_SLAP_EXECUTE_NUMBER, + OPT_SLAP_INITIAL_LOAD, + OPT_SLAP_TEST, + OPT_SLAP_CONCURRENCY, } memcached_options; diff --git a/src/generator.h b/src/generator.h index 6528cce1..8582ffb3 100644 --- a/src/generator.h +++ b/src/generator.h @@ -2,6 +2,9 @@ Code to generate data to be pushed into memcached */ +#ifndef __GENERATOR_H__ +#define __GENERATOR_H__ + typedef struct pairs_st pairs_st; struct pairs_st { @@ -13,3 +16,5 @@ struct pairs_st { pairs_st *pairs_generate(unsigned long long number_of); void pairs_free(pairs_st *pairs); + +#endif diff --git a/src/memcat.c b/src/memcat.c index 81c0a85d..7f3c0419 100644 --- a/src/memcat.c +++ b/src/memcat.c @@ -6,6 +6,9 @@ #include "client_options.h" #include "utilities.h" +#define PROGRAM_NAME "memcat" +#define PROGRAM_DESCRIPTION "Cat a set of key values to stdout." + /* Prototypes */ void options_parse(int argc, char *argv[]); @@ -76,6 +79,11 @@ void options_parse(int argc, char *argv[]) int option_index= 0; int option_rv; + memcached_programs_help_st help_options[]= + { + {0}, + }; + static struct option long_options[]= { {"version", no_argument, NULL, OPT_VERSION}, @@ -102,12 +110,10 @@ void options_parse(int argc, char *argv[]) opt_verbose = OPT_DEBUG; break; case OPT_VERSION: /* --version or -V */ - printf("memcache tools, memcat, v1.0\n"); - exit(0); + version_command(PROGRAM_NAME); break; case OPT_HELP: /* --help or -h */ - printf("useful help messages go here\n"); - exit(0); + help_command(PROGRAM_NAME, PROGRAM_DESCRIPTION, long_options, help_options); break; case OPT_SERVERS: /* --servers or -s */ opt_servers= strdup(optarg); diff --git a/src/memcp.c b/src/memcp.c index 9123028b..3b846e78 100644 --- a/src/memcp.c +++ b/src/memcp.c @@ -12,6 +12,9 @@ #include "client_options.h" #include "utilities.h" +#define PROGRAM_NAME "memcp" +#define PROGRAM_DESCRIPTION "Copy a set of files to a memcached cluster." + /* Prototypes */ void options_parse(int argc, char *argv[]); @@ -122,6 +125,11 @@ void options_parse(int argc, char *argv[]) int option_index= 0; int option_rv; + memcached_programs_help_st help_options[]= + { + {0}, + }; + static struct option long_options[]= { {"version", no_argument, NULL, OPT_VERSION}, @@ -154,11 +162,11 @@ void options_parse(int argc, char *argv[]) opt_verbose = OPT_DEBUG; break; case OPT_VERSION: /* --version or -V */ - printf("memcache tools, memcp, v1.0\n"); - exit(0); + version_command(PROGRAM_NAME); + break; case OPT_HELP: /* --help or -h */ - printf("useful help messages go here\n"); - exit(0); + help_command(PROGRAM_NAME, PROGRAM_DESCRIPTION, long_options, help_options); + break; case OPT_SERVERS: /* --servers or -s */ opt_servers= strdup(optarg); break; diff --git a/src/memflush.c b/src/memflush.c index a1a9ccad..73f7afb6 100644 --- a/src/memflush.c +++ b/src/memflush.c @@ -9,6 +9,9 @@ static int opt_verbose= 0; static time_t opt_expire= 0; static char *opt_servers= NULL; +#define PROGRAM_NAME "memflush" +#define PROGRAM_DESCRIPTION "Erase all data in a server of memcached servers." + /* Prototypes */ void options_parse(int argc, char *argv[]); @@ -46,6 +49,11 @@ int main(int argc, char *argv[]) void options_parse(int argc, char *argv[]) { + memcached_programs_help_st help_options[]= + { + {0}, + }; + static struct option long_options[]= { {"version", no_argument, NULL, OPT_VERSION}, @@ -74,12 +82,10 @@ void options_parse(int argc, char *argv[]) opt_verbose = OPT_DEBUG; break; case OPT_VERSION: /* --version or -V */ - printf("memcache tools, memflush, v1.0\n"); - exit(0); + version_command(PROGRAM_NAME); break; case OPT_HELP: /* --help or -h */ - printf("useful help messages go here\n"); - exit(0); + help_command(PROGRAM_NAME, PROGRAM_DESCRIPTION, long_options, help_options); break; case OPT_SERVERS: /* --servers or -s */ opt_servers= strdup(optarg); diff --git a/src/memrm.c b/src/memrm.c index e07654f3..11dbb82c 100644 --- a/src/memrm.c +++ b/src/memrm.c @@ -9,6 +9,9 @@ static int opt_verbose= 0; static time_t opt_expire= 0; static char *opt_servers= NULL; +#define PROGRAM_NAME "memrm" +#define PROGRAM_DESCRIPTION "Erase a key or set of keys from a memcached cluster." + /* Prototypes */ void options_parse(int argc, char *argv[]); @@ -54,6 +57,11 @@ int main(int argc, char *argv[]) void options_parse(int argc, char *argv[]) { + memcached_programs_help_st help_options[]= + { + {0}, + }; + static struct option long_options[]= { {"version", no_argument, NULL, OPT_VERSION}, @@ -82,12 +90,10 @@ void options_parse(int argc, char *argv[]) opt_verbose = OPT_DEBUG; break; case OPT_VERSION: /* --version or -V */ - printf("memcache tools, memrm, v1.0\n"); - exit(0); + version_command(PROGRAM_NAME); break; case OPT_HELP: /* --help or -h */ - printf("useful help messages go here\n"); - exit(0); + help_command(PROGRAM_NAME, PROGRAM_DESCRIPTION, long_options, help_options); break; case OPT_SERVERS: /* --servers or -s */ opt_servers= strdup(optarg); diff --git a/src/memslap.c b/src/memslap.c index fad016ec..2e83acdb 100644 --- a/src/memslap.c +++ b/src/memslap.c @@ -14,6 +14,14 @@ #include "client_options.h" #include "utilities.h" #include "generator.h" +#include "execute.h" + +#define DEFAULT_INITIAL_LOAD 10000 +#define DEFAULT_EXECUTE_NUMBER 10000 +#define DEFAULT_CONCURRENCY 1 + +#define PROGRAM_NAME "memslap" +#define PROGRAM_DESCRIPTION "Generates a load against a memcached custer of servers." /* Global Thread counter */ unsigned int thread_counter; @@ -29,13 +37,17 @@ void *run_task(void *p); typedef struct conclusions_st conclusions_st; typedef struct thread_context_st thread_context_st; typedef enum { - AC_SET, - AC_GET, -} run_action; + SET_TEST, + GET_TEST, +} test_type; struct thread_context_st { - pairs_st *pairs; - run_action action; + unsigned int key_count; + pairs_st *initial_pairs; + unsigned int initial_number; + pairs_st *execute_pairs; + unsigned int execute_number; + test_type test; memcached_server_st *servers; }; @@ -50,12 +62,16 @@ struct conclusions_st { void options_parse(int argc, char *argv[]); void conclusions_print(conclusions_st *conclusion); void scheduler(memcached_server_st *servers, conclusions_st *conclusion); +pairs_st *load_initial_data(memcached_server_st *servers, unsigned int number_of, + unsigned int *actual_loaded); static int opt_verbose= 0; -static unsigned int opt_default_pairs= 100; -static unsigned int opt_concurrency= 10; +static unsigned int opt_execute_number= 0; +static unsigned int opt_initial_load= 0; +static unsigned int opt_concurrency= 0; static int opt_displayflag= 0; static char *opt_servers= NULL; +test_type opt_test= SET_TEST; int main(int argc, char *argv[]) { @@ -94,16 +110,19 @@ int main(int argc, char *argv[]) void scheduler(memcached_server_st *servers, conclusions_st *conclusion) { unsigned int x; + unsigned int actual_loaded; + struct timeval start_time, end_time; pthread_t mainthread; /* Thread descriptor */ pthread_attr_t attr; /* Thread attributes */ - pairs_st *pairs; + pairs_st *pairs= NULL; pthread_attr_init(&attr); pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); - pairs= pairs_generate(opt_default_pairs); + if (opt_initial_load) + pairs= load_initial_data(servers, opt_initial_load, &actual_loaded); pthread_mutex_lock(&counter_mutex); thread_counter= 0; @@ -118,8 +137,16 @@ void scheduler(memcached_server_st *servers, conclusions_st *conclusion) context= (thread_context_st *)malloc(sizeof(thread_context_st)); context->servers= servers; - context->pairs= pairs; - context->action= AC_SET; + context->test= opt_test; + + context->initial_pairs= pairs; + context->initial_number= actual_loaded; + + if (opt_test == SET_TEST) + { + context->execute_pairs= pairs_generate(opt_execute_number); + context->execute_number= opt_execute_number; + } /* now you create the thread */ if (pthread_create(&mainthread, &attr, run_task, @@ -164,15 +191,23 @@ void scheduler(memcached_server_st *servers, conclusions_st *conclusion) void options_parse(int argc, char *argv[]) { + memcached_programs_help_st help_options[]= + { + {0}, + }; + static struct option long_options[]= { - {"version", no_argument, NULL, OPT_VERSION}, - {"help", no_argument, NULL, OPT_HELP}, - {"verbose", no_argument, &opt_verbose, OPT_VERBOSE}, + {"concurrency", required_argument, NULL, OPT_SLAP_CONCURRENCY}, {"debug", no_argument, &opt_verbose, OPT_DEBUG}, - {"servers", required_argument, NULL, OPT_SERVERS}, + {"execute-number", required_argument, NULL, OPT_SLAP_EXECUTE_NUMBER}, {"flag", no_argument, &opt_displayflag, OPT_FLAG}, - {"default-pairs", required_argument, NULL, OPT_SLAP_DEFAULT_PAIRS}, + {"help", no_argument, NULL, OPT_HELP}, + {"initial-load", required_argument, NULL, OPT_SLAP_INITIAL_LOAD}, /* Number to load initially */ + {"servers", required_argument, NULL, OPT_SERVERS}, + {"test", required_argument, NULL, OPT_SLAP_TEST}, + {"verbose", no_argument, &opt_verbose, OPT_VERBOSE}, + {"version", no_argument, NULL, OPT_VERSION}, {0, 0, 0, 0}, }; @@ -194,18 +229,32 @@ void options_parse(int argc, char *argv[]) opt_verbose = OPT_DEBUG; break; case OPT_VERSION: /* --version or -V */ - printf("memcache tools, memcat, v1.0\n"); - exit(0); + version_command(PROGRAM_NAME); break; case OPT_HELP: /* --help or -h */ - printf("useful help messages go here\n"); - exit(0); + help_command(PROGRAM_NAME, PROGRAM_DESCRIPTION, long_options, help_options); break; case OPT_SERVERS: /* --servers or -s */ opt_servers= strdup(optarg); break; - case OPT_SLAP_DEFAULT_PAIRS: - opt_default_pairs= strtol(optarg, (char **)NULL, 10); + case OPT_SLAP_TEST: + if (!strcmp(optarg, "get")) + opt_test= GET_TEST ; + else if (!strcmp(optarg, "set")) + opt_test= SET_TEST; + else + { + fprintf(stderr, "Your test, %s, is not a known test\n", optarg); + exit(1); + } + break; + case OPT_SLAP_CONCURRENCY: + opt_concurrency= strtol(optarg, (char **)NULL, 10); + case OPT_SLAP_EXECUTE_NUMBER: + opt_execute_number= strtol(optarg, (char **)NULL, 10); + break; + case OPT_SLAP_INITIAL_LOAD: + opt_initial_load= strtol(optarg, (char **)NULL, 10); break; case '?': /* getopt_long already printed an error message. */ @@ -214,25 +263,36 @@ void options_parse(int argc, char *argv[]) abort(); } } + + if (opt_test == GET_TEST && opt_initial_load == 0) + opt_initial_load= DEFAULT_INITIAL_LOAD; + + if (opt_execute_number == 0) + opt_execute_number= DEFAULT_EXECUTE_NUMBER; + + if (opt_concurrency == 0) + opt_concurrency= DEFAULT_CONCURRENCY; } void conclusions_print(conclusions_st *conclusion) { + printf("\tThreads connecting to servers %u\n", opt_concurrency); +#ifdef NOT_FINISHED printf("\tLoaded %u rows\n", conclusion->rows_loaded); printf("\tRead %u rows\n", conclusion->rows_read); - printf("\tTook %ld.%03ld seconds to load data\n", conclusion->load_time / 1000, - conclusion->load_time % 1000); - printf("\tTook %ld.%03ld seconds to read data\n", conclusion->read_time / 1000, - conclusion->read_time % 1000); +#endif + if (opt_test == SET_TEST) + printf("\tTook %ld.%03ld seconds to load data\n", conclusion->load_time / 1000, + conclusion->load_time % 1000); + else + printf("\tTook %ld.%03ld seconds to read data\n", conclusion->read_time / 1000, + conclusion->read_time % 1000); } void *run_task(void *p) { - unsigned int x; thread_context_st *context= (thread_context_st *)p; - memcached_return rc; memcached_st *memc; - pairs_st *pairs= context->pairs; memc= memcached_init(NULL); @@ -246,35 +306,13 @@ void *run_task(void *p) pthread_mutex_unlock(&sleeper_mutex); /* Do Stuff */ - switch (context->action) + switch (context->test) { - case AC_SET: - for (x= 0; x < opt_default_pairs; x++) - { - rc= memcached_set(memc, pairs[x].key, pairs[x].key_length, - pairs[x].value, pairs[x].value_length, - 0, 0); - if (rc != MEMCACHED_SUCCESS) - fprintf(stderr, "Failured on insert of %.*s\n", - (unsigned int)pairs[x].key_length, pairs[x].key); - } + case SET_TEST: + execute_set(memc, context->execute_pairs, context->execute_number); break; - case AC_GET: - for (x= 0; x < opt_default_pairs; x++) - { - char *value; - size_t value_length; - uint16_t flags; - - value= memcached_get(memc, pairs[x].key, pairs[x].key_length, - &value_length, - &flags, &rc); - - if (rc != MEMCACHED_SUCCESS) - fprintf(stderr, "Failured on read of %.*s\n", - (unsigned int)pairs[x].key_length, pairs[x].key); - free(value); - } + case GET_TEST: + execute_get(memc, context->initial_pairs, context->initial_number); break; } @@ -288,3 +326,20 @@ void *run_task(void *p) return NULL; } + +pairs_st *load_initial_data(memcached_server_st *servers, unsigned int number_of, + unsigned int *actual_loaded) +{ + memcached_st *memc; + pairs_st *pairs; + + memc= memcached_init(NULL); + memcached_server_push(memc, servers); + + pairs= pairs_generate(number_of); + *actual_loaded= execute_set(memc, pairs, number_of); + + memcached_deinit(memc); + + return pairs; +} diff --git a/src/memstat.c b/src/memstat.c index a82f3f12..8ba59d11 100644 --- a/src/memstat.c +++ b/src/memstat.c @@ -11,6 +11,9 @@ #include "client_options.h" #include "utilities.h" +#define PROGRAM_NAME "memstat" +#define PROGRAM_DESCRIPTION "Output the state of a memcached cluster." + /* Prototypes */ void options_parse(int argc, char *argv[]); @@ -84,6 +87,11 @@ int main(int argc, char *argv[]) void options_parse(int argc, char *argv[]) { + memcached_programs_help_st help_options[]= + { + {0}, + }; + static struct option long_options[]= { {"version", no_argument, NULL, OPT_VERSION}, @@ -113,12 +121,10 @@ void options_parse(int argc, char *argv[]) opt_verbose = OPT_DEBUG; break; case OPT_VERSION: /* --version or -V */ - printf("memcache tools, memcat, v1.0\n"); - exit(0); + version_command(PROGRAM_NAME); break; case OPT_HELP: /* --help or -h */ - printf("useful help messages go here\n"); - exit(0); + help_command(PROGRAM_NAME, PROGRAM_DESCRIPTION, long_options, help_options); break; case OPT_SERVERS: /* --servers or -s */ opt_servers= strdup(optarg); diff --git a/src/utilities.c b/src/utilities.c index a2dc944f..ed0dbb30 100644 --- a/src/utilities.c +++ b/src/utilities.c @@ -1,6 +1,6 @@ #include #include -#include +#include "utilities.h" memcached_server_st *parse_opt_servers(char *server_strings) { @@ -66,3 +66,27 @@ long int timedif(struct timeval a, struct timeval b) s *= 1000; return s + us; } + +void version_command(char *command_name) +{ + printf("%s v%u.%u\n", command_name, 1, 0); + exit(0); +} + +void help_command(char *command_name, char *description, + const struct option *long_options, + memcached_programs_help_st *options) +{ + unsigned int x; + + printf("%s v%u.%u\n\n", command_name, 1, 0); + printf("\t%s\n\n", description); + printf("Current options. A '=' means the option takes a value.\n\n"); + + for (x= 0; long_options[x].name; x++) + printf("\t --%s%c\n", long_options[x].name, + long_options[x].has_arg ? '=' : ' '); + + printf("\n"); + exit(0); +} diff --git a/src/utilities.h b/src/utilities.h index 12f50349..b9ac7f1c 100644 --- a/src/utilities.h +++ b/src/utilities.h @@ -1,6 +1,18 @@ #include +#include + +typedef struct memcached_programs_help_st memcached_programs_help_st; + +struct memcached_programs_help_st +{ + char *not_used_yet; +}; memcached_server_st *parse_opt_servers(char *server_strings); char *strdup_cleanup(const char *str); void cleanup(void); long int timedif(struct timeval a, struct timeval b); +void version_command(char *command_name); +void help_command(char *command_name, char *description, + const struct option *long_options, + memcached_programs_help_st *options); -- 2.30.2