From a7db49095a0ed78a1418003a00eef6d483c0d854 Mon Sep 17 00:00:00 2001 From: Brian Aker Date: Thu, 4 Oct 2007 08:15:24 -0700 Subject: [PATCH] See changes in changelog, but... 1) memslap --flush option 2) support for no_tcpdelay (though I don't find this to be faster) 3) More tests --- ChangeLog | 9 +++++ README | 1 + configure.in | 2 +- include/memcached.h | 4 ++ lib/memcached_behavior.c | 13 +++++++ lib/memcached_connect.c | 23 +++++++++--- src/client_options.h | 2 + src/memslap.c | 28 ++++++++++++-- support/Makefile.am | 2 +- support/set_benchmark.sh | 5 +++ tests/output.res | 80 ++++++++++++++++++++++++++++++++++++++++ tests/test.c | 15 ++++++++ 12 files changed, 173 insertions(+), 11 deletions(-) create mode 100755 support/set_benchmark.sh diff --git a/ChangeLog b/ChangeLog index 3673fed6..2c8f12ac 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,12 @@ +0.5 + * Ruby maintainer mentioned TCP_NODELAY patch he had added. Added this to C + library as well. (Eric Hodel drbrain@segment7.net) + * Added support script for set_benchmark + * Updated memslap to allow testing of TCP_NODELAY + * Updated memslap to support --flush (aka dump memcache servers before + testing) + * Fixed bug in multiple hosts not being activated + 0.4 Wed Oct 3 10:28:50 PDT 2007 * Added buffered IO to write calls for keys * Added buffered IO for reads diff --git a/README b/README index d2bb56d8..7170c99f 100644 --- a/README +++ b/README @@ -8,6 +8,7 @@ memcat - Copy the value of a key to standard output memflush - Flush the contents of your servers. memrm - Remove a key(s) from the serrver. memstat - Dump the stats of your servers to standard output +memslap - Load generation utility (benchmark!) This code is still in Alpha. More tests have currently been done on the library code then on the utilities. diff --git a/configure.in b/configure.in index 7f00a0fb..fbec25c0 100644 --- a/configure.in +++ b/configure.in @@ -1,7 +1,7 @@ AC_INIT(src/memcat.c) AC_CONFIG_AUX_DIR(config) AM_CONFIG_HEADER(include/libmemcached_config.h) -AM_INIT_AUTOMAKE("libmemcached", 0.4) +AM_INIT_AUTOMAKE("libmemcached", 0.5) AC_PROG_CC AC_PROG_LIBTOOL diff --git a/include/memcached.h b/include/memcached.h index 84eccee2..594b4ade 100644 --- a/include/memcached.h +++ b/include/memcached.h @@ -72,6 +72,8 @@ typedef enum { typedef enum { MEMCACHED_BEHAVIOR_NO_BLOCK, MEMCACHED_BEHAVIOR_BLOCK, + MEMCACHED_BEHAVIOR_TCP_NODELAY, + MEMCACHED_BEHAVIOR_TCP_DELAY, } memcached_behavior; typedef enum { @@ -115,6 +117,8 @@ struct memcached_stat_st { }; #define MEM_NO_BLOCK (1 << 0) +#define MEM_TCP_NODELAY (1 << 1) +#define MEM_REUSE_MEMORY (1 << 2) struct memcached_string_st { char *string; diff --git a/lib/memcached_behavior.c b/lib/memcached_behavior.c index a88bcfcf..e084af35 100644 --- a/lib/memcached_behavior.c +++ b/lib/memcached_behavior.c @@ -1,4 +1,7 @@ #include +#include +#include +#include memcached_return memcached_behavior_set(memcached_st *ptr, memcached_behavior flag, @@ -16,6 +19,16 @@ memcached_return memcached_behavior_set(memcached_st *ptr, memcached_quit(ptr); ptr->flags+= MEM_NO_BLOCK; break; + case MEMCACHED_BEHAVIOR_TCP_NODELAY: + /* We quit all connections so we can reset the sockets */ + memcached_quit(ptr); + ptr->flags|= MEM_TCP_NODELAY; + break; + case MEMCACHED_BEHAVIOR_TCP_DELAY: + /* We quit all connections so we can reset the sockets */ + memcached_quit(ptr); + ptr->flags+= MEM_TCP_NODELAY; + break; } return MEMCACHED_SUCCESS; diff --git a/lib/memcached_connect.c b/lib/memcached_connect.c index 5b024749..78a7d703 100644 --- a/lib/memcached_connect.c +++ b/lib/memcached_connect.c @@ -1,6 +1,9 @@ #include "common.h" #include +#include +#include +#include memcached_return memcached_connect(memcached_st *ptr) { @@ -20,8 +23,6 @@ memcached_return memcached_connect(memcached_st *ptr) { if (ptr->hosts[x].fd == -1) { - int flags; - if ((h= gethostbyname(ptr->hosts[x].hostname)) == NULL) return MEMCACHED_HOST_LOCKUP_FAILURE; @@ -30,7 +31,7 @@ memcached_return memcached_connect(memcached_st *ptr) servAddr.sin_port = htons(ptr->hosts[x].port); /* Create the socket */ - if ((ptr->hosts[0].fd= socket(AF_INET, SOCK_STREAM, 0)) < 0) + if ((ptr->hosts[x].fd= socket(AF_INET, SOCK_STREAM, 0)) < 0) { ptr->my_errno= errno; return MEMCACHED_CONNECTION_SOCKET_CREATE_FAILURE; @@ -45,14 +46,24 @@ memcached_return memcached_connect(memcached_st *ptr) /* For the moment, not getting a nonblocking mode will note be fatal */ if (ptr->flags & MEM_NO_BLOCK) { - flags= fcntl(ptr->hosts[0].fd, F_GETFL, 0); + int flags; + + flags= fcntl(ptr->hosts[x].fd, F_GETFL, 0); if (flags != -1) - (void)fcntl(ptr->hosts[0].fd, F_SETFL, flags | O_NONBLOCK); + (void)fcntl(ptr->hosts[x].fd, F_SETFL, flags | O_NONBLOCK); + } + + if (ptr->flags & MEM_TCP_NODELAY) + { + int flag= 1; + + setsockopt(ptr->hosts[x].fd, IPPROTO_TCP, TCP_NODELAY, + &flag, (socklen_t)sizeof(int)); } /* connect to server */ test_connect: - if (connect(ptr->hosts[0].fd, (struct sockaddr *) &servAddr, sizeof(servAddr)) < 0) + if (connect(ptr->hosts[x].fd, (struct sockaddr *) &servAddr, sizeof(servAddr)) < 0) { switch (errno) { /* We are spinning waiting on connect */ diff --git a/src/client_options.h b/src/client_options.h index 8bce24ef..03eac3cc 100644 --- a/src/client_options.h +++ b/src/client_options.h @@ -14,4 +14,6 @@ typedef enum { OPT_SLAP_TEST, OPT_SLAP_CONCURRENCY, OPT_SLAP_NON_BLOCK, + OPT_SLAP_TCP_NODELAY, + OPT_FLUSH, } memcached_options; diff --git a/src/memslap.c b/src/memslap.c index b7925b73..ff01ddc1 100644 --- a/src/memslap.c +++ b/src/memslap.c @@ -62,11 +62,14 @@ struct conclusions_st { void options_parse(int argc, char *argv[]); void conclusions_print(conclusions_st *conclusion); void scheduler(memcached_server_st *servers, conclusions_st *conclusion); -pairs_st *load_createial_data(memcached_server_st *servers, unsigned int number_of, +pairs_st *load_create_data(memcached_server_st *servers, unsigned int number_of, unsigned int *actual_loaded); +void flush_all(memcached_server_st *servers); static int opt_verbose= 0; +static int opt_flush= 0; static int opt_non_blocking_io= 0; +static int opt_tcp_nodelay= 0; static unsigned int opt_execute_number= 0; static unsigned int opt_createial_load= 0; static unsigned int opt_concurrency= 0; @@ -122,8 +125,10 @@ void scheduler(memcached_server_st *servers, conclusions_st *conclusion) pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); + if (opt_flush) + flush_all(servers); if (opt_createial_load) - pairs= load_createial_data(servers, opt_createial_load, &actual_loaded); + pairs= load_create_data(servers, opt_createial_load, &actual_loaded); pthread_mutex_lock(&counter_mutex); thread_counter= 0; @@ -203,10 +208,12 @@ void options_parse(int argc, char *argv[]) {"debug", no_argument, &opt_verbose, OPT_DEBUG}, {"execute-number", required_argument, NULL, OPT_SLAP_EXECUTE_NUMBER}, {"flag", no_argument, &opt_displayflag, OPT_FLAG}, + {"flush", no_argument, &opt_flush, OPT_FLUSH}, {"help", no_argument, NULL, OPT_HELP}, {"initial-load", required_argument, NULL, OPT_SLAP_INITIAL_LOAD}, /* Number to load initially */ {"non-blocking", no_argument, &opt_non_blocking_io, OPT_SLAP_NON_BLOCK}, {"servers", required_argument, NULL, OPT_SERVERS}, + {"tcp-nodelay", no_argument, &opt_tcp_nodelay, OPT_SLAP_TCP_NODELAY}, {"test", required_argument, NULL, OPT_SLAP_TEST}, {"verbose", no_argument, &opt_verbose, OPT_VERBOSE}, {"version", no_argument, NULL, OPT_VERSION}, @@ -299,6 +306,8 @@ void *run_task(void *p) memc= memcached_create(NULL); if (opt_non_blocking_io) memcached_behavior_set(memc, MEMCACHED_BEHAVIOR_NO_BLOCK, NULL ); + if (opt_tcp_nodelay) + memcached_behavior_set(memc, MEMCACHED_BEHAVIOR_TCP_NODELAY, NULL ); memcached_server_push(memc, context->servers); @@ -333,7 +342,20 @@ void *run_task(void *p) return NULL; } -pairs_st *load_createial_data(memcached_server_st *servers, unsigned int number_of, +void flush_all(memcached_server_st *servers) +{ + memcached_st *memc; + + memc= memcached_create(NULL); + + memcached_server_push(memc, servers); + + memcached_flush(memc, 0); + + memcached_free(memc); +} + +pairs_st *load_create_data(memcached_server_st *servers, unsigned int number_of, unsigned int *actual_loaded) { memcached_st *memc; diff --git a/support/Makefile.am b/support/Makefile.am index cc4aec68..103bd257 100644 --- a/support/Makefile.am +++ b/support/Makefile.am @@ -1 +1 @@ -EXTRA_DIST = libmemcached.spec +EXTRA_DIST = libmemcached.spec set_benchmark.sh diff --git a/support/set_benchmark.sh b/support/set_benchmark.sh new file mode 100755 index 00000000..1c29a50c --- /dev/null +++ b/support/set_benchmark.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +src/memslap --concurrency=5 --execute-number=5000 --servers=localhost --test=set --flush +src/memslap --concurrency=5 --execute-number=5000 --non-blocking --servers=localhost --test=set --flush +src/memslap --concurrency=5 --execute-number=5000 --non-blocking --tcp-nodelay --servers=localhost --test=set --flush diff --git a/tests/output.res b/tests/output.res index e6a2f73b..94d86fd6 100644 --- a/tests/output.res +++ b/tests/output.res @@ -154,3 +154,83 @@ Found key bytes_read Found key bytes_written Found key limit_maxbytes Found key threads + +WATCHPOINT memcached_flush.c:35 (memcached_flush) +Error 0 -> SUCCESS +Error 1 -> FAILURE +Error 2 -> HOSTNAME LOOKUP FAILURE +Error 3 -> CONNECTION FAILURE +Error 4 -> CONNECTION BIND FAILURE +Error 5 -> WRITE FAILURE +Error 6 -> READ FAILURE +Error 7 -> UNKNOWN READ FAILURE +Error 8 -> PROTOCOL ERROR +Error 9 -> CLIENT ERROR +Error 10 -> SERVER ERROR +Error 11 -> CONNECTION SOCKET CREATE FAILURE +Error 12 -> CONNECTION DATA EXISTS +Error 13 -> CONNECTION DATA DOES NOT EXIST +Error 14 -> NOT STORED +Error 15 -> STORED +Error 16 -> NOT FOUND +Error 17 -> MEMORY ALLOCATION FAILURE +Error 18 -> PARTIAL READ +Error 19 -> SOME ERRORS WERE REPORTED +Error 20 -> NO SERVERS DEFINED +Error 21 -> SERVER END +Error 22 -> SERVER DELETE +Error 23 -> SERVER VALUE + +WATCHPOINT memcached_flush.c:35 (memcached_flush) +Found key pid +Found key uptime +Found key time +Found key version +Found key pointer_size +Found key rusage_user +Found key rusage_system +Found key rusage_user_seconds +Found key rusage_user_microseconds +Found key rusage_system_seconds +Found key rusage_system_microseconds +Found key curr_items +Found key total_items +Found key bytes +Found key curr_connections +Found key total_connections +Found key connection_structures +Found key cmd_get +Found key cmd_set +Found key get_hits +Found key get_misses +Found key evictions +Found key bytes_read +Found key bytes_written +Found key limit_maxbytes +Found key threads +Found key pid +Found key uptime +Found key time +Found key version +Found key pointer_size +Found key rusage_user +Found key rusage_system +Found key rusage_user_seconds +Found key rusage_user_microseconds +Found key rusage_system_seconds +Found key rusage_system_microseconds +Found key curr_items +Found key total_items +Found key bytes +Found key curr_connections +Found key total_connections +Found key connection_structures +Found key cmd_get +Found key cmd_set +Found key get_hits +Found key get_misses +Found key evictions +Found key bytes_read +Found key bytes_written +Found key limit_maxbytes +Found key threads diff --git a/tests/test.c b/tests/test.c index e12df9fc..f43793ee 100644 --- a/tests/test.c +++ b/tests/test.c @@ -626,6 +626,21 @@ int main(int argc, char *argv[]) memcached_free(memc); } + fprintf(stderr, "\nTCP Nodelay tests\n\n"); + for (x= 0; tests[x].function_name; x++) + { + memcached_st *memc; + memc= memcached_create(NULL); + assert(memc); + fprintf(stderr, "Testing %s", tests[x].function_name); + memcached_behavior_set(memc, MEMCACHED_BEHAVIOR_NO_BLOCK, NULL); + memcached_behavior_set(memc, MEMCACHED_BEHAVIOR_TCP_NODELAY, NULL); + tests[x].function(memc); + fprintf(stderr, "\t\t\t\t\t[ ok ]\n"); + assert(memc); + memcached_free(memc); + } + /* The multiple tests */ if (argc == 2) -- 2.30.2