From cc5be1b75fe90e8c73871f4bd135c9e7cf1d1984 Mon Sep 17 00:00:00 2001 From: Toru Maesaka Date: Fri, 23 Jan 2009 17:02:41 +0900 Subject: [PATCH] Added cluster analysis functionality to the library and memstat --- clients/client_options.h | 1 + clients/memstat.c | 64 ++++++++++++++++++++- clients/utilities.c | 1 + docs/Makefile.am | 9 ++- docs/memcached_analyze.pod | 52 +++++++++++++++++ libmemcached/Makefile.am | 3 +- libmemcached/memcached.h | 11 ++++ libmemcached/memcached_analyze.c | 99 ++++++++++++++++++++++++++++++++ libmemcached/memcached_server.h | 2 + libmemcached/memcached_types.h | 1 + tests/function.c | 21 +++++++ 11 files changed, 258 insertions(+), 6 deletions(-) create mode 100644 docs/memcached_analyze.pod create mode 100644 libmemcached/memcached_analyze.c diff --git a/clients/client_options.h b/clients/client_options.h index 33a0f81b..a3431336 100644 --- a/clients/client_options.h +++ b/clients/client_options.h @@ -9,6 +9,7 @@ typedef enum { OPT_HELP= 'h', OPT_VERBOSE= 'v', OPT_DEBUG= 'd', + OPT_ANALYZE= 'a', OPT_FLAG= 257, OPT_EXPIRE, OPT_SET, diff --git a/clients/memstat.c b/clients/memstat.c index f2dcfd89..68c78e40 100644 --- a/clients/memstat.c +++ b/clients/memstat.c @@ -19,9 +19,13 @@ static void options_parse(int argc, char *argv[]); static void print_server_listing(memcached_st *memc, memcached_stat_st *stat, memcached_server_st *server_list); +static void print_analysis_report(memcached_st *memc, + memcached_analysis_st *report, + memcached_server_st *server_list); static int opt_verbose= 0; static int opt_displayflag= 0; +static int opt_analyze= 0; static char *opt_servers= NULL; static struct option long_options[]= @@ -32,6 +36,7 @@ static struct option long_options[]= {"debug", no_argument, &opt_verbose, OPT_DEBUG}, {"servers", required_argument, NULL, OPT_SERVERS}, {"flag", no_argument, &opt_displayflag, OPT_FLAG}, + {"analyze", no_argument, NULL, OPT_ANALYZE}, {0, 0, 0, 0}, }; @@ -42,6 +47,7 @@ int main(int argc, char *argv[]) memcached_stat_st *stat; memcached_server_st *servers; memcached_server_st *server_list; + memcached_analysis_st *report; options_parse(argc, argv); @@ -70,12 +76,26 @@ int main(int argc, char *argv[]) if (rc != MEMCACHED_SUCCESS && rc != MEMCACHED_SOME_ERRORS) { printf("Failure to communicate with servers (%s)\n", - memcached_strerror(memc, rc)); + memcached_strerror(memc, rc)); exit(1); } server_list= memcached_server_list(memc); - print_server_listing(memc, stat, server_list); + + if (opt_analyze) + { + report= memcached_analyze(memc, stat, &rc); + if (rc != MEMCACHED_SUCCESS || report == NULL) + { + printf("Failure to analyze servers (%s)\n", + memcached_strerror(memc, rc)); + exit(1); + } + print_analysis_report(memc, report, server_list); + free(report); + } + else + print_server_listing(memc, stat, server_list); free(stat); free(opt_servers); @@ -115,6 +135,41 @@ static void print_server_listing(memcached_st *memc, memcached_stat_st *stat, } } +static void print_analysis_report(memcached_st *memc, + memcached_analysis_st *report, + memcached_server_st *server_list) +{ + uint32_t server_count= memcached_server_count(memc); + + printf("Memcached Cluster Analysis Report\n\n"); + + printf("\tNumber of Servers Analyzed : %d\n", server_count); + printf("\tAverage Item Size (incl/overhead) : %u bytes\n", + report->average_item_size); + + if (server_count == 1) + { + printf("\nFor a detailed report, you must supply multiple servers.\n"); + return; + } + + printf("\n"); + printf("\tNode with most memory consumption : %s:%u (%u bytes)\n", + memcached_server_name(memc, server_list[report->most_consumed_server]), + memcached_server_port(memc, server_list[report->most_consumed_server]), + report->most_used_bytes); + printf("\tNode with least free space : %s:%u (%u bytes remaining)\n", + memcached_server_name(memc, server_list[report->least_free_server]), + memcached_server_port(memc, server_list[report->least_free_server]), + report->least_remaining_bytes); + printf("\tNode with longest uptime : %s:%u (%us)\n", + memcached_server_name(memc, server_list[report->oldest_server]), + memcached_server_port(memc, server_list[report->oldest_server]), + report->longest_uptime); + printf("\tPool-wide Hit Ratio : %1.f%%\n", report->pool_hit_ratio); + printf("\n"); +} + static void options_parse(int argc, char *argv[]) { memcached_programs_help_st help_options[]= @@ -127,7 +182,7 @@ static void options_parse(int argc, char *argv[]) while (1) { - option_rv= getopt_long(argc, argv, "Vhvds:", long_options, &option_index); + option_rv= getopt_long(argc, argv, "Vhvds:a", long_options, &option_index); if (option_rv == -1) break; switch (option_rv) { @@ -148,6 +203,9 @@ static void options_parse(int argc, char *argv[]) case OPT_SERVERS: /* --servers or -s */ opt_servers= strdup(optarg); break; + case OPT_ANALYZE: /* --analyze or -a */ + opt_analyze= OPT_ANALYZE; + break; case '?': /* getopt_long already printed an error message. */ exit(1); diff --git a/clients/utilities.c b/clients/utilities.c index 1e8e89d7..adab23fa 100644 --- a/clients/utilities.c +++ b/clients/utilities.c @@ -44,6 +44,7 @@ static char *lookup_help(memcached_options option) case OPT_FLUSH: return("Flush servers before running tests."); case OPT_HASH: return("Select hash type."); case OPT_BINARY: return("Switch to binary protocol."); + case OPT_ANALYZE: return("Analyze the provided servers."); }; WATCHPOINT_ASSERT(0); diff --git a/docs/Makefile.am b/docs/Makefile.am index e21650e8..2c21bae0 100644 --- a/docs/Makefile.am +++ b/docs/Makefile.am @@ -24,7 +24,8 @@ EXTRA_DIST = libmemcached.pod\ memcached_set.pod\ memcached_version.pod\ memflush.pod\ - memcached_flush_buffers.pod + memcached_flush_buffers.pod\ + memcached_analyze.pod man_MANS = libmemcached.3\ libmemcached_examples.3\ @@ -39,6 +40,7 @@ man_MANS = libmemcached.3\ memcached_add_by_key.3\ memcached_append.3\ memcached_append_by_key.3\ + memcached_analyze.3\ memcached_behavior_get.3\ memcached_behavior_set.3\ memcached_callback_get.3\ @@ -82,7 +84,7 @@ man_MANS = libmemcached.3\ memcached_verbosity.3\ memcached_lib_version.3\ memcached_version.3\ - memcached_flush_buffers.3 + memcached_flush_buffers.3 libmemcached.3: libmemcached.pod @POD2MAN@ -c "libmemcached" -r "" -s 3 libmemcached.pod > libmemcached.3 @@ -267,6 +269,9 @@ memcached_lib_version.3: memcached_version.pod memcached_flush_buffers.3: memcached_flush_buffers.pod @POD2MAN@ -c "libmemcached" -r "" -s 3 memcached_flush_buffers.pod > memcached_flush_buffers.3 +memcached_analyze.3: memcached_analyze.pod + @POD2MAN@ -c "libmemcached" -r "" -s 3 memcached_analyze.pod > memcached_analyze.3 + memcp.1: memcp.pod @POD2MAN@ -c "libmemcached" -r "" -s 1 memcp.pod > memcp.1 diff --git a/docs/memcached_analyze.pod b/docs/memcached_analyze.pod new file mode 100644 index 00000000..42e8c4d6 --- /dev/null +++ b/docs/memcached_analyze.pod @@ -0,0 +1,52 @@ +=head1 NAME + +memcached_analyze + +=head1 LIBRARY + +C Client Library for memcached (libmemcached, -lmemcached) + +=head1 SYNOPSIS + + #include + + memcached_analysis_st *memcached_analyze(memcached_st *ptr, + memcached_stat_st *stat, + memcached_return *error); + +=head1 DESCRIPTION + +libmemcached(3) has the ability to query a memcached server (or collection +of servers) for their current state. Queries to find state return a +C structure. You are responsible for freeing this structure. + +memcached_analyze() analyzes useful information based on the provided servers +and sets the result to the C structure. The return value +must be freed by the calling application. + +A command line tool, memstat(1) with the option --analyze, is provided so that +you do not have to write an application to use this method. + +=head1 RETURN + +A pointer to the allocated C structure on success and +a NULL pointer on failure. You may inspect the error detail by checking the +C value. + +Any method returning a C expects you to free the +memory allocated for it. + +=head1 HOME + +To find out more information please check: +L + +=head1 AUTHOR + +Toru Maesaka, Edev@torum.netE + +=head1 SEE ALSO + +memcached(1) libmemcached(3) memcached_strerror(3) + +=cut diff --git a/libmemcached/Makefile.am b/libmemcached/Makefile.am index bb5a483c..bcb3cd65 100644 --- a/libmemcached/Makefile.am +++ b/libmemcached/Makefile.am @@ -32,6 +32,7 @@ libmemcached_la_SOURCES = crc.c \ hsieh_hash.c \ memcached.c \ memcached_auto.c \ + memcached_analyze.c \ memcached_behavior.c \ memcached_callback.c \ memcached_connect.c \ @@ -56,7 +57,7 @@ libmemcached_la_SOURCES = crc.c \ memcached_storage.c \ memcached_string.c \ memcached_stats.c \ - memcached_strerror.c \ + memcached_strerror.c \ memcached_verbosity.c \ memcached_version.c \ murmur_hash.c \ diff --git a/libmemcached/memcached.h b/libmemcached/memcached.h index 770efecf..8db371a9 100644 --- a/libmemcached/memcached.h +++ b/libmemcached/memcached.h @@ -44,6 +44,17 @@ struct memcached_continuum_item_st { #define LIBMEMCACHED_VERSION_STRING "0.25" +struct memcached_analysis_st { + uint64_t most_used_bytes; + uint64_t least_remaining_bytes; + uint32_t average_item_size; + uint32_t longest_uptime; + uint32_t least_free_server; + uint32_t most_consumed_server; + uint32_t oldest_server; + double pool_hit_ratio; +}; + struct memcached_stat_st { uint32_t pid; uint32_t uptime; diff --git a/libmemcached/memcached_analyze.c b/libmemcached/memcached_analyze.c new file mode 100644 index 00000000..6e5f54e7 --- /dev/null +++ b/libmemcached/memcached_analyze.c @@ -0,0 +1,99 @@ +#include "common.h" + +static void calc_largest_consumption(memcached_analysis_st *result, + const uint32_t server_num, + const uint64_t nbytes) +{ + if (result->most_used_bytes < nbytes) + { + result->most_used_bytes= nbytes; + result->most_consumed_server= server_num; + } +} + +static void calc_oldest_node(memcached_analysis_st *result, + const uint32_t server_num, + const uint32_t uptime) +{ + if (result->longest_uptime < uptime) + { + result->longest_uptime= uptime; + result->oldest_server= server_num; + } +} + +static void calc_least_free_node(memcached_analysis_st *result, + const uint32_t server_num, + const long max_allowed_bytes, + const long used_bytes) +{ + long remaining_bytes= max_allowed_bytes - used_bytes; + + if (result->least_remaining_bytes == 0 || + remaining_bytes < result->least_remaining_bytes) + { + result->least_remaining_bytes= remaining_bytes; + result->least_free_server= server_num; + } +} + +static void calc_average_item_size(memcached_analysis_st *result, + const uint64_t total_items, + const uint64_t total_bytes) +{ + if (total_items > 0 && total_bytes > 0) + result->average_item_size= total_bytes / total_items; +} + +static void calc_hit_ratio(memcached_analysis_st *result, + const uint64_t total_get_hits, + const uint64_t total_get_cmds) +{ + if (total_get_hits == 0 || total_get_cmds == 0) + { + result->pool_hit_ratio= 0; + return; + } + + double temp= (double)total_get_hits/total_get_cmds; + result->pool_hit_ratio= temp * 100; +} + +memcached_analysis_st *memcached_analyze(memcached_st *memc, + memcached_stat_st *stat, + memcached_return *error) +{ + uint64_t total_items= 0, total_bytes= 0; + uint64_t total_get_cmds= 0, total_get_hits= 0; + uint32_t server_count, x; + memcached_analysis_st *result; + + *error= MEMCACHED_SUCCESS; + server_count= memcached_server_count(memc); + result= (memcached_analysis_st*)malloc(sizeof(memcached_analysis_st) + * (memc->number_of_hosts)); + if (!result) + { + *error= MEMCACHED_MEMORY_ALLOCATION_FAILURE; + return NULL; + } + + memset(result, 0, sizeof(*result)); + + for (x= 0; x < server_count; x++) + { + calc_largest_consumption(result, x, stat[x].bytes); + calc_oldest_node(result, x, stat[x].uptime); + calc_least_free_node(result, x, stat[x].limit_maxbytes, stat[x].bytes); + + total_get_hits+= stat[x].get_hits; + total_get_cmds+= stat[x].cmd_get; + total_items+= stat[x].curr_items; + total_bytes+= stat[x].bytes; + } + + calc_average_item_size(result, total_items, total_bytes); + calc_hit_ratio(result, total_get_hits, total_get_cmds); + + return result; +} diff --git a/libmemcached/memcached_server.h b/libmemcached/memcached_server.h index 8b60a384..ced58313 100644 --- a/libmemcached/memcached_server.h +++ b/libmemcached/memcached_server.h @@ -64,6 +64,8 @@ memcached_server_st *memcached_server_create_with(memcached_st *memc, memcached_ void memcached_server_free(memcached_server_st *ptr); memcached_server_st *memcached_server_clone(memcached_server_st *clone, memcached_server_st *ptr); +memcached_analysis_st *memcached_analyze(memcached_st *memc, memcached_stat_st *stat, + memcached_return *error); #ifdef __cplusplus diff --git a/libmemcached/memcached_types.h b/libmemcached/memcached_types.h index f00f9afa..98ba5775 100644 --- a/libmemcached/memcached_types.h +++ b/libmemcached/memcached_types.h @@ -15,6 +15,7 @@ extern "C" { typedef struct memcached_st memcached_st; typedef struct memcached_stat_st memcached_stat_st; +typedef struct memcached_analysis_st memcached_analysis_st; typedef struct memcached_result_st memcached_result_st; typedef struct memcached_string_st memcached_string_st; typedef struct memcached_server_st memcached_server_st; diff --git a/tests/function.c b/tests/function.c index fcb5f2ba..6ee9f27b 100644 --- a/tests/function.c +++ b/tests/function.c @@ -3073,6 +3073,26 @@ static test_return noreply_test(memcached_st *memc) return TEST_SUCCESS; } +static memcached_return analyzer_test(memcached_st *memc) +{ + memcached_return rc; + memcached_stat_st *stat; + memcached_analysis_st *report; + + stat= memcached_stat(memc, NULL, &rc); + assert(rc == MEMCACHED_SUCCESS); + assert(stat); + + report= memcached_analyze(memc, stat, &rc); + assert(rc == MEMCACHED_SUCCESS); + assert(report); + + free(report); + memcached_stat_free(NULL, stat); + + return MEMCACHED_SUCCESS; +} + /* Clean the server before beginning testing */ test_st tests[] ={ {"flush", 0, flush_test }, @@ -3114,6 +3134,7 @@ test_st tests[] ={ {"read_through", 1, read_through }, {"delete_through", 1, delete_through }, {"noreply", 1, noreply_test}, + {"analyzer", 1, analyzer_test}, {0, 0, 0} }; -- 2.30.2