From 96cdb1bc8279c9c0cef602b8b536228ead9f1154 Mon Sep 17 00:00:00 2001 From: Brian Aker Date: Tue, 23 Oct 2007 16:41:49 -0700 Subject: [PATCH] Allocation fixes in string library (now we can run bigger tests!) Still a bug somewhere in get I suspect... mainly because of lazy execution. --- include/memcached.h | 2 +- lib/common.h | 2 +- lib/memcached_get.c | 30 ++++-- lib/memcached_string.c | 57 +++++++---- tests/Makefile.am | 2 +- tests/output.res | 8 +- tests/test.c | 210 ++++++++++++++++++++++++++++++++++++++--- 7 files changed, 264 insertions(+), 47 deletions(-) diff --git a/include/memcached.h b/include/memcached.h index 023ba425..64afedc0 100644 --- a/include/memcached.h +++ b/include/memcached.h @@ -214,7 +214,7 @@ char ** memcached_stat_get_keys(memcached_st *ptr, memcached_stat_st *stat, #define WATCHPOINT printf("\nWATCHPOINT %s:%d (%s)\n", __FILE__, __LINE__,__func__);fflush(stdout); #define WATCHPOINT_ERROR(A) printf("\nWATCHPOINT %s:%d %s\n", __FILE__, __LINE__, memcached_strerror(NULL, A));fflush(stdout); #define WATCHPOINT_STRING(A) printf("\nWATCHPOINT %s:%d (%s) %s\n", __FILE__, __LINE__,__func__,A);fflush(stdout); -#define WATCHPOINT_NUMBER(A) printf("\nWATCHPOINT %s:%d (%s) %d\n", __FILE__, __LINE__,__func__,A);fflush(stdout); +#define WATCHPOINT_NUMBER(A) printf("\nWATCHPOINT %s:%d (%s) %zu\n", __FILE__, __LINE__,__func__,(size_t)(A));fflush(stdout); #define WATCHPOINT_ERRNO(A) printf("\nWATCHPOINT %s:%d (%s) %s\n", __FILE__, __LINE__,__func__, strerror(A));A= 0;fflush(stdout); diff --git a/lib/common.h b/lib/common.h index d7c9cd61..1f6f678e 100644 --- a/lib/common.h +++ b/lib/common.h @@ -22,7 +22,7 @@ #include -#include +#include "memcached_io.h" #include diff --git a/lib/memcached_get.c b/lib/memcached_get.c index 147b6105..1224d8ab 100644 --- a/lib/memcached_get.c +++ b/lib/memcached_get.c @@ -12,14 +12,11 @@ static char *memcached_value_fetch(memcached_st *ptr, char *key, size_t *key_len char *string_ptr; char *end_ptr; - assert(value_length); - assert(flags); - assert(error); - memset(buffer, 0, MEMCACHED_DEFAULT_COMMAND_SIZE); end_ptr= buffer + MEMCACHED_DEFAULT_COMMAND_SIZE; *value_length= 0; + *flags= 0; *error= memcached_response(ptr, buffer, MEMCACHED_DEFAULT_COMMAND_SIZE, server_key); @@ -224,8 +221,15 @@ memcached_return memcached_mget(memcached_st *ptr, { memcached_string_st *string= cursor_key_exec[server_key]; - memcached_string_append_character(ptr, string, ' '); - memcached_string_append(ptr, string, keys[x], key_length[x]); + rc= memcached_string_append_character(ptr, string, ' '); + + if (rc != MEMCACHED_SUCCESS) + assert(0); + + rc= memcached_string_append(ptr, string, keys[x], key_length[x]); + + if (rc != MEMCACHED_SUCCESS) + assert(0); } else { @@ -235,8 +239,13 @@ memcached_return memcached_mget(memcached_st *ptr, if (!string) assert(0); - memcached_string_append(ptr, string, "get ", 4); - memcached_string_append(ptr, string, keys[x], key_length[x]); + rc= memcached_string_append(ptr, string, "get ", 4); + if (rc != MEMCACHED_SUCCESS) + assert(0); + + rc= memcached_string_append(ptr, string, keys[x], key_length[x]); + if (rc != MEMCACHED_SUCCESS) + assert(0); cursor_key_exec[server_key]= string; } @@ -254,7 +263,10 @@ memcached_return memcached_mget(memcached_st *ptr, rc= memcached_connect(ptr, x); memcached_string_st *string= cursor_key_exec[x]; - memcached_string_append(ptr, string, "\r\n", 2); + + rc= memcached_string_append(ptr, string, "\r\n", 2); + if (rc != MEMCACHED_SUCCESS) + assert(0); if ((memcached_io_write(ptr, x, string->string, memcached_string_length(ptr, string), 1)) == -1) diff --git a/lib/memcached_string.c b/lib/memcached_string.c index d8a99649..b79fc161 100644 --- a/lib/memcached_string.c +++ b/lib/memcached_string.c @@ -6,19 +6,23 @@ memcached_return memcached_string_check(memcached_string_st *string, size_t need { size_t current_offset= string->end - string->string; char *new_value; + size_t adjust= (need - (size_t)(string->current_size - (size_t)(string->end - string->string))) / string->block_size; - new_value= (char *)realloc(string->string, sizeof(char) * string->block_size); + adjust++; - if (!new_value) + new_value= (char *)realloc(string->string, + sizeof(char) * ((adjust * string->block_size) + string->current_size)); + + if (new_value == NULL) return MEMCACHED_MEMORY_ALLOCATION_FAILURE; string->string= new_value; string->end= string->string + current_offset; - string->current_size+= string->block_size; + string->current_size+= (string->block_size * adjust); /* We zero the block structure we just realloced */ - memset((string->string + string->current_size) - string->block_size , 0, + memset((string->string + current_offset), 0, sizeof(char) * string->block_size); } @@ -27,15 +31,25 @@ memcached_return memcached_string_check(memcached_string_st *string, size_t need memcached_string_st *memcached_string_create(memcached_st *ptr, size_t initial_size) { + memcached_return rc; memcached_string_st *string; /* Saving malloc calls :) */ - string= (memcached_string_st *)malloc(sizeof(memcached_string_st) + (sizeof(char) * initial_size)); + string= (memcached_string_st *)malloc(sizeof(memcached_string_st)); if (!string) return NULL; + string->end= string->string; memset(string, 0, sizeof(memcached_string_st)); string->block_size= initial_size; - memcached_string_check(string, initial_size); + + rc= memcached_string_check(string, initial_size); + if (rc != MEMCACHED_SUCCESS) + { + free(string); + return NULL; + } + + assert(string->string == string->end); return string; } @@ -44,7 +58,13 @@ memcached_return memcached_string_append_character(memcached_st *ptr, memcached_string_st *string, char character) { - memcached_string_check(string, 1); + memcached_return rc; + + rc= memcached_string_check(string, 1); + + if (rc != MEMCACHED_SUCCESS) + return rc; + *string->end= ' '; string->end++; @@ -54,25 +74,22 @@ memcached_return memcached_string_append_character(memcached_st *ptr, memcached_return memcached_string_append(memcached_st *ptr, memcached_string_st *string, char *value, size_t length) { - memcached_string_check(string, length); + memcached_return rc; + + rc= memcached_string_check(string, length); + + if (rc != MEMCACHED_SUCCESS) + return rc; + + assert(string->string); + assert(string->end >= string->string && string->end <= string->string + string->current_size); + memcpy(string->end, value, length); string->end+= length; return MEMCACHED_SUCCESS; } -#ifdef CRAP -size_t memcached_string_length(memcached_st *ptr, memcached_string_st *string) -{ - return (size_t)(string->end - string->string); -} - -size_t memcached_string_size(memcached_st *ptr, memcached_string_st *string) -{ - return string->current_size; -} -#endif - size_t memcached_string_backspace(memcached_st *ptr, memcached_string_st *string, size_t remove) { if (string->end - string->string > remove) diff --git a/tests/Makefile.am b/tests/Makefile.am index 25b1962e..27ccdab3 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -1,4 +1,4 @@ -INCLUDES = -I$(top_builddir)/include +INCLUDES = -I$(top_builddir)/include -I$(top_builddir)/lib LDADDS = ../lib/libmemcached.la EXTRA_DIST = output.res output2.res\ diff --git a/tests/output.res b/tests/output.res index 813c18b1..7060e562 100644 --- a/tests/output.res +++ b/tests/output.res @@ -26,7 +26,7 @@ Error 21 -> SERVER END Error 22 -> SERVER DELETE Error 23 -> SERVER VALUE Error 24 -> STAT VALUE -Error 25 -> UNKOWN ERROR SEE MY_ERRNO +Error 25 -> SYSTEM ERROR Found key pid Found key uptime Found key time @@ -130,7 +130,7 @@ Error 21 -> SERVER END Error 22 -> SERVER DELETE Error 23 -> SERVER VALUE Error 24 -> STAT VALUE -Error 25 -> UNKOWN ERROR SEE MY_ERRNO +Error 25 -> SYSTEM ERROR Found key pid Found key uptime Found key time @@ -234,7 +234,7 @@ Error 21 -> SERVER END Error 22 -> SERVER DELETE Error 23 -> SERVER VALUE Error 24 -> STAT VALUE -Error 25 -> UNKOWN ERROR SEE MY_ERRNO +Error 25 -> SYSTEM ERROR Found key pid Found key uptime Found key time @@ -338,7 +338,7 @@ Error 21 -> SERVER END Error 22 -> SERVER DELETE Error 23 -> SERVER VALUE Error 24 -> STAT VALUE -Error 25 -> UNKOWN ERROR SEE MY_ERRNO +Error 25 -> SYSTEM ERROR Found key pid Found key uptime Found key time diff --git a/tests/test.c b/tests/test.c index dce28e36..e7391dfe 100644 --- a/tests/test.c +++ b/tests/test.c @@ -8,6 +8,7 @@ #include #include #include +#include "../lib/common.h" long int timedif(struct timeval a, struct timeval b) { @@ -572,24 +573,24 @@ void user_supplied_bug2(memcached_st *memc) for (x= 0, errors= 0, total= 0 ; total < 20 * 1024576 ; x++) { - memcached_return rc; + memcached_return rc= MEMCACHED_SUCCESS; char buffer[SMALL_STRING_LEN]; - uint16_t flags; - size_t val_len; + uint16_t flags= 0; + size_t val_len= 0; char *getval; + memset(buffer, 0, SMALL_STRING_LEN); + snprintf(buffer, SMALL_STRING_LEN, "%u", x); getval= memcached_get(memc, buffer, strlen(buffer), &val_len, &flags, &rc); if (rc != MEMCACHED_SUCCESS) { - WATCHPOINT_ERROR(rc); - errors++; - if ( errors == 10 ) - { - fprintf(stderr, "last: %s: len %zu flags: %u\n", buffer, val_len, flags); + if (rc == MEMCACHED_NOTFOUND) + errors++; + else assert(0); - } + continue; } total+= val_len; @@ -598,6 +599,134 @@ void user_supplied_bug2(memcached_st *memc) } } +#define KEY_COUNT 2000 // * 1024576 +void user_supplied_bug3(memcached_st *memc) +{ + memcached_return rc; + unsigned int setter; + unsigned int x; + char **keys; + size_t key_lengths[KEY_COUNT]; + + setter= 1; + memcached_behavior_set(memc, MEMCACHED_BEHAVIOR_NO_BLOCK, &setter); + memcached_behavior_set(memc, MEMCACHED_BEHAVIOR_TCP_NODELAY, &setter); +#ifdef NOT_YET + setter = 20 * 1024576; + memcached_behavior_set(memc, MEMCACHED_BEHAVIOR_SOCKET_SEND_SIZE, &setter); + setter = 20 * 1024576; + memcached_behavior_set(memc, MEMCACHED_BEHAVIOR_SOCKET_RECV_SIZE, &setter); + getter = memcached_behavior_get(memc, MEMCACHED_BEHAVIOR_SOCKET_SEND_SIZE); + getter = memcached_behavior_get(memc, MEMCACHED_BEHAVIOR_SOCKET_RECV_SIZE); +#endif + + keys= (char **)malloc(sizeof(char *) * KEY_COUNT); + assert(keys); + memset(keys, 0, (sizeof(char *) * KEY_COUNT)); + for (x= 0; x < KEY_COUNT; x++) + { + char buffer[20]; + + snprintf(buffer, 30, "%u", x); + keys[x]= strdup(buffer); + key_lengths[x]= strlen(keys[x]); + } + + rc= memcached_mget(memc, keys, key_lengths, KEY_COUNT); + assert(rc == MEMCACHED_SUCCESS); + + /* Turn this into a help function */ + { + char *return_key; + size_t return_key_length; + char *return_value; + size_t return_value_length; + uint16_t flags; + + while ((return_value= memcached_fetch(memc, return_key, &return_key_length, + &return_value_length, &flags, &rc))) + { + assert(return_value); + assert(rc == MEMCACHED_SUCCESS); + free(return_value); + } + } + + for (x= 0; x < KEY_COUNT; x++) + free(keys[x]); + free(keys); +} + +void string_alloc_null(memcached_st *memc) +{ + memcached_string_st *string; + + string= memcached_string_create(memc, 0); + assert(string); + memcached_string_free(memc, string); +} + +void string_alloc_with_size(memcached_st *memc) +{ + memcached_string_st *string; + + string= memcached_string_create(memc, 1024); + assert(string); + memcached_string_free(memc, string); +} + +void string_alloc_with_size_toobig(memcached_st *memc) +{ + memcached_string_st *string; + + string= memcached_string_create(memc, 1024*100000000000); + assert(string == NULL); +} + +void string_alloc_append(memcached_st *memc) +{ + unsigned int x; + char buffer[SMALL_STRING_LEN]; + memcached_string_st *string; + + /* Ring the bell! */ + memset(buffer, 6, SMALL_STRING_LEN); + + string= memcached_string_create(memc, 100); + assert(string); + + for (x= 0; x < 1024; x++) + { + memcached_return rc; + rc= memcached_string_append(memc, string, buffer, SMALL_STRING_LEN); + assert(rc == MEMCACHED_SUCCESS); + } + memcached_string_free(memc, string); +} + +void string_alloc_append_toobig(memcached_st *memc) +{ + memcached_return rc; + unsigned int x; + char buffer[SMALL_STRING_LEN]; + memcached_string_st *string; + + /* Ring the bell! */ + memset(buffer, 6, SMALL_STRING_LEN); + + string= memcached_string_create(memc, 100); + assert(string); + + for (x= 0; x < 1024; x++) + { + rc= memcached_string_append(memc, string, buffer, SMALL_STRING_LEN); + assert(rc == MEMCACHED_SUCCESS); + } + rc= memcached_string_append(memc, string, buffer, 1024*100000000000); + assert(rc == MEMCACHED_MEMORY_ALLOCATION_FAILURE); + memcached_string_free(memc, string); +} + void add_host_test1(memcached_st *memc) { unsigned int x; @@ -681,7 +810,7 @@ int main(int argc, char *argv[]) {"add", 0, add_test }, {"replace", 0, replace_test }, {"delete", 1, delete_test }, - {"get", 0, get_test }, + {"get", 1, get_test }, {"get2", 0, get_test2 }, {"get3", 0, get_test3 }, {"get4", 0, get_test4 }, @@ -697,9 +826,19 @@ int main(int argc, char *argv[]) {0, 0, 0} }; + test_st string_tests[] ={ + {"string alloc with null", 0, string_alloc_null }, + {"string alloc with 1K", 0, string_alloc_with_size }, + {"string alloc with malloc failure", 0, string_alloc_with_size_toobig }, + {"string append", 0, string_alloc_append }, + {"string append failure (too big)", 0, string_alloc_append_toobig }, + {0, 0, 0} + }; + test_st user_tests[] ={ {"user_supplied_bug1", 0, user_supplied_bug1 }, {"user_supplied_bug2", 0, user_supplied_bug2 }, + // {"user_supplied_bug3", 0, user_supplied_bug3 }, {0, 0, 0} }; @@ -719,6 +858,9 @@ int main(int argc, char *argv[]) memc= memcached_create(NULL); assert(memc); + if (tests[x].requires_flush) + memcached_flush(memc, 0); + rc= memcached_server_push(memc, servers); assert(rc == MEMCACHED_SUCCESS); @@ -758,6 +900,9 @@ int main(int argc, char *argv[]) memc= memcached_create(NULL); assert(memc); + if (tests[x].requires_flush) + memcached_flush(memc, 0); + rc= memcached_server_push(memc, servers); assert(rc == MEMCACHED_SUCCESS); @@ -790,6 +935,9 @@ int main(int argc, char *argv[]) memc= memcached_create(NULL); assert(memc); + if (tests[x].requires_flush) + memcached_flush(memc, 0); + rc= memcached_server_push(memc, servers); assert(rc == MEMCACHED_SUCCESS); @@ -823,6 +971,9 @@ int main(int argc, char *argv[]) memc= memcached_create(NULL); assert(memc); + if (tests[x].requires_flush) + memcached_flush(memc, 0); + rc= memcached_server_push(memc, servers); assert(rc == MEMCACHED_SUCCESS); @@ -839,13 +990,47 @@ int main(int argc, char *argv[]) } } + if ((test_to_run && !strcmp(test_to_run, "string")) || !test_to_run) + { + fprintf(stderr, "\nString tests (internal API)\n\n"); + for (x= 0; string_tests[x].function_name; x++) + { + if (wildcard) + if (strcmp(wildcard, string_tests[x].function_name)) + continue; + + memcached_st *memc; + memcached_return rc; + struct timeval start_time, end_time; + + memc= memcached_create(NULL); + assert(memc); + + if (tests[x].requires_flush) + memcached_flush(memc, 0); + + rc= memcached_server_push(memc, servers); + assert(rc == MEMCACHED_SUCCESS); + + fprintf(stderr, "Testing %s", string_tests[x].function_name); + gettimeofday(&start_time, NULL); + string_tests[x].function(memc); + gettimeofday(&end_time, NULL); + long int load_time= timedif(end_time, start_time); + fprintf(stderr, "\t\t\t\t\t %ld.%03ld [ ok ]\n", load_time / 1000, + load_time % 1000); + assert(memc); + memcached_free(memc); + } + } + if ((test_to_run && !strcmp(test_to_run, "user")) || !test_to_run) { fprintf(stderr, "\nUser Supplied tests\n\n"); for (x= 0; user_tests[x].function_name; x++) { if (wildcard) - if (strcmp(wildcard, tests[x].function_name)) + if (strcmp(wildcard, user_tests[x].function_name)) continue; memcached_st *memc; @@ -855,6 +1040,9 @@ int main(int argc, char *argv[]) memc= memcached_create(NULL); assert(memc); + if (tests[x].requires_flush) + memcached_flush(memc, 0); + rc= memcached_server_push(memc, servers); assert(rc == MEMCACHED_SUCCESS); -- 2.30.2