From abc87c2867661669e897142dbfd531e32a14981b Mon Sep 17 00:00:00 2001 From: Brian Aker Date: Mon, 28 Mar 2011 17:16:54 -0700 Subject: [PATCH] Update for virtual buckets/wasn't this just originally the wheel? --- libmemcached/behavior.c | 37 ++++++++- libmemcached/behavior.h | 7 ++ libmemcached/constants.h | 1 + libmemcached/hash.c | 48 ++++++++++-- libmemcached/hash.h | 38 +++++++-- libmemcached/hosts.c | 1 + libmemcached/include.am | 6 +- libmemcached/memcached.c | 46 +++++++++-- libmemcached/memcached.h | 43 +++++++++-- libmemcached/virtual_bucket.c | 117 ++++++++++++++++++++++++++++ libmemcached/virtual_bucket.h | 59 ++++++++++++++ tests/include.am | 8 +- tests/mem_functions.c | 10 +++ tests/parser.cc | 2 +- tests/virtual_buckets.cc | 141 ++++++++++++++++++++++++++++++++++ tests/virtual_buckets.h | 53 +++++++++++++ 16 files changed, 581 insertions(+), 36 deletions(-) create mode 100644 libmemcached/virtual_bucket.c create mode 100644 libmemcached/virtual_bucket.h create mode 100644 tests/virtual_buckets.cc create mode 100644 tests/virtual_buckets.h diff --git a/libmemcached/behavior.c b/libmemcached/behavior.c index 2fe17dc8..0df96033 100644 --- a/libmemcached/behavior.c +++ b/libmemcached/behavior.c @@ -9,7 +9,9 @@ * */ -#include "common.h" +#include +#include + #include #include @@ -462,7 +464,40 @@ const char *libmemcached_string_distribution(const memcached_server_distribution case MEMCACHED_DISTRIBUTION_RANDOM: return "MEMCACHED_DISTRIBUTION_RANDOM"; case MEMCACHED_DISTRIBUTION_CONSISTENT_KETAMA_SPY: return "MEMCACHED_DISTRIBUTION_CONSISTENT_KETAMA_SPY"; case MEMCACHED_DISTRIBUTION_CONSISTENT_WEIGHTED: return "MEMCACHED_DISTRIBUTION_CONSISTENT_WEIGHTED"; + case MEMCACHED_DISTRIBUTION_VIRTUAL_BUCKET: return "MEMCACHED_DISTRIBUTION_VIRTUAL_BUCKET"; default: case MEMCACHED_DISTRIBUTION_CONSISTENT_MAX: return "INVALID memcached_server_distribution_t"; } } + +memcached_return_t memcached_bucket_set(memcached_st *self, + const uint32_t *host_map, + const uint32_t *forward_map, + const uint32_t buckets, + const uint32_t replicas) +{ + memcached_return_t rc; + + if (! self) + return MEMCACHED_INVALID_ARGUMENTS; + + if (! host_map) + return MEMCACHED_INVALID_ARGUMENTS; + + memcached_server_distribution_t old; + old= memcached_behavior_get_distribution(self); + + rc =memcached_behavior_set_distribution(self, MEMCACHED_DISTRIBUTION_VIRTUAL_BUCKET); + if (rc != MEMCACHED_SUCCESS) + { + return rc; + } + + rc= memcached_virtual_bucket_create(self, host_map, forward_map, buckets, replicas); + if (rc != MEMCACHED_SUCCESS) + { + memcached_behavior_set_distribution(self, old); + } + + return rc; +} diff --git a/libmemcached/behavior.h b/libmemcached/behavior.h index 1c8d42e2..ca6fb492 100644 --- a/libmemcached/behavior.h +++ b/libmemcached/behavior.h @@ -49,6 +49,13 @@ LIBMEMCACHED_LOCAL LIBMEMCACHED_LOCAL const char *libmemcached_string_distribution(const memcached_server_distribution_t flag); +LIBMEMCACHED_API + memcached_return_t memcached_bucket_set(memcached_st *self, + const uint32_t *host_map, + const uint32_t *forward_map, + const uint32_t buckets, + const uint32_t replicas); + #ifdef __cplusplus } #endif diff --git a/libmemcached/constants.h b/libmemcached/constants.h index ec7c66a2..e2d69448 100644 --- a/libmemcached/constants.h +++ b/libmemcached/constants.h @@ -91,6 +91,7 @@ enum memcached_server_distribution_t { MEMCACHED_DISTRIBUTION_RANDOM, MEMCACHED_DISTRIBUTION_CONSISTENT_KETAMA_SPY, MEMCACHED_DISTRIBUTION_CONSISTENT_WEIGHTED, + MEMCACHED_DISTRIBUTION_VIRTUAL_BUCKET, MEMCACHED_DISTRIBUTION_CONSISTENT_MAX }; diff --git a/libmemcached/hash.c b/libmemcached/hash.c index 9cb02b07..bf946423 100644 --- a/libmemcached/hash.c +++ b/libmemcached/hash.c @@ -1,15 +1,43 @@ -/* LibMemcached - * Copyright (C) 2006-2010 Brian Aker - * All rights reserved. +/* vim:expandtab:shiftwidth=2:tabstop=2:smarttab: + * + * Libmemcached library * - * Use and distribution licensed under the BSD license. See - * the COPYING file in the parent directory for full text. + * Copyright (C) 2011 Data Differential, http://datadifferential.com/ + * Copyright (C) 2006-2009 Brian Aker All rights reserved. * - * Summary: + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * + * * The names of its contributors may not be used to endorse or + * promote products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * */ -#include "common.h" + +#include +#include uint32_t memcached_generate_hash_value(const char *key, size_t key_length, memcached_hash_t hash_algorithm) @@ -55,8 +83,12 @@ static uint32_t dispatch_host(const memcached_st *ptr, uint32_t hash) return hash % memcached_server_count(ptr); case MEMCACHED_DISTRIBUTION_RANDOM: return (uint32_t) random() % memcached_server_count(ptr); - case MEMCACHED_DISTRIBUTION_CONSISTENT_MAX: + case MEMCACHED_DISTRIBUTION_VIRTUAL_BUCKET: + { + return memcached_virtual_bucket_get(ptr, hash); + } default: + case MEMCACHED_DISTRIBUTION_CONSISTENT_MAX: WATCHPOINT_ASSERT(0); /* We have added a distribution without extending the logic */ return hash % memcached_server_count(ptr); } diff --git a/libmemcached/hash.h b/libmemcached/hash.h index 1480c3b9..f36ad286 100644 --- a/libmemcached/hash.h +++ b/libmemcached/hash.h @@ -1,11 +1,37 @@ -/* LibMemcached - * Copyright (C) 2010 Brian Aker - * All rights reserved. +/* vim:expandtab:shiftwidth=2:tabstop=2:smarttab: + * + * Libmemcached library * - * Use and distribution licensed under the BSD license. See - * the COPYING file in the parent directory for full text. + * Copyright (C) 2011 Data Differential, http://datadifferential.com/ + * Copyright (C) 2006-2009 Brian Aker All rights reserved. * - * Summary: hash interface code + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * + * * The names of its contributors may not be used to endorse or + * promote products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * */ diff --git a/libmemcached/hosts.c b/libmemcached/hosts.c index caff9add..d1fe29b8 100644 --- a/libmemcached/hosts.c +++ b/libmemcached/hosts.c @@ -61,6 +61,7 @@ memcached_return_t run_distribution(memcached_st *ptr) case MEMCACHED_DISTRIBUTION_CONSISTENT_KETAMA_SPY: case MEMCACHED_DISTRIBUTION_CONSISTENT_WEIGHTED: return update_continuum(ptr); + case MEMCACHED_DISTRIBUTION_VIRTUAL_BUCKET: case MEMCACHED_DISTRIBUTION_MODULA: break; case MEMCACHED_DISTRIBUTION_RANDOM: diff --git a/libmemcached/include.am b/libmemcached/include.am index 15139365..42e251d6 100644 --- a/libmemcached/include.am +++ b/libmemcached/include.am @@ -22,7 +22,8 @@ noinst_HEADERS+= \ libmemcached/protocol/ascii_handler.h \ libmemcached/protocol/binary_handler.h \ libmemcached/protocol/common.h \ - libmemcached/response.h + libmemcached/response.h \ + libmemcached/virtual_bucket.h nobase_include_HEADERS+= \ libmemcached/allocators.h \ @@ -127,7 +128,8 @@ libmemcached_libmemcached_la_SOURCES+= \ libmemcached/storage.c \ libmemcached/strerror.c \ libmemcached/verbosity.c \ - libmemcached/version.c + libmemcached/version.c \ + libmemcached/virtual_bucket.c libmemcached/options.cc: libmemcached/options/parser.h diff --git a/libmemcached/memcached.c b/libmemcached/memcached.c index 7324de41..149437af 100644 --- a/libmemcached/memcached.c +++ b/libmemcached/memcached.c @@ -1,15 +1,42 @@ -/* LibMemcached - * Copyright (C) 2006-2010 Brian Aker - * All rights reserved. +/* vim:expandtab:shiftwidth=2:tabstop=2:smarttab: + * + * Libmemcached library * - * Use and distribution licensed under the BSD license. See - * the COPYING file in the parent directory for full text. + * Copyright (C) 2011 Data Differential, http://datadifferential.com/ + * Copyright (C) 2006-2009 Brian Aker All rights reserved. * - * Summary: + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * + * * The names of its contributors may not be used to endorse or + * promote products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * */ -#include "common.h" +#include +#include static const memcached_st global_copy= { .state= { @@ -33,13 +60,14 @@ static const memcached_st global_copy= { .use_udp= false, .verify_key= false, .tcp_keepalive= false, - } + }, }; static inline bool _memcached_init(memcached_st *self) { self->state= global_copy.state; self->flags= global_copy.flags; + self->virtual_bucket= NULL; self->distribution= MEMCACHED_DISTRIBUTION_MODULA; @@ -106,6 +134,8 @@ static void _free(memcached_st *ptr, bool release_st) memcached_server_list_free(memcached_server_list(ptr)); memcached_result_free(&ptr->result); + memcached_virtual_bucket_free(ptr); + if (ptr->last_disconnected_server) memcached_server_free(ptr->last_disconnected_server); diff --git a/libmemcached/memcached.h b/libmemcached/memcached.h index 435a6f60..311d5f26 100644 --- a/libmemcached/memcached.h +++ b/libmemcached/memcached.h @@ -1,12 +1,37 @@ -/* LibMemcached - * Copyright (C) 2006-2009 Brian Aker - * All rights reserved. +/* vim:expandtab:shiftwidth=2:tabstop=2:smarttab: + * + * Libmemcached library * - * Use and distribution licensed under the BSD license. See - * the COPYING file in the parent directory for full text. + * Copyright (C) 2011 Data Differential, http://datadifferential.com/ + * Copyright (C) 2006-2009 Brian Aker All rights reserved. * - * Summary: interface for memcached server - * Description: main include file for libmemcached + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * + * * The names of its contributors may not be used to endorse or + * promote products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * */ @@ -66,6 +91,7 @@ struct memcached_st { bool is_processing_input:1; bool is_time_for_rebuild:1; } state; + struct { // Everything below here is pretty static. bool auto_eject_hosts:1; @@ -84,6 +110,7 @@ struct memcached_st { bool verify_key:1; bool tcp_keepalive:1; } flags; + memcached_server_distribution_t distribution; hashkit_st hashkit; uint32_t number_of_hosts; @@ -114,6 +141,8 @@ struct memcached_st { memcached_continuum_item_st *continuum; // Ketama } ketama; + struct memcached_virtual_bucket_t *virtual_bucket; + struct _allocators_st { memcached_calloc_fn calloc; memcached_free_fn free; diff --git a/libmemcached/virtual_bucket.c b/libmemcached/virtual_bucket.c new file mode 100644 index 00000000..26dd90c4 --- /dev/null +++ b/libmemcached/virtual_bucket.c @@ -0,0 +1,117 @@ +/* vim:expandtab:shiftwidth=2:tabstop=2:smarttab: + * + * Libmemcached library + * + * Copyright (C) 2011 Data Differential, http://datadifferential.com/ + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * + * * The names of its contributors may not be used to endorse or + * promote products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include +#include + +struct bucket_t { + uint32_t master; + uint32_t forward; +}; + +struct memcached_virtual_bucket_t { + bool has_forward; + uint32_t size; + uint32_t replicas; + struct bucket_t buckets[]; +}; + +memcached_return_t memcached_virtual_bucket_create(memcached_st *self, + const uint32_t *host_map, + const uint32_t *forward_map, + const uint32_t buckets, + const uint32_t replicas) +{ + if (! self || ! host_map || ! buckets) + return MEMCACHED_INVALID_ARGUMENTS; + + memcached_virtual_bucket_free(self); + + struct memcached_virtual_bucket_t *virtual_bucket= (struct memcached_virtual_bucket_t *)malloc(sizeof(struct memcached_virtual_bucket_t) + sizeof(struct bucket_t) *buckets); + + if (! virtual_bucket) + return MEMCACHED_MEMORY_ALLOCATION_FAILURE; + + virtual_bucket->size= buckets; + virtual_bucket->replicas= replicas; + self->virtual_bucket= virtual_bucket; + + for (uint32_t x=0; x < buckets; x++) + { + virtual_bucket->buckets[x].master= host_map[x]; + if (forward_map) + { + virtual_bucket->buckets[x].forward= forward_map[x]; + } + else + { + virtual_bucket->buckets[x].forward= 0; + } + } + + + return MEMCACHED_SUCCESS; +} + +void memcached_virtual_bucket_free(memcached_st *self) +{ + if (! self) + return; + + if (! self->virtual_bucket) + return; + + free(self->virtual_bucket); + self->virtual_bucket= NULL; +} + +uint32_t memcached_virtual_bucket_get(const memcached_st *self, uint32_t digest) +{ + if (! self) + return 0; + + if (! self->virtual_bucket) + return 0; + + if (self->virtual_bucket) + { + uint32_t result= (uint32_t) (digest & (self->virtual_bucket->size -1)); + return self->virtual_bucket->buckets[result].master; + } + + return (uint32_t) (digest & (self->number_of_hosts -1)); +} diff --git a/libmemcached/virtual_bucket.h b/libmemcached/virtual_bucket.h new file mode 100644 index 00000000..e1c38536 --- /dev/null +++ b/libmemcached/virtual_bucket.h @@ -0,0 +1,59 @@ +/* vim:expandtab:shiftwidth=2:tabstop=2:smarttab: + * + * Libmemcached library + * + * Copyright (C) 2011 Data Differential, http://datadifferential.com/ + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * + * * The names of its contributors may not be used to endorse or + * promote products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#pragma once + +#ifdef __cplusplus +extern "C" { +#endif + +LIBMEMCACHED_LOCAL +memcached_return_t memcached_virtual_bucket_create(memcached_st *self, + const uint32_t *host_map, + const uint32_t *forward_map, + const uint32_t buckets, + const uint32_t replicas); + +LIBMEMCACHED_LOCAL +uint32_t memcached_virtual_bucket_get(const memcached_st *self, uint32_t digest); + +LIBMEMCACHED_LOCAL +void memcached_virtual_bucket_free(memcached_st *self); + +#ifdef __cplusplus +} +#endif diff --git a/tests/include.am b/tests/include.am index bd5ef260..490ed023 100644 --- a/tests/include.am +++ b/tests/include.am @@ -37,8 +37,9 @@ noinst_HEADERS+= \ tests/parser.h \ tests/print.h \ tests/replication.h \ - tests/string.h - + tests/string.h \ + tests/virtual_buckets.h + noinst_PROGRAMS+= \ tests/atomsmasher \ @@ -58,7 +59,8 @@ tests_testapp_SOURCES= \ tests/parser.cc \ tests/print.cc \ tests/replication.cc \ - tests/string.cc + tests/string.cc \ + tests/virtual_buckets.cc tests_testapp_DEPENDENCIES= \ $(BUILT_SOURCES) \ diff --git a/tests/mem_functions.c b/tests/mem_functions.c index 68cab451..18669d48 100644 --- a/tests/mem_functions.c +++ b/tests/mem_functions.c @@ -40,6 +40,7 @@ #include "tests/basic.h" #include "tests/error_conditions.h" #include "tests/print.h" +#include "tests/virtual_buckets.h" #ifdef HAVE_LIBMEMCACHEDUTIL @@ -209,6 +210,7 @@ static test_return_t memcached_server_remove_test(memcached_st *ptr) memc= memcached_create(&local_memc); servers= memcached_servers_parse(server_string); + assert(servers); rc= memcached_server_push(memc, servers); memcached_server_list_free(servers); @@ -4961,6 +4963,7 @@ static test_return_t ketama_compatibility_spymemcached(memcached_st *trash) test_true(memcached_behavior_get_distribution(memc) == MEMCACHED_DISTRIBUTION_CONSISTENT_KETAMA_SPY); server_pool = memcached_servers_parse("10.0.1.1:11211 600,10.0.1.2:11211 300,10.0.1.3:11211 200,10.0.1.4:11211 350,10.0.1.5:11211 1000,10.0.1.6:11211 800,10.0.1.7:11211 950,10.0.1.8:11211 100"); + assert(server_pool); memcached_server_push(memc, server_pool); /* verify that the server list was parsed okay. */ @@ -5761,6 +5764,7 @@ static test_return_t regression_bug_728286(memcached_st *unused) { (void)unused; memcached_server_st *servers = memcached_servers_parse("1.2.3.4:99"); + assert(servers); memcached_server_free(servers); return TEST_SUCCESS; @@ -6225,6 +6229,11 @@ test_st parser_tests[] ={ {0, 0, (test_callback_fn)0} }; +test_st virtual_bucket_tests[] ={ + {"basic", 0, (test_callback_fn)virtual_back_map }, + {0, 0, (test_callback_fn)0} +}; + collection_st collection[] ={ #if 0 {"hash_sanity", 0, 0, hash_sanity}, @@ -6289,6 +6298,7 @@ collection_st collection[] ={ {"regression_binary_vs_block", (test_callback_fn)key_setup, (test_callback_fn)key_teardown, regression_binary_vs_block}, {"error_conditions", 0, 0, error_conditions}, {"parser", 0, 0, parser_tests}, + {"virtual buckets", 0, 0, virtual_bucket_tests}, {0, 0, 0, 0} }; diff --git a/tests/parser.cc b/tests/parser.cc index 4fccaeee..61d846c9 100644 --- a/tests/parser.cc +++ b/tests/parser.cc @@ -1,6 +1,6 @@ /* vim:expandtab:shiftwidth=2:tabstop=2:smarttab: * - * Gearmand client and server library. + * Libmemcached Client and Server * * Copyright (C) 2011 Data Differential, http://datadifferential.com/ * All rights reserved. diff --git a/tests/virtual_buckets.cc b/tests/virtual_buckets.cc new file mode 100644 index 00000000..60837d11 --- /dev/null +++ b/tests/virtual_buckets.cc @@ -0,0 +1,141 @@ +/* vim:expandtab:shiftwidth=2:tabstop=2:smarttab: + * + * Libmemcached Client and Server + * + * Copyright (C) 2011 Data Differential, http://datadifferential.com/ + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * + * * The names of its contributors may not be used to endorse or + * promote products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include + +#include + +#include + +struct libtest_string_t { + const char *c_str; + size_t size; +}; + +static inline libtest_string_t libtest_string(const char *arg, size_t arg_size) +{ + libtest_string_t local= { arg, arg_size }; + return local; +} + +#define make_libtest_string(X) libtest_string((X), static_cast(sizeof(X) - 1)) + +static libtest_string_t libtest_string_t_null= { 0, 0}; + +bool libtest_string_is_null(const libtest_string_t &string) +{ + if (string.c_str == 0 and string.size == 0) + return true; + + return false; +} + +struct expect_t { + libtest_string_t key; + uint32_t server_id; + uint32_t bucket_id; +}; + +expect_t basic_keys[]= { + { make_libtest_string("hello"), 0, 0 }, + { make_libtest_string("doctor"), 0, 0 }, + { make_libtest_string("name"), 1, 3 }, + { make_libtest_string("continue"), 1, 3 }, + { make_libtest_string("yesterday"), 0, 0 }, + { make_libtest_string("tomorrow"), 1, 1 }, + { make_libtest_string("another key"), 2, 2 }, + { libtest_string_t_null, 0, 0 } +}; + +test_return_t virtual_back_map(memcached_st *) +{ + memcached_return_t rc; + memcached_server_st *server_pool; + memcached_st *memc; + + memc= memcached_create(NULL); + test_true(memc); + + uint32_t server_map[] = { 0, 1, 2, 1 }; + rc= memcached_bucket_set(memc, server_map, NULL, 4, 2); + test_true(rc == MEMCACHED_SUCCESS); + + memcached_server_distribution_t dt; + dt= memcached_behavior_get_distribution(memc); + test_true(dt == MEMCACHED_DISTRIBUTION_VIRTUAL_BUCKET); + + memcached_behavior_set_key_hash(memc, MEMCACHED_HASH_CRC); + test_true(rc == MEMCACHED_SUCCESS); + + memcached_hash_t hash_type= memcached_behavior_get_key_hash(memc); + test_true(hash_type == MEMCACHED_HASH_CRC); + + server_pool = memcached_servers_parse("localhost:11211, localhost1:11210, localhost2:11211"); + test_true(server_pool); + memcached_server_push(memc, server_pool); + + /* verify that the server list was parsed okay. */ + test_true(memcached_server_count(memc) == 3); + test_true(strcmp(server_pool[0].hostname, "localhost") == 0); + test_true(server_pool[0].port == 11211); + + test_true(strcmp(server_pool[1].hostname, "localhost1") == 0); + test_true(server_pool[1].port == 11210); + + test_true(strcmp(server_pool[2].hostname, "localhost2") == 0); + test_true(server_pool[2].port == 11211); + + dt= memcached_behavior_get_distribution(memc); + hash_type= memcached_behavior_get_key_hash(memc); + test_true(dt == MEMCACHED_DISTRIBUTION_VIRTUAL_BUCKET); + test_true(hash_type == MEMCACHED_HASH_CRC); + + /* verify the standard ketama set. */ + for (expect_t *ptr= basic_keys; not libtest_string_is_null(ptr->key); ptr++) + { + uint32_t server_idx = memcached_generate_hash(memc, ptr->key.c_str, ptr->key.size); + + char buffer[1024]; + snprintf(buffer, sizeof(buffer), "%.*s:%lu Got/Expected %u == %u", (int)ptr->key.size, ptr->key.c_str, (unsigned long)ptr->key.size, server_idx, ptr->server_id); + test_true_got(server_idx == ptr->server_id, buffer); + } + + memcached_server_list_free(server_pool); + memcached_free(memc); + + return TEST_SUCCESS; +} diff --git a/tests/virtual_buckets.h b/tests/virtual_buckets.h new file mode 100644 index 00000000..054ec492 --- /dev/null +++ b/tests/virtual_buckets.h @@ -0,0 +1,53 @@ +/* vim:expandtab:shiftwidth=2:tabstop=2:smarttab: + * + * Libmemcached Client and Server + * + * Copyright (C) 2011 Data Differential, http://datadifferential.com/ + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * + * * The names of its contributors may not be used to endorse or + * promote products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#pragma once + +#include + +struct memcached_st; + +#ifdef __cplusplus +extern "C" { +#endif + +LIBTEST_INTERNAL_API +test_return_t virtual_back_map(memcached_st *); + +#ifdef __cplusplus +} +#endif -- 2.30.2