From: Brian Aker Date: Wed, 31 Oct 2007 17:20:28 +0000 (-0700) Subject: More Hashing methods X-Git-Tag: 0.8~7 X-Git-Url: https://git.m6w6.name/?a=commitdiff_plain;h=1207354f3bcebf2bb489ec0e56152918d56be81e;p=m6w6%2Flibmemcached More Hashing methods --- diff --git a/ChangeLog b/ChangeLog index 91638fcc..df62657e 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,6 +1,7 @@ 0.8 * Adding support for CRC hash method * Adding support for UNIX sockets + * Added additional HASHing methods of FNV1_64,FNV1A_64, FNV1_32, FNV1A_32 0.7 Tue Oct 30 09:24:05 PDT 2007 * Poved to poll() from select() diff --git a/include/memcached.h b/include/memcached.h index 7a9f3e56..a859b221 100644 --- a/include/memcached.h +++ b/include/memcached.h @@ -76,6 +76,11 @@ typedef enum { MEMCACHED_HASH_DEFAULT= 0, MEMCACHED_HASH_MD5, MEMCACHED_HASH_CRC, + MEMCACHED_HASH_FNV1_64, + MEMCACHED_HASH_FNV1A_64, + MEMCACHED_HASH_FNV1_32, + MEMCACHED_HASH_FNV1A_32, + MEMCACHED_HASH_KETAMA, } memcached_hash; typedef enum { diff --git a/lib/crc.c b/lib/crc.c index ec7e0d44..ed22adc4 100644 --- a/lib/crc.c +++ b/lib/crc.c @@ -74,14 +74,14 @@ static const uint32_t crc32tab[256] = { }; -uint32_t hash_crc32(const char *data, size_t data_len) +uint32_t hash_crc32(const char *key, size_t key_length) { uint32_t x; uint32_t crc; - crc = ~0; + crc= ~0; - for (x = 0; x < data_len; x++) - crc = (crc >> 8) ^ crc32tab[(crc ^ (data[x])) & 0xff]; + for (x= 0; x < key_length; x++) + crc= (crc >> 8) ^ crc32tab[(crc ^ (key[x])) & 0xff]; return ~crc; } diff --git a/lib/memcached_hash.c b/lib/memcached_hash.c index 04319189..0d83cb75 100644 --- a/lib/memcached_hash.c +++ b/lib/memcached_hash.c @@ -11,10 +11,12 @@ static uint32_t FNV_32_PRIME= 16777619; static unsigned int internal_generate_hash(char *key, size_t key_length); static uint32_t internal_generate_md5(char *key, size_t key_length); static uint32_t internal_generate_md5(char *key, size_t key_length); +static uint32_t internal_generate_ketama_md5(char *key, size_t key_length); unsigned int memcached_generate_hash(memcached_st *ptr, char *key, size_t key_length) { uint32_t hash; + unsigned int x; switch (ptr->hash) { @@ -25,10 +27,58 @@ unsigned int memcached_generate_hash(memcached_st *ptr, char *key, size_t key_le hash= internal_generate_md5(key, key_length); break; case MEMCACHED_HASH_CRC: - hash= hash_crc32(key, key_length); + hash= ((hash_crc32(key, key_length) >> 16) & 0x7fff); break; + /* FNV hash'es lifted from Dustin Sallings work */ + case MEMCACHED_HASH_FNV1_64: + { + /* Thanks to pierre@demartines.com for the pointer */ + hash = FNV_64_INIT; + for (x= 0; x < key_length; x++) + { + hash *= FNV_64_PRIME; + hash ^= key[x]; + } + } + break; + case MEMCACHED_HASH_FNV1A_64: + { + hash= FNV_64_INIT; + for (x= 0; x < key_length; x++) + { + hash ^= key[x]; + hash *= FNV_64_PRIME; + } + } + break; + case MEMCACHED_HASH_FNV1_32: + { + hash= FNV_32_INIT; + for (x= 0; x < key_length; x++) + { + hash *= FNV_32_PRIME; + hash ^= key[x]; + } + } + break; + case MEMCACHED_HASH_FNV1A_32: + { + hash= FNV_32_INIT; + for (x= 0; x < key_length; x++) + { + hash ^= key[x]; + hash *= FNV_32_PRIME; + } + } + break; + case MEMCACHED_HASH_KETAMA: + { + hash= internal_generate_ketama_md5(key, key_length); + break; + } } + WATCHPOINT_ASSERT(hash); if (ptr->flags & MEM_USE_KETAMA) { WATCHPOINT_ASSERT(0); @@ -67,3 +117,15 @@ static uint32_t internal_generate_md5(char *key, size_t key_length) | ( results[1] << 8 ) | results[0] ); } + +static uint32_t internal_generate_ketama_md5(char *key, size_t key_length) +{ + unsigned char results[16]; + + md5_signature((unsigned char*)key, (unsigned int)key_length, results); + + return ((uint32_t) (results[3] & 0xFF) << 24) + | ((uint32_t) (results[2] & 0xFF) << 16) + | ((uint32_t) (results[1] & 0xFF) << 8) + | (results[0] & 0xFF); +} diff --git a/tests/output.res b/tests/output.res index 23ab2daf..f9dcc0fc 100644 --- a/tests/output.res +++ b/tests/output.res @@ -743,3 +743,533 @@ Found key bytes_read Found key bytes_written Found key limit_maxbytes Found key threads +Error 0 -> SUCCESS +Error 1 -> FAILURE +Error 2 -> HOSTNAME LOOKUP FAILURE +Error 3 -> CONNECTION FAILURE +Error 4 -> CONNECTION BIND FAILURE +Error 5 -> WRITE FAILURE +Error 6 -> READ FAILURE +Error 7 -> UNKNOWN READ FAILURE +Error 8 -> PROTOCOL ERROR +Error 9 -> CLIENT ERROR +Error 10 -> SERVER ERROR +Error 11 -> CONNECTION SOCKET CREATE FAILURE +Error 12 -> CONNECTION DATA EXISTS +Error 13 -> CONNECTION DATA DOES NOT EXIST +Error 14 -> NOT STORED +Error 15 -> STORED +Error 16 -> NOT FOUND +Error 17 -> MEMORY ALLOCATION FAILURE +Error 18 -> PARTIAL READ +Error 19 -> SOME ERRORS WERE REPORTED +Error 20 -> NO SERVERS DEFINED +Error 21 -> SERVER END +Error 22 -> SERVER DELETE +Error 23 -> SERVER VALUE +Error 24 -> STAT VALUE +Error 25 -> SYSTEM ERROR +Error 26 -> COULD NOT OPEN UNIX SOCKET +Error 27 -> ACTION NOT SUPPORTED +Found key pid +Found key uptime +Found key time +Found key version +Found key pointer_size +Found key rusage_user +Found key rusage_system +Found key rusage_user_seconds +Found key rusage_user_microseconds +Found key rusage_system_seconds +Found key rusage_system_microseconds +Found key curr_items +Found key total_items +Found key bytes +Found key curr_connections +Found key total_connections +Found key connection_structures +Found key cmd_get +Found key cmd_set +Found key get_hits +Found key get_misses +Found key evictions +Found key bytes_read +Found key bytes_written +Found key limit_maxbytes +Found key threads +Found key pid +Found key uptime +Found key time +Found key version +Found key pointer_size +Found key rusage_user +Found key rusage_system +Found key rusage_user_seconds +Found key rusage_user_microseconds +Found key rusage_system_seconds +Found key rusage_system_microseconds +Found key curr_items +Found key total_items +Found key bytes +Found key curr_connections +Found key total_connections +Found key connection_structures +Found key cmd_get +Found key cmd_set +Found key get_hits +Found key get_misses +Found key evictions +Found key bytes_read +Found key bytes_written +Found key limit_maxbytes +Found key threads +Found key pid +Found key uptime +Found key time +Found key version +Found key pointer_size +Found key rusage_user +Found key rusage_system +Found key rusage_user_seconds +Found key rusage_user_microseconds +Found key rusage_system_seconds +Found key rusage_system_microseconds +Found key curr_items +Found key total_items +Found key bytes +Found key curr_connections +Found key total_connections +Found key connection_structures +Found key cmd_get +Found key cmd_set +Found key get_hits +Found key get_misses +Found key evictions +Found key bytes_read +Found key bytes_written +Found key limit_maxbytes +Found key threads +Error 0 -> SUCCESS +Error 1 -> FAILURE +Error 2 -> HOSTNAME LOOKUP FAILURE +Error 3 -> CONNECTION FAILURE +Error 4 -> CONNECTION BIND FAILURE +Error 5 -> WRITE FAILURE +Error 6 -> READ FAILURE +Error 7 -> UNKNOWN READ FAILURE +Error 8 -> PROTOCOL ERROR +Error 9 -> CLIENT ERROR +Error 10 -> SERVER ERROR +Error 11 -> CONNECTION SOCKET CREATE FAILURE +Error 12 -> CONNECTION DATA EXISTS +Error 13 -> CONNECTION DATA DOES NOT EXIST +Error 14 -> NOT STORED +Error 15 -> STORED +Error 16 -> NOT FOUND +Error 17 -> MEMORY ALLOCATION FAILURE +Error 18 -> PARTIAL READ +Error 19 -> SOME ERRORS WERE REPORTED +Error 20 -> NO SERVERS DEFINED +Error 21 -> SERVER END +Error 22 -> SERVER DELETE +Error 23 -> SERVER VALUE +Error 24 -> STAT VALUE +Error 25 -> SYSTEM ERROR +Error 26 -> COULD NOT OPEN UNIX SOCKET +Error 27 -> ACTION NOT SUPPORTED +Found key pid +Found key uptime +Found key time +Found key version +Found key pointer_size +Found key rusage_user +Found key rusage_system +Found key rusage_user_seconds +Found key rusage_user_microseconds +Found key rusage_system_seconds +Found key rusage_system_microseconds +Found key curr_items +Found key total_items +Found key bytes +Found key curr_connections +Found key total_connections +Found key connection_structures +Found key cmd_get +Found key cmd_set +Found key get_hits +Found key get_misses +Found key evictions +Found key bytes_read +Found key bytes_written +Found key limit_maxbytes +Found key threads +Found key pid +Found key uptime +Found key time +Found key version +Found key pointer_size +Found key rusage_user +Found key rusage_system +Found key rusage_user_seconds +Found key rusage_user_microseconds +Found key rusage_system_seconds +Found key rusage_system_microseconds +Found key curr_items +Found key total_items +Found key bytes +Found key curr_connections +Found key total_connections +Found key connection_structures +Found key cmd_get +Found key cmd_set +Found key get_hits +Found key get_misses +Found key evictions +Found key bytes_read +Found key bytes_written +Found key limit_maxbytes +Found key threads +Found key pid +Found key uptime +Found key time +Found key version +Found key pointer_size +Found key rusage_user +Found key rusage_system +Found key rusage_user_seconds +Found key rusage_user_microseconds +Found key rusage_system_seconds +Found key rusage_system_microseconds +Found key curr_items +Found key total_items +Found key bytes +Found key curr_connections +Found key total_connections +Found key connection_structures +Found key cmd_get +Found key cmd_set +Found key get_hits +Found key get_misses +Found key evictions +Found key bytes_read +Found key bytes_written +Found key limit_maxbytes +Found key threads +Error 0 -> SUCCESS +Error 1 -> FAILURE +Error 2 -> HOSTNAME LOOKUP FAILURE +Error 3 -> CONNECTION FAILURE +Error 4 -> CONNECTION BIND FAILURE +Error 5 -> WRITE FAILURE +Error 6 -> READ FAILURE +Error 7 -> UNKNOWN READ FAILURE +Error 8 -> PROTOCOL ERROR +Error 9 -> CLIENT ERROR +Error 10 -> SERVER ERROR +Error 11 -> CONNECTION SOCKET CREATE FAILURE +Error 12 -> CONNECTION DATA EXISTS +Error 13 -> CONNECTION DATA DOES NOT EXIST +Error 14 -> NOT STORED +Error 15 -> STORED +Error 16 -> NOT FOUND +Error 17 -> MEMORY ALLOCATION FAILURE +Error 18 -> PARTIAL READ +Error 19 -> SOME ERRORS WERE REPORTED +Error 20 -> NO SERVERS DEFINED +Error 21 -> SERVER END +Error 22 -> SERVER DELETE +Error 23 -> SERVER VALUE +Error 24 -> STAT VALUE +Error 25 -> SYSTEM ERROR +Error 26 -> COULD NOT OPEN UNIX SOCKET +Error 27 -> ACTION NOT SUPPORTED +Found key pid +Found key uptime +Found key time +Found key version +Found key pointer_size +Found key rusage_user +Found key rusage_system +Found key rusage_user_seconds +Found key rusage_user_microseconds +Found key rusage_system_seconds +Found key rusage_system_microseconds +Found key curr_items +Found key total_items +Found key bytes +Found key curr_connections +Found key total_connections +Found key connection_structures +Found key cmd_get +Found key cmd_set +Found key get_hits +Found key get_misses +Found key evictions +Found key bytes_read +Found key bytes_written +Found key limit_maxbytes +Found key threads +Found key pid +Found key uptime +Found key time +Found key version +Found key pointer_size +Found key rusage_user +Found key rusage_system +Found key rusage_user_seconds +Found key rusage_user_microseconds +Found key rusage_system_seconds +Found key rusage_system_microseconds +Found key curr_items +Found key total_items +Found key bytes +Found key curr_connections +Found key total_connections +Found key connection_structures +Found key cmd_get +Found key cmd_set +Found key get_hits +Found key get_misses +Found key evictions +Found key bytes_read +Found key bytes_written +Found key limit_maxbytes +Found key threads +Found key pid +Found key uptime +Found key time +Found key version +Found key pointer_size +Found key rusage_user +Found key rusage_system +Found key rusage_user_seconds +Found key rusage_user_microseconds +Found key rusage_system_seconds +Found key rusage_system_microseconds +Found key curr_items +Found key total_items +Found key bytes +Found key curr_connections +Found key total_connections +Found key connection_structures +Found key cmd_get +Found key cmd_set +Found key get_hits +Found key get_misses +Found key evictions +Found key bytes_read +Found key bytes_written +Found key limit_maxbytes +Found key threads +Error 0 -> SUCCESS +Error 1 -> FAILURE +Error 2 -> HOSTNAME LOOKUP FAILURE +Error 3 -> CONNECTION FAILURE +Error 4 -> CONNECTION BIND FAILURE +Error 5 -> WRITE FAILURE +Error 6 -> READ FAILURE +Error 7 -> UNKNOWN READ FAILURE +Error 8 -> PROTOCOL ERROR +Error 9 -> CLIENT ERROR +Error 10 -> SERVER ERROR +Error 11 -> CONNECTION SOCKET CREATE FAILURE +Error 12 -> CONNECTION DATA EXISTS +Error 13 -> CONNECTION DATA DOES NOT EXIST +Error 14 -> NOT STORED +Error 15 -> STORED +Error 16 -> NOT FOUND +Error 17 -> MEMORY ALLOCATION FAILURE +Error 18 -> PARTIAL READ +Error 19 -> SOME ERRORS WERE REPORTED +Error 20 -> NO SERVERS DEFINED +Error 21 -> SERVER END +Error 22 -> SERVER DELETE +Error 23 -> SERVER VALUE +Error 24 -> STAT VALUE +Error 25 -> SYSTEM ERROR +Error 26 -> COULD NOT OPEN UNIX SOCKET +Error 27 -> ACTION NOT SUPPORTED +Found key pid +Found key uptime +Found key time +Found key version +Found key pointer_size +Found key rusage_user +Found key rusage_system +Found key rusage_user_seconds +Found key rusage_user_microseconds +Found key rusage_system_seconds +Found key rusage_system_microseconds +Found key curr_items +Found key total_items +Found key bytes +Found key curr_connections +Found key total_connections +Found key connection_structures +Found key cmd_get +Found key cmd_set +Found key get_hits +Found key get_misses +Found key evictions +Found key bytes_read +Found key bytes_written +Found key limit_maxbytes +Found key threads +Found key pid +Found key uptime +Found key time +Found key version +Found key pointer_size +Found key rusage_user +Found key rusage_system +Found key rusage_user_seconds +Found key rusage_user_microseconds +Found key rusage_system_seconds +Found key rusage_system_microseconds +Found key curr_items +Found key total_items +Found key bytes +Found key curr_connections +Found key total_connections +Found key connection_structures +Found key cmd_get +Found key cmd_set +Found key get_hits +Found key get_misses +Found key evictions +Found key bytes_read +Found key bytes_written +Found key limit_maxbytes +Found key threads +Found key pid +Found key uptime +Found key time +Found key version +Found key pointer_size +Found key rusage_user +Found key rusage_system +Found key rusage_user_seconds +Found key rusage_user_microseconds +Found key rusage_system_seconds +Found key rusage_system_microseconds +Found key curr_items +Found key total_items +Found key bytes +Found key curr_connections +Found key total_connections +Found key connection_structures +Found key cmd_get +Found key cmd_set +Found key get_hits +Found key get_misses +Found key evictions +Found key bytes_read +Found key bytes_written +Found key limit_maxbytes +Found key threads +Error 0 -> SUCCESS +Error 1 -> FAILURE +Error 2 -> HOSTNAME LOOKUP FAILURE +Error 3 -> CONNECTION FAILURE +Error 4 -> CONNECTION BIND FAILURE +Error 5 -> WRITE FAILURE +Error 6 -> READ FAILURE +Error 7 -> UNKNOWN READ FAILURE +Error 8 -> PROTOCOL ERROR +Error 9 -> CLIENT ERROR +Error 10 -> SERVER ERROR +Error 11 -> CONNECTION SOCKET CREATE FAILURE +Error 12 -> CONNECTION DATA EXISTS +Error 13 -> CONNECTION DATA DOES NOT EXIST +Error 14 -> NOT STORED +Error 15 -> STORED +Error 16 -> NOT FOUND +Error 17 -> MEMORY ALLOCATION FAILURE +Error 18 -> PARTIAL READ +Error 19 -> SOME ERRORS WERE REPORTED +Error 20 -> NO SERVERS DEFINED +Error 21 -> SERVER END +Error 22 -> SERVER DELETE +Error 23 -> SERVER VALUE +Error 24 -> STAT VALUE +Error 25 -> SYSTEM ERROR +Error 26 -> COULD NOT OPEN UNIX SOCKET +Error 27 -> ACTION NOT SUPPORTED +Found key pid +Found key uptime +Found key time +Found key version +Found key pointer_size +Found key rusage_user +Found key rusage_system +Found key rusage_user_seconds +Found key rusage_user_microseconds +Found key rusage_system_seconds +Found key rusage_system_microseconds +Found key curr_items +Found key total_items +Found key bytes +Found key curr_connections +Found key total_connections +Found key connection_structures +Found key cmd_get +Found key cmd_set +Found key get_hits +Found key get_misses +Found key evictions +Found key bytes_read +Found key bytes_written +Found key limit_maxbytes +Found key threads +Found key pid +Found key uptime +Found key time +Found key version +Found key pointer_size +Found key rusage_user +Found key rusage_system +Found key rusage_user_seconds +Found key rusage_user_microseconds +Found key rusage_system_seconds +Found key rusage_system_microseconds +Found key curr_items +Found key total_items +Found key bytes +Found key curr_connections +Found key total_connections +Found key connection_structures +Found key cmd_get +Found key cmd_set +Found key get_hits +Found key get_misses +Found key evictions +Found key bytes_read +Found key bytes_written +Found key limit_maxbytes +Found key threads +Found key pid +Found key uptime +Found key time +Found key version +Found key pointer_size +Found key rusage_user +Found key rusage_system +Found key rusage_user_seconds +Found key rusage_user_microseconds +Found key rusage_system_seconds +Found key rusage_system_microseconds +Found key curr_items +Found key total_items +Found key bytes +Found key curr_connections +Found key total_connections +Found key connection_structures +Found key cmd_get +Found key cmd_set +Found key get_hits +Found key get_misses +Found key evictions +Found key bytes_read +Found key bytes_written +Found key limit_maxbytes +Found key threads diff --git a/tests/test.c b/tests/test.c index ef6afb74..7b8c462e 100644 --- a/tests/test.c +++ b/tests/test.c @@ -833,6 +833,46 @@ memcached_return pre_crc(memcached_st *memc) return MEMCACHED_SUCCESS; } +memcached_return pre_hash_fnv1_64(memcached_st *memc) +{ + memcached_hash value= MEMCACHED_HASH_FNV1_64; + memcached_behavior_set(memc, MEMCACHED_BEHAVIOR_HASH, &value); + + return MEMCACHED_SUCCESS; +} + +memcached_return pre_hash_fnv1a_64(memcached_st *memc) +{ + memcached_hash value= MEMCACHED_HASH_FNV1A_64; + memcached_behavior_set(memc, MEMCACHED_BEHAVIOR_HASH, &value); + + return MEMCACHED_SUCCESS; +} + +memcached_return pre_hash_fnv1_32(memcached_st *memc) +{ + memcached_hash value= MEMCACHED_HASH_FNV1_32; + memcached_behavior_set(memc, MEMCACHED_BEHAVIOR_HASH, &value); + + return MEMCACHED_SUCCESS; +} + +memcached_return pre_hash_fnv1a_32(memcached_st *memc) +{ + memcached_hash value= MEMCACHED_HASH_FNV1A_32; + memcached_behavior_set(memc, MEMCACHED_BEHAVIOR_HASH, &value); + + return MEMCACHED_SUCCESS; +} + +memcached_return pre_hash_ketama(memcached_st *memc) +{ + memcached_hash value= MEMCACHED_HASH_KETAMA; + memcached_behavior_set(memc, MEMCACHED_BEHAVIOR_HASH, &value); + + return MEMCACHED_SUCCESS; +} + memcached_return pre_unix_socket(memcached_st *memc) { memcached_return rc; @@ -960,6 +1000,11 @@ int main(int argc, char *argv[]) {"nodelay", pre_nodelay, 0, tests}, {"md5", pre_md5, 0, tests}, {"crc", pre_crc, 0, tests}, + {"fnv1_64", pre_hash_fnv1_64, 0, tests}, + {"fnv1a_64", pre_hash_fnv1a_64, 0, tests}, + {"fnv1_32", pre_hash_fnv1_32, 0, tests}, + {"fnv1a_32", pre_hash_fnv1a_32, 0, tests}, + {"ketama", pre_hash_ketama, 0, tests}, {"unix_socket", pre_unix_socket, 0, tests}, {"unix_socket_nodelay", pre_nodelay, 0, tests}, {"string", 0, 0, string_tests},