More Hashing methods
authorBrian Aker <brian@tangent.org>
Wed, 31 Oct 2007 17:20:28 +0000 (10:20 -0700)
committerBrian Aker <brian@tangent.org>
Wed, 31 Oct 2007 17:20:28 +0000 (10:20 -0700)
ChangeLog
include/memcached.h
lib/crc.c
lib/memcached_hash.c
tests/output.res
tests/test.c

index 91638fccae72f084961fd9ebdafef5cf03bfd0c5..df62657eed75c2d5fe5a4aca709bf67a17e7fdcb 100644 (file)
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,7 @@
 0.8
   * Adding support for CRC hash method 
   * Adding support for UNIX sockets
+  * Added additional HASHing methods of FNV1_64,FNV1A_64, FNV1_32, FNV1A_32
 
 0.7 Tue Oct 30 09:24:05 PDT 2007
   * Poved to poll() from select()
index 7a9f3e56dcc79d5cea3a59bcba5dc2256481bd51..a859b22117d4f2ae4d7484ed5ded2902521aae88 100644 (file)
@@ -76,6 +76,11 @@ typedef enum {
   MEMCACHED_HASH_DEFAULT= 0,
   MEMCACHED_HASH_MD5,
   MEMCACHED_HASH_CRC,
+  MEMCACHED_HASH_FNV1_64,
+  MEMCACHED_HASH_FNV1A_64,
+  MEMCACHED_HASH_FNV1_32,
+  MEMCACHED_HASH_FNV1A_32,
+  MEMCACHED_HASH_KETAMA,
 } memcached_hash;
 
 typedef enum {
index ec7e0d44971dce9fe3b8e807fc1ffa21a9077fba..ed22adc46dd868c97120384db7a19abeb31ec714 100644 (file)
--- a/lib/crc.c
+++ b/lib/crc.c
@@ -74,14 +74,14 @@ static const uint32_t crc32tab[256] = {
 };
 
 
-uint32_t hash_crc32(const char *data, size_t data_len)
+uint32_t hash_crc32(const char *key, size_t key_length)
 {
   uint32_t x;
   uint32_t crc;
-  crc = ~0;
+  crc= ~0;
 
-  for (x = 0; x < data_len; x++)
-    crc = (crc >> 8) ^ crc32tab[(crc ^ (data[x])) & 0xff];
+  for (x= 0; x < key_length; x++)
+    crc= (crc >> 8) ^ crc32tab[(crc ^ (key[x])) & 0xff];
 
   return ~crc;
 }
index 04319189a69b7d9c94e5be72a0e8d1cbb45a8027..0d83cb75a7d0b0591b9ae4f2cbb8c0ccbd960862 100644 (file)
@@ -11,10 +11,12 @@ static uint32_t FNV_32_PRIME= 16777619;
 static unsigned int internal_generate_hash(char *key, size_t key_length);
 static uint32_t internal_generate_md5(char *key, size_t key_length);
 static uint32_t internal_generate_md5(char *key, size_t key_length);
+static uint32_t internal_generate_ketama_md5(char *key, size_t key_length);
 
 unsigned int memcached_generate_hash(memcached_st *ptr, char *key, size_t key_length)
 {
   uint32_t hash;
+  unsigned int x;
 
   switch (ptr->hash)
   {
@@ -25,10 +27,58 @@ unsigned int memcached_generate_hash(memcached_st *ptr, char *key, size_t key_le
     hash= internal_generate_md5(key, key_length);
     break;
   case MEMCACHED_HASH_CRC:
-    hash= hash_crc32(key, key_length);
+    hash= ((hash_crc32(key, key_length) >> 16) & 0x7fff);
     break;
+    /* FNV hash'es lifted from Dustin Sallings work */
+  case MEMCACHED_HASH_FNV1_64: 
+    {
+      /* Thanks to pierre@demartines.com for the pointer */
+      hash = FNV_64_INIT;
+      for (x= 0; x < key_length; x++) 
+      {
+        hash *= FNV_64_PRIME;
+        hash ^= key[x];
+      }
+    }
+    break;
+  case MEMCACHED_HASH_FNV1A_64: 
+    {
+      hash= FNV_64_INIT;
+      for (x= 0; x < key_length; x++) 
+      {
+        hash ^= key[x];
+        hash *= FNV_64_PRIME;
+      }
+    }
+    break;
+  case MEMCACHED_HASH_FNV1_32: 
+    {
+      hash= FNV_32_INIT;
+      for (x= 0; x < key_length; x++) 
+      {
+        hash *= FNV_32_PRIME;
+        hash ^= key[x];
+      }
+    }
+    break;
+  case MEMCACHED_HASH_FNV1A_32: 
+    {
+      hash= FNV_32_INIT;
+      for (x= 0; x < key_length; x++) 
+      {
+        hash ^= key[x];
+        hash *= FNV_32_PRIME;
+      }
+    }
+    break;
+    case MEMCACHED_HASH_KETAMA: 
+    {
+      hash= internal_generate_ketama_md5(key, key_length);
+      break;
+    }
   }
 
+  WATCHPOINT_ASSERT(hash);
   if (ptr->flags & MEM_USE_KETAMA)
   {
     WATCHPOINT_ASSERT(0);
@@ -67,3 +117,15 @@ static uint32_t internal_generate_md5(char *key, size_t key_length)
                     | ( results[1] <<  8 )
                     |   results[0] );
 }
+
+static uint32_t internal_generate_ketama_md5(char *key, size_t key_length)
+{
+  unsigned char results[16];
+
+  md5_signature((unsigned char*)key, (unsigned int)key_length, results);
+
+  return ((uint32_t) (results[3] & 0xFF) << 24)
+    | ((uint32_t) (results[2] & 0xFF) << 16)
+    | ((uint32_t) (results[1] & 0xFF) << 8)
+    | (results[0] & 0xFF);
+}
index 23ab2dafc809458ff535cc68ade2ef38a2521ce3..f9dcc0fcfa572a509a25547c9583949259f140be 100644 (file)
@@ -743,3 +743,533 @@ Found key bytes_read
 Found key bytes_written
 Found key limit_maxbytes
 Found key threads
+Error 0 -> SUCCESS
+Error 1 -> FAILURE
+Error 2 -> HOSTNAME LOOKUP FAILURE
+Error 3 -> CONNECTION FAILURE
+Error 4 -> CONNECTION BIND FAILURE
+Error 5 -> WRITE FAILURE
+Error 6 -> READ FAILURE
+Error 7 -> UNKNOWN READ FAILURE
+Error 8 -> PROTOCOL ERROR
+Error 9 -> CLIENT ERROR
+Error 10 -> SERVER ERROR
+Error 11 -> CONNECTION SOCKET CREATE FAILURE
+Error 12 -> CONNECTION DATA EXISTS
+Error 13 -> CONNECTION DATA DOES NOT EXIST
+Error 14 -> NOT STORED
+Error 15 -> STORED
+Error 16 -> NOT FOUND
+Error 17 -> MEMORY ALLOCATION FAILURE
+Error 18 -> PARTIAL READ
+Error 19 -> SOME ERRORS WERE REPORTED
+Error 20 -> NO SERVERS DEFINED
+Error 21 -> SERVER END
+Error 22 -> SERVER DELETE
+Error 23 -> SERVER VALUE
+Error 24 -> STAT VALUE
+Error 25 -> SYSTEM ERROR
+Error 26 -> COULD NOT OPEN UNIX SOCKET
+Error 27 -> ACTION NOT SUPPORTED
+Found key pid
+Found key uptime
+Found key time
+Found key version
+Found key pointer_size
+Found key rusage_user
+Found key rusage_system
+Found key rusage_user_seconds
+Found key rusage_user_microseconds
+Found key rusage_system_seconds
+Found key rusage_system_microseconds
+Found key curr_items
+Found key total_items
+Found key bytes
+Found key curr_connections
+Found key total_connections
+Found key connection_structures
+Found key cmd_get
+Found key cmd_set
+Found key get_hits
+Found key get_misses
+Found key evictions
+Found key bytes_read
+Found key bytes_written
+Found key limit_maxbytes
+Found key threads
+Found key pid
+Found key uptime
+Found key time
+Found key version
+Found key pointer_size
+Found key rusage_user
+Found key rusage_system
+Found key rusage_user_seconds
+Found key rusage_user_microseconds
+Found key rusage_system_seconds
+Found key rusage_system_microseconds
+Found key curr_items
+Found key total_items
+Found key bytes
+Found key curr_connections
+Found key total_connections
+Found key connection_structures
+Found key cmd_get
+Found key cmd_set
+Found key get_hits
+Found key get_misses
+Found key evictions
+Found key bytes_read
+Found key bytes_written
+Found key limit_maxbytes
+Found key threads
+Found key pid
+Found key uptime
+Found key time
+Found key version
+Found key pointer_size
+Found key rusage_user
+Found key rusage_system
+Found key rusage_user_seconds
+Found key rusage_user_microseconds
+Found key rusage_system_seconds
+Found key rusage_system_microseconds
+Found key curr_items
+Found key total_items
+Found key bytes
+Found key curr_connections
+Found key total_connections
+Found key connection_structures
+Found key cmd_get
+Found key cmd_set
+Found key get_hits
+Found key get_misses
+Found key evictions
+Found key bytes_read
+Found key bytes_written
+Found key limit_maxbytes
+Found key threads
+Error 0 -> SUCCESS
+Error 1 -> FAILURE
+Error 2 -> HOSTNAME LOOKUP FAILURE
+Error 3 -> CONNECTION FAILURE
+Error 4 -> CONNECTION BIND FAILURE
+Error 5 -> WRITE FAILURE
+Error 6 -> READ FAILURE
+Error 7 -> UNKNOWN READ FAILURE
+Error 8 -> PROTOCOL ERROR
+Error 9 -> CLIENT ERROR
+Error 10 -> SERVER ERROR
+Error 11 -> CONNECTION SOCKET CREATE FAILURE
+Error 12 -> CONNECTION DATA EXISTS
+Error 13 -> CONNECTION DATA DOES NOT EXIST
+Error 14 -> NOT STORED
+Error 15 -> STORED
+Error 16 -> NOT FOUND
+Error 17 -> MEMORY ALLOCATION FAILURE
+Error 18 -> PARTIAL READ
+Error 19 -> SOME ERRORS WERE REPORTED
+Error 20 -> NO SERVERS DEFINED
+Error 21 -> SERVER END
+Error 22 -> SERVER DELETE
+Error 23 -> SERVER VALUE
+Error 24 -> STAT VALUE
+Error 25 -> SYSTEM ERROR
+Error 26 -> COULD NOT OPEN UNIX SOCKET
+Error 27 -> ACTION NOT SUPPORTED
+Found key pid
+Found key uptime
+Found key time
+Found key version
+Found key pointer_size
+Found key rusage_user
+Found key rusage_system
+Found key rusage_user_seconds
+Found key rusage_user_microseconds
+Found key rusage_system_seconds
+Found key rusage_system_microseconds
+Found key curr_items
+Found key total_items
+Found key bytes
+Found key curr_connections
+Found key total_connections
+Found key connection_structures
+Found key cmd_get
+Found key cmd_set
+Found key get_hits
+Found key get_misses
+Found key evictions
+Found key bytes_read
+Found key bytes_written
+Found key limit_maxbytes
+Found key threads
+Found key pid
+Found key uptime
+Found key time
+Found key version
+Found key pointer_size
+Found key rusage_user
+Found key rusage_system
+Found key rusage_user_seconds
+Found key rusage_user_microseconds
+Found key rusage_system_seconds
+Found key rusage_system_microseconds
+Found key curr_items
+Found key total_items
+Found key bytes
+Found key curr_connections
+Found key total_connections
+Found key connection_structures
+Found key cmd_get
+Found key cmd_set
+Found key get_hits
+Found key get_misses
+Found key evictions
+Found key bytes_read
+Found key bytes_written
+Found key limit_maxbytes
+Found key threads
+Found key pid
+Found key uptime
+Found key time
+Found key version
+Found key pointer_size
+Found key rusage_user
+Found key rusage_system
+Found key rusage_user_seconds
+Found key rusage_user_microseconds
+Found key rusage_system_seconds
+Found key rusage_system_microseconds
+Found key curr_items
+Found key total_items
+Found key bytes
+Found key curr_connections
+Found key total_connections
+Found key connection_structures
+Found key cmd_get
+Found key cmd_set
+Found key get_hits
+Found key get_misses
+Found key evictions
+Found key bytes_read
+Found key bytes_written
+Found key limit_maxbytes
+Found key threads
+Error 0 -> SUCCESS
+Error 1 -> FAILURE
+Error 2 -> HOSTNAME LOOKUP FAILURE
+Error 3 -> CONNECTION FAILURE
+Error 4 -> CONNECTION BIND FAILURE
+Error 5 -> WRITE FAILURE
+Error 6 -> READ FAILURE
+Error 7 -> UNKNOWN READ FAILURE
+Error 8 -> PROTOCOL ERROR
+Error 9 -> CLIENT ERROR
+Error 10 -> SERVER ERROR
+Error 11 -> CONNECTION SOCKET CREATE FAILURE
+Error 12 -> CONNECTION DATA EXISTS
+Error 13 -> CONNECTION DATA DOES NOT EXIST
+Error 14 -> NOT STORED
+Error 15 -> STORED
+Error 16 -> NOT FOUND
+Error 17 -> MEMORY ALLOCATION FAILURE
+Error 18 -> PARTIAL READ
+Error 19 -> SOME ERRORS WERE REPORTED
+Error 20 -> NO SERVERS DEFINED
+Error 21 -> SERVER END
+Error 22 -> SERVER DELETE
+Error 23 -> SERVER VALUE
+Error 24 -> STAT VALUE
+Error 25 -> SYSTEM ERROR
+Error 26 -> COULD NOT OPEN UNIX SOCKET
+Error 27 -> ACTION NOT SUPPORTED
+Found key pid
+Found key uptime
+Found key time
+Found key version
+Found key pointer_size
+Found key rusage_user
+Found key rusage_system
+Found key rusage_user_seconds
+Found key rusage_user_microseconds
+Found key rusage_system_seconds
+Found key rusage_system_microseconds
+Found key curr_items
+Found key total_items
+Found key bytes
+Found key curr_connections
+Found key total_connections
+Found key connection_structures
+Found key cmd_get
+Found key cmd_set
+Found key get_hits
+Found key get_misses
+Found key evictions
+Found key bytes_read
+Found key bytes_written
+Found key limit_maxbytes
+Found key threads
+Found key pid
+Found key uptime
+Found key time
+Found key version
+Found key pointer_size
+Found key rusage_user
+Found key rusage_system
+Found key rusage_user_seconds
+Found key rusage_user_microseconds
+Found key rusage_system_seconds
+Found key rusage_system_microseconds
+Found key curr_items
+Found key total_items
+Found key bytes
+Found key curr_connections
+Found key total_connections
+Found key connection_structures
+Found key cmd_get
+Found key cmd_set
+Found key get_hits
+Found key get_misses
+Found key evictions
+Found key bytes_read
+Found key bytes_written
+Found key limit_maxbytes
+Found key threads
+Found key pid
+Found key uptime
+Found key time
+Found key version
+Found key pointer_size
+Found key rusage_user
+Found key rusage_system
+Found key rusage_user_seconds
+Found key rusage_user_microseconds
+Found key rusage_system_seconds
+Found key rusage_system_microseconds
+Found key curr_items
+Found key total_items
+Found key bytes
+Found key curr_connections
+Found key total_connections
+Found key connection_structures
+Found key cmd_get
+Found key cmd_set
+Found key get_hits
+Found key get_misses
+Found key evictions
+Found key bytes_read
+Found key bytes_written
+Found key limit_maxbytes
+Found key threads
+Error 0 -> SUCCESS
+Error 1 -> FAILURE
+Error 2 -> HOSTNAME LOOKUP FAILURE
+Error 3 -> CONNECTION FAILURE
+Error 4 -> CONNECTION BIND FAILURE
+Error 5 -> WRITE FAILURE
+Error 6 -> READ FAILURE
+Error 7 -> UNKNOWN READ FAILURE
+Error 8 -> PROTOCOL ERROR
+Error 9 -> CLIENT ERROR
+Error 10 -> SERVER ERROR
+Error 11 -> CONNECTION SOCKET CREATE FAILURE
+Error 12 -> CONNECTION DATA EXISTS
+Error 13 -> CONNECTION DATA DOES NOT EXIST
+Error 14 -> NOT STORED
+Error 15 -> STORED
+Error 16 -> NOT FOUND
+Error 17 -> MEMORY ALLOCATION FAILURE
+Error 18 -> PARTIAL READ
+Error 19 -> SOME ERRORS WERE REPORTED
+Error 20 -> NO SERVERS DEFINED
+Error 21 -> SERVER END
+Error 22 -> SERVER DELETE
+Error 23 -> SERVER VALUE
+Error 24 -> STAT VALUE
+Error 25 -> SYSTEM ERROR
+Error 26 -> COULD NOT OPEN UNIX SOCKET
+Error 27 -> ACTION NOT SUPPORTED
+Found key pid
+Found key uptime
+Found key time
+Found key version
+Found key pointer_size
+Found key rusage_user
+Found key rusage_system
+Found key rusage_user_seconds
+Found key rusage_user_microseconds
+Found key rusage_system_seconds
+Found key rusage_system_microseconds
+Found key curr_items
+Found key total_items
+Found key bytes
+Found key curr_connections
+Found key total_connections
+Found key connection_structures
+Found key cmd_get
+Found key cmd_set
+Found key get_hits
+Found key get_misses
+Found key evictions
+Found key bytes_read
+Found key bytes_written
+Found key limit_maxbytes
+Found key threads
+Found key pid
+Found key uptime
+Found key time
+Found key version
+Found key pointer_size
+Found key rusage_user
+Found key rusage_system
+Found key rusage_user_seconds
+Found key rusage_user_microseconds
+Found key rusage_system_seconds
+Found key rusage_system_microseconds
+Found key curr_items
+Found key total_items
+Found key bytes
+Found key curr_connections
+Found key total_connections
+Found key connection_structures
+Found key cmd_get
+Found key cmd_set
+Found key get_hits
+Found key get_misses
+Found key evictions
+Found key bytes_read
+Found key bytes_written
+Found key limit_maxbytes
+Found key threads
+Found key pid
+Found key uptime
+Found key time
+Found key version
+Found key pointer_size
+Found key rusage_user
+Found key rusage_system
+Found key rusage_user_seconds
+Found key rusage_user_microseconds
+Found key rusage_system_seconds
+Found key rusage_system_microseconds
+Found key curr_items
+Found key total_items
+Found key bytes
+Found key curr_connections
+Found key total_connections
+Found key connection_structures
+Found key cmd_get
+Found key cmd_set
+Found key get_hits
+Found key get_misses
+Found key evictions
+Found key bytes_read
+Found key bytes_written
+Found key limit_maxbytes
+Found key threads
+Error 0 -> SUCCESS
+Error 1 -> FAILURE
+Error 2 -> HOSTNAME LOOKUP FAILURE
+Error 3 -> CONNECTION FAILURE
+Error 4 -> CONNECTION BIND FAILURE
+Error 5 -> WRITE FAILURE
+Error 6 -> READ FAILURE
+Error 7 -> UNKNOWN READ FAILURE
+Error 8 -> PROTOCOL ERROR
+Error 9 -> CLIENT ERROR
+Error 10 -> SERVER ERROR
+Error 11 -> CONNECTION SOCKET CREATE FAILURE
+Error 12 -> CONNECTION DATA EXISTS
+Error 13 -> CONNECTION DATA DOES NOT EXIST
+Error 14 -> NOT STORED
+Error 15 -> STORED
+Error 16 -> NOT FOUND
+Error 17 -> MEMORY ALLOCATION FAILURE
+Error 18 -> PARTIAL READ
+Error 19 -> SOME ERRORS WERE REPORTED
+Error 20 -> NO SERVERS DEFINED
+Error 21 -> SERVER END
+Error 22 -> SERVER DELETE
+Error 23 -> SERVER VALUE
+Error 24 -> STAT VALUE
+Error 25 -> SYSTEM ERROR
+Error 26 -> COULD NOT OPEN UNIX SOCKET
+Error 27 -> ACTION NOT SUPPORTED
+Found key pid
+Found key uptime
+Found key time
+Found key version
+Found key pointer_size
+Found key rusage_user
+Found key rusage_system
+Found key rusage_user_seconds
+Found key rusage_user_microseconds
+Found key rusage_system_seconds
+Found key rusage_system_microseconds
+Found key curr_items
+Found key total_items
+Found key bytes
+Found key curr_connections
+Found key total_connections
+Found key connection_structures
+Found key cmd_get
+Found key cmd_set
+Found key get_hits
+Found key get_misses
+Found key evictions
+Found key bytes_read
+Found key bytes_written
+Found key limit_maxbytes
+Found key threads
+Found key pid
+Found key uptime
+Found key time
+Found key version
+Found key pointer_size
+Found key rusage_user
+Found key rusage_system
+Found key rusage_user_seconds
+Found key rusage_user_microseconds
+Found key rusage_system_seconds
+Found key rusage_system_microseconds
+Found key curr_items
+Found key total_items
+Found key bytes
+Found key curr_connections
+Found key total_connections
+Found key connection_structures
+Found key cmd_get
+Found key cmd_set
+Found key get_hits
+Found key get_misses
+Found key evictions
+Found key bytes_read
+Found key bytes_written
+Found key limit_maxbytes
+Found key threads
+Found key pid
+Found key uptime
+Found key time
+Found key version
+Found key pointer_size
+Found key rusage_user
+Found key rusage_system
+Found key rusage_user_seconds
+Found key rusage_user_microseconds
+Found key rusage_system_seconds
+Found key rusage_system_microseconds
+Found key curr_items
+Found key total_items
+Found key bytes
+Found key curr_connections
+Found key total_connections
+Found key connection_structures
+Found key cmd_get
+Found key cmd_set
+Found key get_hits
+Found key get_misses
+Found key evictions
+Found key bytes_read
+Found key bytes_written
+Found key limit_maxbytes
+Found key threads
index ef6afb7460414974089410564e82673ace1800ef..7b8c462e247380f326811caaf6184e7b60fe828e 100644 (file)
@@ -833,6 +833,46 @@ memcached_return pre_crc(memcached_st *memc)
   return MEMCACHED_SUCCESS;
 }
 
+memcached_return pre_hash_fnv1_64(memcached_st *memc)
+{
+  memcached_hash value= MEMCACHED_HASH_FNV1_64;
+  memcached_behavior_set(memc, MEMCACHED_BEHAVIOR_HASH, &value);
+
+  return MEMCACHED_SUCCESS;
+}
+
+memcached_return pre_hash_fnv1a_64(memcached_st *memc)
+{
+  memcached_hash value= MEMCACHED_HASH_FNV1A_64;
+  memcached_behavior_set(memc, MEMCACHED_BEHAVIOR_HASH, &value);
+
+  return MEMCACHED_SUCCESS;
+}
+
+memcached_return pre_hash_fnv1_32(memcached_st *memc)
+{
+  memcached_hash value= MEMCACHED_HASH_FNV1_32;
+  memcached_behavior_set(memc, MEMCACHED_BEHAVIOR_HASH, &value);
+
+  return MEMCACHED_SUCCESS;
+}
+
+memcached_return pre_hash_fnv1a_32(memcached_st *memc)
+{
+  memcached_hash value= MEMCACHED_HASH_FNV1A_32;
+  memcached_behavior_set(memc, MEMCACHED_BEHAVIOR_HASH, &value);
+
+  return MEMCACHED_SUCCESS;
+}
+
+memcached_return pre_hash_ketama(memcached_st *memc)
+{
+  memcached_hash value= MEMCACHED_HASH_KETAMA;
+  memcached_behavior_set(memc, MEMCACHED_BEHAVIOR_HASH, &value);
+
+  return MEMCACHED_SUCCESS;
+}
+
 memcached_return pre_unix_socket(memcached_st *memc)
 {
   memcached_return rc;
@@ -960,6 +1000,11 @@ int main(int argc, char *argv[])
     {"nodelay", pre_nodelay, 0, tests},
     {"md5", pre_md5, 0, tests},
     {"crc", pre_crc, 0, tests},
+    {"fnv1_64", pre_hash_fnv1_64, 0, tests},
+    {"fnv1a_64", pre_hash_fnv1a_64, 0, tests},
+    {"fnv1_32", pre_hash_fnv1_32, 0, tests},
+    {"fnv1a_32", pre_hash_fnv1a_32, 0, tests},
+    {"ketama", pre_hash_ketama, 0, tests},
     {"unix_socket", pre_unix_socket, 0, tests},
     {"unix_socket_nodelay", pre_nodelay, 0, tests},
     {"string", 0, 0, string_tests},