1 /* vim:expandtab:shiftwidth=2:tabstop=2:smarttab:
5 * Copyright (C) 2011 Data Differential, http://datadifferential.com/
6 * Copyright (C) 2006-2009 Brian Aker All rights reserved.
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions are
12 * * Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
15 * * Redistributions in binary form must reproduce the above
16 * copyright notice, this list of conditions and the following disclaimer
17 * in the documentation and/or other materials provided with the
20 * * The names of its contributors may not be used to endorse or
21 * promote products derived from this software without specific prior
24 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
25 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
26 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
27 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
28 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
29 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
30 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
31 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
32 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
33 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
34 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
38 #include "libmemcached/common.h"
42 What happens if no servers exist?
44 char *memcached_get(memcached_st
*ptr
, const char *key
,
48 memcached_return_t
*error
)
50 return memcached_get_by_key(ptr
, NULL
, 0, key
, key_length
, value_length
,
54 static memcached_return_t
memcached_mget_by_key_real(memcached_st
*ptr
,
55 const char *group_key
,
56 size_t group_key_length
,
57 const char * const *keys
,
58 const size_t *key_length
,
59 size_t number_of_keys
,
62 char *memcached_get_by_key(memcached_st
*ptr
,
63 const char *group_key
,
64 size_t group_key_length
,
65 const char *key
, size_t key_length
,
68 memcached_return_t
*error
)
73 memcached_return_t dummy_error
;
75 unlikely (ptr
->flags
.use_udp
)
77 *error
= MEMCACHED_NOT_SUPPORTED
;
82 *error
= memcached_mget_by_key_real(ptr
, group_key
, group_key_length
,
83 (const char * const *)&key
,
84 &key_length
, 1, false);
86 value
= memcached_fetch(ptr
, NULL
, NULL
,
87 value_length
, flags
, error
);
88 /* This is for historical reasons */
89 if (*error
== MEMCACHED_END
)
90 *error
= MEMCACHED_NOTFOUND
;
94 if (ptr
->get_key_failure
&& *error
== MEMCACHED_NOTFOUND
)
97 memcached_result_reset(&ptr
->result
);
98 memcached_return_t rc
= ptr
->get_key_failure(ptr
, key
, key_length
, &ptr
->result
);
100 /* On all failure drop to returning NULL */
101 if (rc
== MEMCACHED_SUCCESS
|| rc
== MEMCACHED_BUFFERED
)
103 if (rc
== MEMCACHED_BUFFERED
)
105 uint64_t latch
; /* We use latch to track the state of the original socket */
106 latch
= memcached_behavior_get(ptr
, MEMCACHED_BEHAVIOR_BUFFER_REQUESTS
);
108 memcached_behavior_set(ptr
, MEMCACHED_BEHAVIOR_BUFFER_REQUESTS
, 1);
110 rc
= memcached_set(ptr
, key
, key_length
,
111 (memcached_result_value(&ptr
->result
)),
112 (memcached_result_length(&ptr
->result
)),
114 (memcached_result_flags(&ptr
->result
)));
116 if (rc
== MEMCACHED_BUFFERED
&& latch
== 0)
117 memcached_behavior_set(ptr
, MEMCACHED_BEHAVIOR_BUFFER_REQUESTS
, 0);
121 rc
= memcached_set(ptr
, key
, key_length
,
122 (memcached_result_value(&ptr
->result
)),
123 (memcached_result_length(&ptr
->result
)),
125 (memcached_result_flags(&ptr
->result
)));
128 if (rc
== MEMCACHED_SUCCESS
|| rc
== MEMCACHED_BUFFERED
)
131 *value_length
= memcached_result_length(&ptr
->result
);
132 *flags
= memcached_result_flags(&ptr
->result
);
133 return memcached_string_c_copy(&ptr
->result
.value
);
141 (void)memcached_fetch(ptr
, NULL
, NULL
,
142 &dummy_length
, &dummy_flags
,
144 WATCHPOINT_ASSERT(dummy_length
== 0);
149 memcached_return_t
memcached_mget(memcached_st
*ptr
,
150 const char * const *keys
,
151 const size_t *key_length
,
152 size_t number_of_keys
)
154 return memcached_mget_by_key(ptr
, NULL
, 0, keys
, key_length
, number_of_keys
);
157 static memcached_return_t
binary_mget_by_key(memcached_st
*ptr
,
158 uint32_t master_server_key
,
159 bool is_group_key_set
,
160 const char * const *keys
,
161 const size_t *key_length
,
162 size_t number_of_keys
,
165 static memcached_return_t
memcached_mget_by_key_real(memcached_st
*ptr
,
166 const char *group_key
,
167 size_t group_key_length
,
168 const char * const *keys
,
169 const size_t *key_length
,
170 size_t number_of_keys
,
173 bool failures_occured_in_sending
= false;
174 const char *get_command
= "get ";
175 uint8_t get_command_length
= 4;
176 unsigned int master_server_key
= (unsigned int)-1; /* 0 is a valid server id! */
177 bool is_group_key_set
= false;
179 memcached_return_t rc
;
180 if (memcached_failed(rc
= initialize_query(ptr
)))
185 unlikely (ptr
->flags
.use_udp
)
186 return MEMCACHED_NOT_SUPPORTED
;
188 LIBMEMCACHED_MEMCACHED_MGET_START();
190 if (number_of_keys
== 0)
191 return MEMCACHED_NOTFOUND
;
193 if (ptr
->flags
.verify_key
&& (memcached_key_test(keys
, key_length
, number_of_keys
) == MEMCACHED_BAD_KEY_PROVIDED
))
195 return MEMCACHED_BAD_KEY_PROVIDED
;
198 if (group_key
&& group_key_length
)
200 if (ptr
->flags
.verify_key
and (memcached_key_test((const char * const *)&group_key
, &group_key_length
, 1) == MEMCACHED_BAD_KEY_PROVIDED
))
201 return MEMCACHED_BAD_KEY_PROVIDED
;
203 master_server_key
= memcached_generate_hash_with_redistribution(ptr
, group_key
, group_key_length
);
204 is_group_key_set
= true;
208 Here is where we pay for the non-block API. We need to remove any data sitting
209 in the queue before we start our get.
211 It might be optimum to bounce the connection if count > some number.
213 for (uint32_t x
= 0; x
< memcached_server_count(ptr
); x
++)
215 memcached_server_write_instance_st instance
=
216 memcached_server_instance_fetch(ptr
, x
);
218 if (memcached_server_response_count(instance
))
220 char buffer
[MEMCACHED_DEFAULT_COMMAND_SIZE
];
222 if (ptr
->flags
.no_block
)
223 (void)memcached_io_write(instance
, NULL
, 0, true);
225 while(memcached_server_response_count(instance
))
226 (void)memcached_response(instance
, buffer
, MEMCACHED_DEFAULT_COMMAND_SIZE
, &ptr
->result
);
230 if (ptr
->flags
.binary_protocol
)
232 return binary_mget_by_key(ptr
, master_server_key
, is_group_key_set
, keys
,
233 key_length
, number_of_keys
, mget_mode
);
236 if (ptr
->flags
.support_cas
)
238 get_command
= "gets ";
239 get_command_length
= 5;
243 If a server fails we warn about errors and start all over with sending keys
246 WATCHPOINT_ASSERT(rc
== MEMCACHED_SUCCESS
);
247 size_t hosts_connected
= 0;
248 for (uint32_t x
= 0; x
< number_of_keys
; x
++)
250 memcached_server_write_instance_st instance
;
253 if (is_group_key_set
)
255 server_key
= master_server_key
;
259 server_key
= memcached_generate_hash_with_redistribution(ptr
, keys
[x
], key_length
[x
]);
262 instance
= memcached_server_instance_fetch(ptr
, server_key
);
264 struct libmemcached_io_vector_st vector
[]=
266 { get_command_length
, get_command
},
267 { memcached_array_size(ptr
->prefix_key
), memcached_array_string(ptr
->prefix_key
) },
268 { key_length
[x
], keys
[x
] },
273 if (memcached_server_response_count(instance
) == 0)
275 rc
= memcached_connect(instance
);
277 if (rc
!= MEMCACHED_SUCCESS
)
283 if ((memcached_io_writev(instance
, vector
, 4, false)) == -1)
285 failures_occured_in_sending
= true;
288 WATCHPOINT_ASSERT(instance
->cursor_active
== 0);
289 memcached_server_response_increment(instance
);
290 WATCHPOINT_ASSERT(instance
->cursor_active
== 1);
294 if ((memcached_io_writev(instance
, (vector
+ 1), 3, false)) == -1)
296 memcached_server_response_reset(instance
);
297 failures_occured_in_sending
= true;
303 if (hosts_connected
== 0)
305 LIBMEMCACHED_MEMCACHED_MGET_END();
307 if (rc
!= MEMCACHED_SUCCESS
)
310 return MEMCACHED_NO_SERVERS
;
315 Should we muddle on if some servers are dead?
317 bool success_happened
= false;
318 for (uint32_t x
= 0; x
< memcached_server_count(ptr
); x
++)
320 memcached_server_write_instance_st instance
=
321 memcached_server_instance_fetch(ptr
, x
);
323 if (memcached_server_response_count(instance
))
325 /* We need to do something about non-connnected hosts in the future */
326 if ((memcached_io_write(instance
, "\r\n", 2, true)) == -1)
328 failures_occured_in_sending
= true;
332 success_happened
= true;
337 LIBMEMCACHED_MEMCACHED_MGET_END();
339 if (failures_occured_in_sending
&& success_happened
)
340 return MEMCACHED_SOME_ERRORS
;
342 if (success_happened
)
343 return MEMCACHED_SUCCESS
;
345 return MEMCACHED_FAILURE
;
348 memcached_return_t
memcached_mget_by_key(memcached_st
*ptr
,
349 const char *group_key
,
350 size_t group_key_length
,
351 const char * const *keys
,
352 const size_t *key_length
,
353 size_t number_of_keys
)
355 return memcached_mget_by_key_real(ptr
, group_key
, group_key_length
, keys
,
356 key_length
, number_of_keys
, true);
359 memcached_return_t
memcached_mget_execute(memcached_st
*ptr
,
360 const char * const *keys
,
361 const size_t *key_length
,
362 size_t number_of_keys
,
363 memcached_execute_fn
*callback
,
365 unsigned int number_of_callbacks
)
367 return memcached_mget_execute_by_key(ptr
, NULL
, 0, keys
, key_length
,
368 number_of_keys
, callback
,
369 context
, number_of_callbacks
);
372 memcached_return_t
memcached_mget_execute_by_key(memcached_st
*ptr
,
373 const char *group_key
,
374 size_t group_key_length
,
375 const char * const *keys
,
376 const size_t *key_length
,
377 size_t number_of_keys
,
378 memcached_execute_fn
*callback
,
380 unsigned int number_of_callbacks
)
382 if ((ptr
->flags
.binary_protocol
) == 0)
383 return MEMCACHED_NOT_SUPPORTED
;
385 memcached_return_t rc
;
386 memcached_callback_st
*original_callbacks
= ptr
->callbacks
;
387 memcached_callback_st cb
= {
394 rc
= memcached_mget_by_key(ptr
, group_key
, group_key_length
, keys
,
395 key_length
, number_of_keys
);
396 ptr
->callbacks
= original_callbacks
;
400 static memcached_return_t
simple_binary_mget(memcached_st
*ptr
,
401 uint32_t master_server_key
,
402 bool is_group_key_set
,
403 const char * const *keys
,
404 const size_t *key_length
,
405 size_t number_of_keys
, bool mget_mode
)
407 memcached_return_t rc
= MEMCACHED_NOTFOUND
;
409 bool flush
= (number_of_keys
== 1);
412 If a server fails we warn about errors and start all over with sending keys
415 for (uint32_t x
= 0; x
< number_of_keys
; ++x
)
418 memcached_server_write_instance_st instance
;
420 if (is_group_key_set
)
422 server_key
= master_server_key
;
426 server_key
= memcached_generate_hash_with_redistribution(ptr
, keys
[x
], key_length
[x
]);
429 instance
= memcached_server_instance_fetch(ptr
, server_key
);
431 if (memcached_server_response_count(instance
) == 0)
433 rc
= memcached_connect(instance
);
434 if (rc
!= MEMCACHED_SUCCESS
)
438 protocol_binary_request_getk request
= { }; //= {.bytes= {0}};
439 request
.message
.header
.request
.magic
= PROTOCOL_BINARY_REQ
;
441 request
.message
.header
.request
.opcode
= PROTOCOL_BINARY_CMD_GETKQ
;
443 request
.message
.header
.request
.opcode
= PROTOCOL_BINARY_CMD_GETK
;
445 memcached_return_t vk
;
446 vk
= memcached_validate_key_length(key_length
[x
],
447 ptr
->flags
.binary_protocol
);
448 unlikely (vk
!= MEMCACHED_SUCCESS
)
452 memcached_io_reset(instance
);
458 request
.message
.header
.request
.keylen
= htons((uint16_t)(key_length
[x
] + memcached_array_size(ptr
->prefix_key
)));
459 request
.message
.header
.request
.datatype
= PROTOCOL_BINARY_RAW_BYTES
;
460 request
.message
.header
.request
.bodylen
= htonl((uint32_t)( key_length
[x
] + memcached_array_size(ptr
->prefix_key
)));
462 struct libmemcached_io_vector_st vector
[]=
464 { sizeof(request
.bytes
), request
.bytes
},
465 { memcached_array_size(ptr
->prefix_key
), memcached_array_string(ptr
->prefix_key
) },
466 { key_length
[x
], keys
[x
] }
469 if (memcached_io_writev(instance
, vector
, 3, flush
) == -1)
471 memcached_server_response_reset(instance
);
472 rc
= MEMCACHED_SOME_ERRORS
;
476 /* We just want one pending response per server */
477 memcached_server_response_reset(instance
);
478 memcached_server_response_increment(instance
);
479 if ((x
> 0 && x
== ptr
->io_key_prefetch
) && memcached_flush_buffers(ptr
) != MEMCACHED_SUCCESS
)
481 rc
= MEMCACHED_SOME_ERRORS
;
488 Send a noop command to flush the buffers
490 protocol_binary_request_noop request
= {}; //= {.bytes= {0}};
491 request
.message
.header
.request
.magic
= PROTOCOL_BINARY_REQ
;
492 request
.message
.header
.request
.opcode
= PROTOCOL_BINARY_CMD_NOOP
;
493 request
.message
.header
.request
.datatype
= PROTOCOL_BINARY_RAW_BYTES
;
495 for (uint32_t x
= 0; x
< memcached_server_count(ptr
); ++x
)
497 memcached_server_write_instance_st instance
=
498 memcached_server_instance_fetch(ptr
, x
);
500 if (memcached_server_response_count(instance
))
502 if (memcached_io_write(instance
, NULL
, 0, true) == -1)
504 memcached_server_response_reset(instance
);
505 memcached_io_reset(instance
);
506 rc
= MEMCACHED_SOME_ERRORS
;
509 if (memcached_io_write(instance
, request
.bytes
,
510 sizeof(request
.bytes
), true) == -1)
512 memcached_server_response_reset(instance
);
513 memcached_io_reset(instance
);
514 rc
= MEMCACHED_SOME_ERRORS
;
524 static memcached_return_t
replication_binary_mget(memcached_st
*ptr
,
527 const char *const *keys
,
528 const size_t *key_length
,
529 size_t number_of_keys
)
531 memcached_return_t rc
= MEMCACHED_NOTFOUND
;
533 uint64_t randomize_read
= memcached_behavior_get(ptr
, MEMCACHED_BEHAVIOR_RANDOMIZE_REPLICA_READ
);
536 start
= (uint32_t)random() % (uint32_t)(ptr
->number_of_replicas
+ 1);
538 /* Loop for each replica */
539 for (uint32_t replica
= 0; replica
<= ptr
->number_of_replicas
; ++replica
)
543 for (uint32_t x
= 0; x
< number_of_keys
; ++x
)
545 if (hash
[x
] == memcached_server_count(ptr
))
546 continue; /* Already successfully sent */
548 uint32_t server
= hash
[x
] + replica
;
550 /* In case of randomized reads */
551 if (randomize_read
&& ((server
+ start
) <= (hash
[x
] + ptr
->number_of_replicas
)))
554 while (server
>= memcached_server_count(ptr
))
555 server
-= memcached_server_count(ptr
);
557 if (dead_servers
[server
])
560 memcached_server_write_instance_st instance
= memcached_server_instance_fetch(ptr
, server
);
562 if (memcached_server_response_count(instance
) == 0)
564 rc
= memcached_connect(instance
);
565 if (memcached_failed(rc
))
567 memcached_io_reset(instance
);
568 dead_servers
[server
]= true;
574 protocol_binary_request_getk request
= {};
575 request
.message
.header
.request
.magic
= PROTOCOL_BINARY_REQ
;
576 request
.message
.header
.request
.opcode
= PROTOCOL_BINARY_CMD_GETK
;
577 request
.message
.header
.request
.keylen
= htons((uint16_t)(key_length
[x
] + memcached_array_size(ptr
->prefix_key
)));
578 request
.message
.header
.request
.datatype
= PROTOCOL_BINARY_RAW_BYTES
;
579 request
.message
.header
.request
.bodylen
= htonl((uint32_t)(key_length
[x
] + memcached_array_size(ptr
->prefix_key
)));
582 * We need to disable buffering to actually know that the request was
583 * successfully sent to the server (so that we should expect a result
584 * back). It would be nice to do this in buffered mode, but then it
585 * would be complex to handle all error situations if we got to send
586 * some of the messages, and then we failed on writing out some others
587 * and we used the callback interface from memcached_mget_execute so
588 * that we might have processed some of the responses etc. For now,
589 * just make sure we work _correctly_
591 struct libmemcached_io_vector_st vector
[]=
593 { sizeof(request
.bytes
), request
.bytes
},
594 { memcached_array_size(ptr
->prefix_key
), memcached_array_string(ptr
->prefix_key
) },
595 { key_length
[x
], keys
[x
] }
598 if (memcached_io_writev(instance
, vector
, 3, true) == -1)
600 memcached_io_reset(instance
);
601 dead_servers
[server
]= true;
606 memcached_server_response_increment(instance
);
607 hash
[x
]= memcached_server_count(ptr
);
617 static memcached_return_t
binary_mget_by_key(memcached_st
*ptr
,
618 uint32_t master_server_key
,
619 bool is_group_key_set
,
620 const char * const *keys
,
621 const size_t *key_length
,
622 size_t number_of_keys
,
625 memcached_return_t rc
;
627 if (ptr
->number_of_replicas
== 0)
629 rc
= simple_binary_mget(ptr
, master_server_key
, is_group_key_set
,
630 keys
, key_length
, number_of_keys
, mget_mode
);
634 uint32_t* hash
= static_cast<uint32_t*>(libmemcached_malloc(ptr
, sizeof(uint32_t) * number_of_keys
));
635 bool* dead_servers
= static_cast<bool*>(libmemcached_calloc(ptr
, memcached_server_count(ptr
), sizeof(bool)));
637 if (hash
== NULL
|| dead_servers
== NULL
)
639 libmemcached_free(ptr
, hash
);
640 libmemcached_free(ptr
, dead_servers
);
641 return MEMCACHED_MEMORY_ALLOCATION_FAILURE
;
644 if (is_group_key_set
)
646 for (size_t x
= 0; x
< number_of_keys
; x
++)
648 hash
[x
]= master_server_key
;
653 for (size_t x
= 0; x
< number_of_keys
; x
++)
655 hash
[x
]= memcached_generate_hash_with_redistribution(ptr
, keys
[x
], key_length
[x
]);
659 rc
= replication_binary_mget(ptr
, hash
, dead_servers
, keys
,
660 key_length
, number_of_keys
);
662 libmemcached_free(ptr
, hash
);
663 libmemcached_free(ptr
, dead_servers
);
665 return MEMCACHED_SUCCESS
;