2 * Copyright (C) 2006-2009 Brian Aker
5 * Use and distribution licensed under the BSD license. See
6 * the COPYING file in the parent directory for full text.
8 * Summary: Get functions for libmemcached
15 What happens if no servers exist?
17 char *memcached_get(memcached_st
*ptr
, const char *key
,
21 memcached_return_t
*error
)
23 return memcached_get_by_key(ptr
, NULL
, 0, key
, key_length
, value_length
,
27 static memcached_return_t
memcached_mget_by_key_real(memcached_st
*ptr
,
28 const char *group_key
,
29 size_t group_key_length
,
30 const char * const *keys
,
31 const size_t *key_length
,
32 size_t number_of_keys
,
35 char *memcached_get_by_key(memcached_st
*ptr
,
36 const char *group_key
,
37 size_t group_key_length
,
38 const char *key
, size_t key_length
,
41 memcached_return_t
*error
)
46 memcached_return_t dummy_error
;
48 unlikely (ptr
->flags
.use_udp
)
50 *error
= MEMCACHED_NOT_SUPPORTED
;
55 *error
= memcached_mget_by_key_real(ptr
, group_key
, group_key_length
,
56 (const char * const *)&key
,
57 &key_length
, 1, false);
59 value
= memcached_fetch(ptr
, NULL
, NULL
,
60 value_length
, flags
, error
);
61 /* This is for historical reasons */
62 if (*error
== MEMCACHED_END
)
63 *error
= MEMCACHED_NOTFOUND
;
67 if (ptr
->get_key_failure
&& *error
== MEMCACHED_NOTFOUND
)
69 memcached_return_t rc
;
71 memcached_result_reset(&ptr
->result
);
72 rc
= ptr
->get_key_failure(ptr
, key
, key_length
, &ptr
->result
);
74 /* On all failure drop to returning NULL */
75 if (rc
== MEMCACHED_SUCCESS
|| rc
== MEMCACHED_BUFFERED
)
77 if (rc
== MEMCACHED_BUFFERED
)
79 uint64_t latch
; /* We use latch to track the state of the original socket */
80 latch
= memcached_behavior_get(ptr
, MEMCACHED_BEHAVIOR_BUFFER_REQUESTS
);
82 memcached_behavior_set(ptr
, MEMCACHED_BEHAVIOR_BUFFER_REQUESTS
, 1);
84 rc
= memcached_set(ptr
, key
, key_length
,
85 (memcached_result_value(&ptr
->result
)),
86 (memcached_result_length(&ptr
->result
)),
88 (memcached_result_flags(&ptr
->result
)));
90 if (rc
== MEMCACHED_BUFFERED
&& latch
== 0)
91 memcached_behavior_set(ptr
, MEMCACHED_BEHAVIOR_BUFFER_REQUESTS
, 0);
95 rc
= memcached_set(ptr
, key
, key_length
,
96 (memcached_result_value(&ptr
->result
)),
97 (memcached_result_length(&ptr
->result
)),
99 (memcached_result_flags(&ptr
->result
)));
102 if (rc
== MEMCACHED_SUCCESS
|| rc
== MEMCACHED_BUFFERED
)
105 *value_length
= memcached_result_length(&ptr
->result
);
106 *flags
= memcached_result_flags(&ptr
->result
);
107 return memcached_string_c_copy(&ptr
->result
.value
);
115 (void)memcached_fetch(ptr
, NULL
, NULL
,
116 &dummy_length
, &dummy_flags
,
118 WATCHPOINT_ASSERT(dummy_length
== 0);
123 memcached_return_t
memcached_mget(memcached_st
*ptr
,
124 const char * const *keys
,
125 const size_t *key_length
,
126 size_t number_of_keys
)
128 return memcached_mget_by_key(ptr
, NULL
, 0, keys
, key_length
, number_of_keys
);
131 static memcached_return_t
binary_mget_by_key(memcached_st
*ptr
,
132 uint32_t master_server_key
,
133 bool is_group_key_set
,
134 const char * const *keys
,
135 const size_t *key_length
,
136 size_t number_of_keys
,
139 static memcached_return_t
memcached_mget_by_key_real(memcached_st
*ptr
,
140 const char *group_key
,
141 size_t group_key_length
,
142 const char * const *keys
,
143 const size_t *key_length
,
144 size_t number_of_keys
,
147 bool failures_occured_in_sending
= false;
148 const char *get_command
= "get ";
149 uint8_t get_command_length
= 4;
150 unsigned int master_server_key
= (unsigned int)-1; /* 0 is a valid server id! */
151 bool is_group_key_set
= false;
153 memcached_return_t rc
;
154 if (memcached_failed(rc
= initialize_query(ptr
)))
159 unlikely (ptr
->flags
.use_udp
)
160 return MEMCACHED_NOT_SUPPORTED
;
162 LIBMEMCACHED_MEMCACHED_MGET_START();
164 if (number_of_keys
== 0)
165 return MEMCACHED_NOTFOUND
;
167 if (ptr
->flags
.verify_key
&& (memcached_key_test(keys
, key_length
, number_of_keys
) == MEMCACHED_BAD_KEY_PROVIDED
))
169 return MEMCACHED_BAD_KEY_PROVIDED
;
172 if (group_key
&& group_key_length
)
174 if (ptr
->flags
.verify_key
and (memcached_key_test((const char * const *)&group_key
, &group_key_length
, 1) == MEMCACHED_BAD_KEY_PROVIDED
))
175 return MEMCACHED_BAD_KEY_PROVIDED
;
177 master_server_key
= memcached_generate_hash_with_redistribution(ptr
, group_key
, group_key_length
);
178 is_group_key_set
= true;
182 Here is where we pay for the non-block API. We need to remove any data sitting
183 in the queue before we start our get.
185 It might be optimum to bounce the connection if count > some number.
187 for (uint32_t x
= 0; x
< memcached_server_count(ptr
); x
++)
189 memcached_server_write_instance_st instance
=
190 memcached_server_instance_fetch(ptr
, x
);
192 if (memcached_server_response_count(instance
))
194 char buffer
[MEMCACHED_DEFAULT_COMMAND_SIZE
];
196 if (ptr
->flags
.no_block
)
197 (void)memcached_io_write(instance
, NULL
, 0, true);
199 while(memcached_server_response_count(instance
))
200 (void)memcached_response(instance
, buffer
, MEMCACHED_DEFAULT_COMMAND_SIZE
, &ptr
->result
);
204 if (ptr
->flags
.binary_protocol
)
206 return binary_mget_by_key(ptr
, master_server_key
, is_group_key_set
, keys
,
207 key_length
, number_of_keys
, mget_mode
);
210 if (ptr
->flags
.support_cas
)
212 get_command
= "gets ";
213 get_command_length
= 5;
217 If a server fails we warn about errors and start all over with sending keys
220 WATCHPOINT_ASSERT(rc
== MEMCACHED_SUCCESS
);
221 size_t hosts_connected
= 0;
222 for (uint32_t x
= 0; x
< number_of_keys
; x
++)
224 memcached_server_write_instance_st instance
;
227 if (is_group_key_set
)
229 server_key
= master_server_key
;
233 server_key
= memcached_generate_hash_with_redistribution(ptr
, keys
[x
], key_length
[x
]);
236 instance
= memcached_server_instance_fetch(ptr
, server_key
);
238 struct libmemcached_io_vector_st vector
[]=
240 { get_command_length
, get_command
},
241 { memcached_array_size(ptr
->prefix_key
), memcached_array_string(ptr
->prefix_key
) },
242 { key_length
[x
], keys
[x
] },
247 if (memcached_server_response_count(instance
) == 0)
249 rc
= memcached_connect(instance
);
251 if (rc
!= MEMCACHED_SUCCESS
)
257 if ((memcached_io_writev(instance
, vector
, 4, false)) == -1)
259 failures_occured_in_sending
= true;
262 WATCHPOINT_ASSERT(instance
->cursor_active
== 0);
263 memcached_server_response_increment(instance
);
264 WATCHPOINT_ASSERT(instance
->cursor_active
== 1);
268 if ((memcached_io_writev(instance
, (vector
+ 1), 3, false)) == -1)
270 memcached_server_response_reset(instance
);
271 failures_occured_in_sending
= true;
277 if (hosts_connected
== 0)
279 LIBMEMCACHED_MEMCACHED_MGET_END();
281 if (rc
!= MEMCACHED_SUCCESS
)
284 return MEMCACHED_NO_SERVERS
;
289 Should we muddle on if some servers are dead?
291 bool success_happened
= false;
292 for (uint32_t x
= 0; x
< memcached_server_count(ptr
); x
++)
294 memcached_server_write_instance_st instance
=
295 memcached_server_instance_fetch(ptr
, x
);
297 if (memcached_server_response_count(instance
))
299 /* We need to do something about non-connnected hosts in the future */
300 if ((memcached_io_write(instance
, "\r\n", 2, true)) == -1)
302 failures_occured_in_sending
= true;
306 success_happened
= true;
311 LIBMEMCACHED_MEMCACHED_MGET_END();
313 if (failures_occured_in_sending
&& success_happened
)
314 return MEMCACHED_SOME_ERRORS
;
316 if (success_happened
)
317 return MEMCACHED_SUCCESS
;
319 return MEMCACHED_FAILURE
;
322 memcached_return_t
memcached_mget_by_key(memcached_st
*ptr
,
323 const char *group_key
,
324 size_t group_key_length
,
325 const char * const *keys
,
326 const size_t *key_length
,
327 size_t number_of_keys
)
329 return memcached_mget_by_key_real(ptr
, group_key
, group_key_length
, keys
,
330 key_length
, number_of_keys
, true);
333 memcached_return_t
memcached_mget_execute(memcached_st
*ptr
,
334 const char * const *keys
,
335 const size_t *key_length
,
336 size_t number_of_keys
,
337 memcached_execute_fn
*callback
,
339 unsigned int number_of_callbacks
)
341 return memcached_mget_execute_by_key(ptr
, NULL
, 0, keys
, key_length
,
342 number_of_keys
, callback
,
343 context
, number_of_callbacks
);
346 memcached_return_t
memcached_mget_execute_by_key(memcached_st
*ptr
,
347 const char *group_key
,
348 size_t group_key_length
,
349 const char * const *keys
,
350 const size_t *key_length
,
351 size_t number_of_keys
,
352 memcached_execute_fn
*callback
,
354 unsigned int number_of_callbacks
)
356 if ((ptr
->flags
.binary_protocol
) == 0)
357 return MEMCACHED_NOT_SUPPORTED
;
359 memcached_return_t rc
;
360 memcached_callback_st
*original_callbacks
= ptr
->callbacks
;
361 memcached_callback_st cb
= {
368 rc
= memcached_mget_by_key(ptr
, group_key
, group_key_length
, keys
,
369 key_length
, number_of_keys
);
370 ptr
->callbacks
= original_callbacks
;
374 static memcached_return_t
simple_binary_mget(memcached_st
*ptr
,
375 uint32_t master_server_key
,
376 bool is_group_key_set
,
377 const char * const *keys
,
378 const size_t *key_length
,
379 size_t number_of_keys
, bool mget_mode
)
381 memcached_return_t rc
= MEMCACHED_NOTFOUND
;
383 bool flush
= (number_of_keys
== 1);
386 If a server fails we warn about errors and start all over with sending keys
389 for (uint32_t x
= 0; x
< number_of_keys
; ++x
)
392 memcached_server_write_instance_st instance
;
394 if (is_group_key_set
)
396 server_key
= master_server_key
;
400 server_key
= memcached_generate_hash_with_redistribution(ptr
, keys
[x
], key_length
[x
]);
403 instance
= memcached_server_instance_fetch(ptr
, server_key
);
405 if (memcached_server_response_count(instance
) == 0)
407 rc
= memcached_connect(instance
);
408 if (rc
!= MEMCACHED_SUCCESS
)
412 protocol_binary_request_getk request
= { }; //= {.bytes= {0}};
413 request
.message
.header
.request
.magic
= PROTOCOL_BINARY_REQ
;
415 request
.message
.header
.request
.opcode
= PROTOCOL_BINARY_CMD_GETKQ
;
417 request
.message
.header
.request
.opcode
= PROTOCOL_BINARY_CMD_GETK
;
419 memcached_return_t vk
;
420 vk
= memcached_validate_key_length(key_length
[x
],
421 ptr
->flags
.binary_protocol
);
422 unlikely (vk
!= MEMCACHED_SUCCESS
)
426 memcached_io_reset(instance
);
432 request
.message
.header
.request
.keylen
= htons((uint16_t)(key_length
[x
] + memcached_array_size(ptr
->prefix_key
)));
433 request
.message
.header
.request
.datatype
= PROTOCOL_BINARY_RAW_BYTES
;
434 request
.message
.header
.request
.bodylen
= htonl((uint32_t)( key_length
[x
] + memcached_array_size(ptr
->prefix_key
)));
436 struct libmemcached_io_vector_st vector
[]=
438 { sizeof(request
.bytes
), request
.bytes
},
439 { memcached_array_size(ptr
->prefix_key
), memcached_array_string(ptr
->prefix_key
) },
440 { key_length
[x
], keys
[x
] }
443 if (memcached_io_writev(instance
, vector
, 3, flush
) == -1)
445 memcached_server_response_reset(instance
);
446 rc
= MEMCACHED_SOME_ERRORS
;
450 /* We just want one pending response per server */
451 memcached_server_response_reset(instance
);
452 memcached_server_response_increment(instance
);
453 if ((x
> 0 && x
== ptr
->io_key_prefetch
) && memcached_flush_buffers(ptr
) != MEMCACHED_SUCCESS
)
455 rc
= MEMCACHED_SOME_ERRORS
;
462 Send a noop command to flush the buffers
464 protocol_binary_request_noop request
= {}; //= {.bytes= {0}};
465 request
.message
.header
.request
.magic
= PROTOCOL_BINARY_REQ
;
466 request
.message
.header
.request
.opcode
= PROTOCOL_BINARY_CMD_NOOP
;
467 request
.message
.header
.request
.datatype
= PROTOCOL_BINARY_RAW_BYTES
;
469 for (uint32_t x
= 0; x
< memcached_server_count(ptr
); ++x
)
471 memcached_server_write_instance_st instance
=
472 memcached_server_instance_fetch(ptr
, x
);
474 if (memcached_server_response_count(instance
))
476 if (memcached_io_write(instance
, NULL
, 0, true) == -1)
478 memcached_server_response_reset(instance
);
479 memcached_io_reset(instance
);
480 rc
= MEMCACHED_SOME_ERRORS
;
483 if (memcached_io_write(instance
, request
.bytes
,
484 sizeof(request
.bytes
), true) == -1)
486 memcached_server_response_reset(instance
);
487 memcached_io_reset(instance
);
488 rc
= MEMCACHED_SOME_ERRORS
;
498 static memcached_return_t
replication_binary_mget(memcached_st
*ptr
,
501 const char *const *keys
,
502 const size_t *key_length
,
503 size_t number_of_keys
)
505 memcached_return_t rc
= MEMCACHED_NOTFOUND
;
507 uint64_t randomize_read
= memcached_behavior_get(ptr
, MEMCACHED_BEHAVIOR_RANDOMIZE_REPLICA_READ
);
510 start
= (uint32_t)random() % (uint32_t)(ptr
->number_of_replicas
+ 1);
512 /* Loop for each replica */
513 for (uint32_t replica
= 0; replica
<= ptr
->number_of_replicas
; ++replica
)
517 for (uint32_t x
= 0; x
< number_of_keys
; ++x
)
519 memcached_server_write_instance_st instance
;
521 if (hash
[x
] == memcached_server_count(ptr
))
522 continue; /* Already successfully sent */
524 uint32_t server
= hash
[x
] + replica
;
526 /* In case of randomized reads */
527 if (randomize_read
&& ((server
+ start
) <= (hash
[x
] + ptr
->number_of_replicas
)))
530 while (server
>= memcached_server_count(ptr
))
531 server
-= memcached_server_count(ptr
);
533 if (dead_servers
[server
])
536 instance
= memcached_server_instance_fetch(ptr
, server
);
538 if (memcached_server_response_count(instance
) == 0)
540 rc
= memcached_connect(instance
);
541 if (rc
!= MEMCACHED_SUCCESS
)
543 memcached_io_reset(instance
);
544 dead_servers
[server
]= true;
550 protocol_binary_request_getk request
= {};
551 request
.message
.header
.request
.magic
= PROTOCOL_BINARY_REQ
;
552 request
.message
.header
.request
.opcode
= PROTOCOL_BINARY_CMD_GETK
;
553 request
.message
.header
.request
.keylen
= htons((uint16_t)(key_length
[x
] + memcached_array_size(ptr
->prefix_key
)));
554 request
.message
.header
.request
.datatype
= PROTOCOL_BINARY_RAW_BYTES
;
555 request
.message
.header
.request
.bodylen
= htonl((uint32_t)(key_length
[x
] + memcached_array_size(ptr
->prefix_key
)));
558 * We need to disable buffering to actually know that the request was
559 * successfully sent to the server (so that we should expect a result
560 * back). It would be nice to do this in buffered mode, but then it
561 * would be complex to handle all error situations if we got to send
562 * some of the messages, and then we failed on writing out some others
563 * and we used the callback interface from memcached_mget_execute so
564 * that we might have processed some of the responses etc. For now,
565 * just make sure we work _correctly_
567 struct libmemcached_io_vector_st vector
[]=
569 { sizeof(request
.bytes
), request
.bytes
},
570 { memcached_array_size(ptr
->prefix_key
), memcached_array_string(ptr
->prefix_key
) },
571 { key_length
[x
], keys
[x
] }
574 if (memcached_io_writev(instance
, vector
, 3, true) == -1)
576 memcached_io_reset(instance
);
577 dead_servers
[server
]= true;
582 memcached_server_response_increment(instance
);
583 hash
[x
]= memcached_server_count(ptr
);
593 static memcached_return_t
binary_mget_by_key(memcached_st
*ptr
,
594 uint32_t master_server_key
,
595 bool is_group_key_set
,
596 const char * const *keys
,
597 const size_t *key_length
,
598 size_t number_of_keys
,
601 memcached_return_t rc
;
603 if (ptr
->number_of_replicas
== 0)
605 rc
= simple_binary_mget(ptr
, master_server_key
, is_group_key_set
,
606 keys
, key_length
, number_of_keys
, mget_mode
);
613 hash
= static_cast<uint32_t*>(libmemcached_malloc(ptr
, sizeof(uint32_t) * number_of_keys
));
614 dead_servers
= static_cast<bool*>(libmemcached_calloc(ptr
, memcached_server_count(ptr
), sizeof(bool)));
616 if (hash
== NULL
|| dead_servers
== NULL
)
618 libmemcached_free(ptr
, hash
);
619 libmemcached_free(ptr
, dead_servers
);
620 return MEMCACHED_MEMORY_ALLOCATION_FAILURE
;
623 if (is_group_key_set
)
625 for (size_t x
= 0; x
< number_of_keys
; x
++)
627 hash
[x
]= master_server_key
;
632 for (size_t x
= 0; x
< number_of_keys
; x
++)
634 hash
[x
]= memcached_generate_hash_with_redistribution(ptr
, keys
[x
], key_length
[x
]);
638 rc
= replication_binary_mget(ptr
, hash
, dead_servers
, keys
,
639 key_length
, number_of_keys
);
641 libmemcached_free(ptr
, hash
);
642 libmemcached_free(ptr
, dead_servers
);
644 return MEMCACHED_SUCCESS
;