1 /* vim:expandtab:shiftwidth=2:tabstop=2:smarttab:
5 * Copyright (C) 2011 Data Differential, http://datadifferential.com/
6 * Copyright (C) 2006-2009 Brian Aker All rights reserved.
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions are
12 * * Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
15 * * Redistributions in binary form must reproduce the above
16 * copyright notice, this list of conditions and the following disclaimer
17 * in the documentation and/or other materials provided with the
20 * * The names of its contributors may not be used to endorse or
21 * promote products derived from this software without specific prior
24 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
25 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
26 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
27 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
28 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
29 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
30 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
31 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
32 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
33 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
34 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
38 #include <libmemcached/common.h>
41 What happens if no servers exist?
43 char *memcached_get(memcached_st
*ptr
, const char *key
,
47 memcached_return_t
*error
)
49 return memcached_get_by_key(ptr
, NULL
, 0, key
, key_length
, value_length
,
53 static memcached_return_t
memcached_mget_by_key_real(memcached_st
*ptr
,
54 const char *group_key
,
55 size_t group_key_length
,
56 const char * const *keys
,
57 const size_t *key_length
,
58 size_t number_of_keys
,
61 char *memcached_get_by_key(memcached_st
*ptr
,
62 const char *group_key
,
63 size_t group_key_length
,
64 const char *key
, size_t key_length
,
67 memcached_return_t
*error
)
69 memcached_return_t unused
;
75 if (ptr
->flags
.use_udp
)
82 *error
= memcached_set_error(*ptr
, MEMCACHED_NOT_SUPPORTED
, MEMCACHED_AT
);
86 uint64_t query_id
= ptr
->query_id
;
90 *error
= memcached_mget_by_key_real(ptr
, group_key
, group_key_length
,
91 (const char * const *)&key
, &key_length
,
93 assert_msg(ptr
->query_id
== query_id
+1, "Programmer error, the query_id was not incremented.");
96 if (memcached_failed(*error
))
98 if (memcached_has_current_error(*ptr
)) // Find the most accurate error
100 *error
= memcached_last_error(ptr
);
109 char *value
= memcached_fetch(ptr
, NULL
, NULL
,
110 value_length
, flags
, error
);
111 assert_msg(ptr
->query_id
== query_id
+1, "Programmer error, the query_id was not incremented.");
113 /* This is for historical reasons */
114 if (*error
== MEMCACHED_END
)
115 *error
= MEMCACHED_NOTFOUND
;
119 if (ptr
->get_key_failure
&& *error
== MEMCACHED_NOTFOUND
)
121 memcached_result_reset(&ptr
->result
);
122 memcached_return_t rc
= ptr
->get_key_failure(ptr
, key
, key_length
, &ptr
->result
);
124 /* On all failure drop to returning NULL */
125 if (rc
== MEMCACHED_SUCCESS
|| rc
== MEMCACHED_BUFFERED
)
127 if (rc
== MEMCACHED_BUFFERED
)
129 uint64_t latch
; /* We use latch to track the state of the original socket */
130 latch
= memcached_behavior_get(ptr
, MEMCACHED_BEHAVIOR_BUFFER_REQUESTS
);
132 memcached_behavior_set(ptr
, MEMCACHED_BEHAVIOR_BUFFER_REQUESTS
, 1);
134 rc
= memcached_set(ptr
, key
, key_length
,
135 (memcached_result_value(&ptr
->result
)),
136 (memcached_result_length(&ptr
->result
)),
138 (memcached_result_flags(&ptr
->result
)));
140 if (rc
== MEMCACHED_BUFFERED
&& latch
== 0)
142 memcached_behavior_set(ptr
, MEMCACHED_BEHAVIOR_BUFFER_REQUESTS
, 0);
147 rc
= memcached_set(ptr
, key
, key_length
,
148 (memcached_result_value(&ptr
->result
)),
149 (memcached_result_length(&ptr
->result
)),
151 (memcached_result_flags(&ptr
->result
)));
154 if (rc
== MEMCACHED_SUCCESS
|| rc
== MEMCACHED_BUFFERED
)
157 *value_length
= memcached_result_length(&ptr
->result
);
158 *flags
= memcached_result_flags(&ptr
->result
);
159 return memcached_string_take_value(&ptr
->result
.value
);
163 assert_msg(ptr
->query_id
== query_id
+1, "Programmer error, the query_id was not incremented.");
169 uint32_t dummy_flags
;
170 memcached_return_t dummy_error
;
172 char *dummy_value
= memcached_fetch(ptr
, NULL
, NULL
,
173 &dummy_length
, &dummy_flags
,
175 assert_msg(dummy_value
== 0, "memcached_fetch() returned additional values beyond the single get it expected");
176 assert_msg(dummy_length
== 0, "memcached_fetch() returned additional values beyond the single get it expected");
177 assert_msg(ptr
->query_id
== query_id
+1, "Programmer error, the query_id was not incremented.");
182 memcached_return_t
memcached_mget(memcached_st
*ptr
,
183 const char * const *keys
,
184 const size_t *key_length
,
185 size_t number_of_keys
)
187 return memcached_mget_by_key(ptr
, NULL
, 0, keys
, key_length
, number_of_keys
);
190 static memcached_return_t
binary_mget_by_key(memcached_st
*ptr
,
191 uint32_t master_server_key
,
192 bool is_group_key_set
,
193 const char * const *keys
,
194 const size_t *key_length
,
195 size_t number_of_keys
,
198 static memcached_return_t
memcached_mget_by_key_real(memcached_st
*ptr
,
199 const char *group_key
,
200 size_t group_key_length
,
201 const char * const *keys
,
202 const size_t *key_length
,
203 size_t number_of_keys
,
206 bool failures_occured_in_sending
= false;
207 const char *get_command
= "get ";
208 uint8_t get_command_length
= 4;
209 unsigned int master_server_key
= (unsigned int)-1; /* 0 is a valid server id! */
211 memcached_return_t rc
;
212 if (memcached_failed(rc
= initialize_query(ptr
)))
217 if (ptr
->flags
.use_udp
)
219 return memcached_set_error(*ptr
, MEMCACHED_NOT_SUPPORTED
, MEMCACHED_AT
);
222 LIBMEMCACHED_MEMCACHED_MGET_START();
224 if (number_of_keys
== 0)
226 return memcached_set_error(*ptr
, MEMCACHED_NOTFOUND
, MEMCACHED_AT
, memcached_literal_param("number_of_keys was zero"));
229 if (memcached_failed(memcached_key_test(*ptr
, keys
, key_length
, number_of_keys
)))
231 return memcached_set_error(*ptr
, MEMCACHED_BAD_KEY_PROVIDED
, MEMCACHED_AT
, memcached_literal_param("A bad key value was provided"));
234 bool is_group_key_set
= false;
235 if (group_key
and group_key_length
)
237 if (memcached_failed(memcached_key_test(*ptr
, (const char * const *)&group_key
, &group_key_length
, 1)))
239 return memcached_set_error(*ptr
, MEMCACHED_BAD_KEY_PROVIDED
, MEMCACHED_AT
, memcached_literal_param("A bad group key was provided."));
242 master_server_key
= memcached_generate_hash_with_redistribution(ptr
, group_key
, group_key_length
);
243 is_group_key_set
= true;
247 Here is where we pay for the non-block API. We need to remove any data sitting
248 in the queue before we start our get.
250 It might be optimum to bounce the connection if count > some number.
252 for (uint32_t x
= 0; x
< memcached_server_count(ptr
); x
++)
254 memcached_server_write_instance_st instance
=
255 memcached_server_instance_fetch(ptr
, x
);
257 if (memcached_server_response_count(instance
))
259 char buffer
[MEMCACHED_DEFAULT_COMMAND_SIZE
];
261 if (ptr
->flags
.no_block
)
263 (void)memcached_io_write(instance
, NULL
, 0, true);
266 while(memcached_server_response_count(instance
))
267 (void)memcached_response(instance
, buffer
, MEMCACHED_DEFAULT_COMMAND_SIZE
, &ptr
->result
);
271 if (ptr
->flags
.binary_protocol
)
273 return binary_mget_by_key(ptr
, master_server_key
, is_group_key_set
, keys
,
274 key_length
, number_of_keys
, mget_mode
);
277 if (ptr
->flags
.support_cas
)
279 get_command
= "gets ";
280 get_command_length
= 5;
284 If a server fails we warn about errors and start all over with sending keys
287 WATCHPOINT_ASSERT(rc
== MEMCACHED_SUCCESS
);
288 size_t hosts_connected
= 0;
289 for (uint32_t x
= 0; x
< number_of_keys
; x
++)
291 memcached_server_write_instance_st instance
;
294 if (is_group_key_set
)
296 server_key
= master_server_key
;
300 server_key
= memcached_generate_hash_with_redistribution(ptr
, keys
[x
], key_length
[x
]);
303 instance
= memcached_server_instance_fetch(ptr
, server_key
);
305 struct libmemcached_io_vector_st vector
[]=
307 { get_command
, get_command_length
},
308 { memcached_array_string(ptr
->_namespace
), memcached_array_size(ptr
->_namespace
) },
309 { keys
[x
], key_length
[x
] },
310 { memcached_literal_param(" ") }
314 if (memcached_server_response_count(instance
) == 0)
316 rc
= memcached_connect(instance
);
318 if (memcached_failed(rc
))
320 memcached_set_error(*instance
, rc
, MEMCACHED_AT
);
325 if ((memcached_io_writev(instance
, vector
, 4, false)) == -1)
327 failures_occured_in_sending
= true;
330 WATCHPOINT_ASSERT(instance
->cursor_active
== 0);
331 memcached_server_response_increment(instance
);
332 WATCHPOINT_ASSERT(instance
->cursor_active
== 1);
336 if ((memcached_io_writev(instance
, (vector
+ 1), 3, false)) == -1)
338 memcached_server_response_reset(instance
);
339 failures_occured_in_sending
= true;
345 if (hosts_connected
== 0)
347 LIBMEMCACHED_MEMCACHED_MGET_END();
349 if (memcached_failed(rc
))
352 return memcached_set_error(*ptr
, MEMCACHED_NO_SERVERS
, MEMCACHED_AT
);
357 Should we muddle on if some servers are dead?
359 bool success_happened
= false;
360 for (uint32_t x
= 0; x
< memcached_server_count(ptr
); x
++)
362 memcached_server_write_instance_st instance
=
363 memcached_server_instance_fetch(ptr
, x
);
365 if (memcached_server_response_count(instance
))
367 /* We need to do something about non-connnected hosts in the future */
368 if ((memcached_io_write(instance
, "\r\n", 2, true)) == -1)
370 failures_occured_in_sending
= true;
374 success_happened
= true;
379 LIBMEMCACHED_MEMCACHED_MGET_END();
381 if (failures_occured_in_sending
&& success_happened
)
383 return MEMCACHED_SOME_ERRORS
;
386 if (success_happened
)
387 return MEMCACHED_SUCCESS
;
389 return MEMCACHED_FAILURE
; // Complete failure occurred
392 memcached_return_t
memcached_mget_by_key(memcached_st
*ptr
,
393 const char *group_key
,
394 size_t group_key_length
,
395 const char * const *keys
,
396 const size_t *key_length
,
397 size_t number_of_keys
)
399 return memcached_mget_by_key_real(ptr
, group_key
, group_key_length
, keys
,
400 key_length
, number_of_keys
, true);
403 memcached_return_t
memcached_mget_execute(memcached_st
*ptr
,
404 const char * const *keys
,
405 const size_t *key_length
,
406 size_t number_of_keys
,
407 memcached_execute_fn
*callback
,
409 unsigned int number_of_callbacks
)
411 return memcached_mget_execute_by_key(ptr
, NULL
, 0, keys
, key_length
,
412 number_of_keys
, callback
,
413 context
, number_of_callbacks
);
416 memcached_return_t
memcached_mget_execute_by_key(memcached_st
*ptr
,
417 const char *group_key
,
418 size_t group_key_length
,
419 const char * const *keys
,
420 const size_t *key_length
,
421 size_t number_of_keys
,
422 memcached_execute_fn
*callback
,
424 unsigned int number_of_callbacks
)
426 if ((ptr
->flags
.binary_protocol
) == 0)
428 return MEMCACHED_NOT_SUPPORTED
;
431 memcached_return_t rc
;
432 memcached_callback_st
*original_callbacks
= ptr
->callbacks
;
433 memcached_callback_st cb
= {
440 rc
= memcached_mget_by_key(ptr
, group_key
, group_key_length
, keys
,
441 key_length
, number_of_keys
);
442 ptr
->callbacks
= original_callbacks
;
446 static memcached_return_t
simple_binary_mget(memcached_st
*ptr
,
447 uint32_t master_server_key
,
448 bool is_group_key_set
,
449 const char * const *keys
,
450 const size_t *key_length
,
451 size_t number_of_keys
, bool mget_mode
)
453 memcached_return_t rc
= MEMCACHED_NOTFOUND
;
455 bool flush
= (number_of_keys
== 1);
458 If a server fails we warn about errors and start all over with sending keys
461 for (uint32_t x
= 0; x
< number_of_keys
; ++x
)
465 if (is_group_key_set
)
467 server_key
= master_server_key
;
471 server_key
= memcached_generate_hash_with_redistribution(ptr
, keys
[x
], key_length
[x
]);
474 memcached_server_write_instance_st instance
= memcached_server_instance_fetch(ptr
, server_key
);
476 if (memcached_server_response_count(instance
) == 0)
478 rc
= memcached_connect(instance
);
479 if (memcached_failed(rc
))
485 protocol_binary_request_getk request
= { }; //= {.bytes= {0}};
486 request
.message
.header
.request
.magic
= PROTOCOL_BINARY_REQ
;
488 request
.message
.header
.request
.opcode
= PROTOCOL_BINARY_CMD_GETKQ
;
490 request
.message
.header
.request
.opcode
= PROTOCOL_BINARY_CMD_GETK
;
492 memcached_return_t vk
;
493 vk
= memcached_validate_key_length(key_length
[x
],
494 ptr
->flags
.binary_protocol
);
495 unlikely (vk
!= MEMCACHED_SUCCESS
)
499 memcached_io_reset(instance
);
505 request
.message
.header
.request
.keylen
= htons((uint16_t)(key_length
[x
] + memcached_array_size(ptr
->_namespace
)));
506 request
.message
.header
.request
.datatype
= PROTOCOL_BINARY_RAW_BYTES
;
507 request
.message
.header
.request
.bodylen
= htonl((uint32_t)( key_length
[x
] + memcached_array_size(ptr
->_namespace
)));
509 struct libmemcached_io_vector_st vector
[]=
511 { request
.bytes
, sizeof(request
.bytes
) },
512 { memcached_array_string(ptr
->_namespace
), memcached_array_size(ptr
->_namespace
) },
513 { keys
[x
], key_length
[x
] }
516 if (memcached_io_writev(instance
, vector
, 3, flush
) == -1)
518 memcached_server_response_reset(instance
);
519 rc
= MEMCACHED_SOME_ERRORS
;
523 /* We just want one pending response per server */
524 memcached_server_response_reset(instance
);
525 memcached_server_response_increment(instance
);
526 if ((x
> 0 && x
== ptr
->io_key_prefetch
) && memcached_flush_buffers(ptr
) != MEMCACHED_SUCCESS
)
528 rc
= MEMCACHED_SOME_ERRORS
;
535 Send a noop command to flush the buffers
537 protocol_binary_request_noop request
= {}; //= {.bytes= {0}};
538 request
.message
.header
.request
.magic
= PROTOCOL_BINARY_REQ
;
539 request
.message
.header
.request
.opcode
= PROTOCOL_BINARY_CMD_NOOP
;
540 request
.message
.header
.request
.datatype
= PROTOCOL_BINARY_RAW_BYTES
;
542 for (uint32_t x
= 0; x
< memcached_server_count(ptr
); ++x
)
544 memcached_server_write_instance_st instance
=
545 memcached_server_instance_fetch(ptr
, x
);
547 if (memcached_server_response_count(instance
))
549 if (memcached_io_write(instance
, NULL
, 0, true) == -1)
551 memcached_server_response_reset(instance
);
552 memcached_io_reset(instance
);
553 rc
= MEMCACHED_SOME_ERRORS
;
556 if (memcached_io_write(instance
, request
.bytes
,
557 sizeof(request
.bytes
), true) == -1)
559 memcached_server_response_reset(instance
);
560 memcached_io_reset(instance
);
561 rc
= MEMCACHED_SOME_ERRORS
;
571 static memcached_return_t
replication_binary_mget(memcached_st
*ptr
,
574 const char *const *keys
,
575 const size_t *key_length
,
576 size_t number_of_keys
)
578 memcached_return_t rc
= MEMCACHED_NOTFOUND
;
580 uint64_t randomize_read
= memcached_behavior_get(ptr
, MEMCACHED_BEHAVIOR_RANDOMIZE_REPLICA_READ
);
583 start
= (uint32_t)random() % (uint32_t)(ptr
->number_of_replicas
+ 1);
585 /* Loop for each replica */
586 for (uint32_t replica
= 0; replica
<= ptr
->number_of_replicas
; ++replica
)
590 for (uint32_t x
= 0; x
< number_of_keys
; ++x
)
592 if (hash
[x
] == memcached_server_count(ptr
))
593 continue; /* Already successfully sent */
595 uint32_t server
= hash
[x
] + replica
;
597 /* In case of randomized reads */
598 if (randomize_read
&& ((server
+ start
) <= (hash
[x
] + ptr
->number_of_replicas
)))
601 while (server
>= memcached_server_count(ptr
))
603 server
-= memcached_server_count(ptr
);
606 if (dead_servers
[server
])
611 memcached_server_write_instance_st instance
= memcached_server_instance_fetch(ptr
, server
);
613 if (memcached_server_response_count(instance
) == 0)
615 rc
= memcached_connect(instance
);
617 if (memcached_failed(rc
))
619 memcached_io_reset(instance
);
620 dead_servers
[server
]= true;
626 protocol_binary_request_getk request
= {};
627 request
.message
.header
.request
.magic
= PROTOCOL_BINARY_REQ
;
628 request
.message
.header
.request
.opcode
= PROTOCOL_BINARY_CMD_GETK
;
629 request
.message
.header
.request
.keylen
= htons((uint16_t)(key_length
[x
] + memcached_array_size(ptr
->_namespace
)));
630 request
.message
.header
.request
.datatype
= PROTOCOL_BINARY_RAW_BYTES
;
631 request
.message
.header
.request
.bodylen
= htonl((uint32_t)(key_length
[x
] + memcached_array_size(ptr
->_namespace
)));
634 * We need to disable buffering to actually know that the request was
635 * successfully sent to the server (so that we should expect a result
636 * back). It would be nice to do this in buffered mode, but then it
637 * would be complex to handle all error situations if we got to send
638 * some of the messages, and then we failed on writing out some others
639 * and we used the callback interface from memcached_mget_execute so
640 * that we might have processed some of the responses etc. For now,
641 * just make sure we work _correctly_
643 struct libmemcached_io_vector_st vector
[]=
645 { request
.bytes
, sizeof(request
.bytes
) },
646 { memcached_array_string(ptr
->_namespace
), memcached_array_size(ptr
->_namespace
) },
647 { keys
[x
], key_length
[x
] }
650 if (memcached_io_writev(instance
, vector
, 3, true) == -1)
652 memcached_io_reset(instance
);
653 dead_servers
[server
]= true;
658 memcached_server_response_increment(instance
);
659 hash
[x
]= memcached_server_count(ptr
);
671 static memcached_return_t
binary_mget_by_key(memcached_st
*ptr
,
672 uint32_t master_server_key
,
673 bool is_group_key_set
,
674 const char * const *keys
,
675 const size_t *key_length
,
676 size_t number_of_keys
,
679 if (ptr
->number_of_replicas
== 0)
681 return simple_binary_mget(ptr
, master_server_key
, is_group_key_set
,
682 keys
, key_length
, number_of_keys
, mget_mode
);
685 uint32_t* hash
= libmemcached_xvalloc(ptr
, number_of_keys
, uint32_t);
686 bool* dead_servers
= libmemcached_xcalloc(ptr
, memcached_server_count(ptr
), bool);
688 if (hash
== NULL
|| dead_servers
== NULL
)
690 libmemcached_free(ptr
, hash
);
691 libmemcached_free(ptr
, dead_servers
);
692 return MEMCACHED_MEMORY_ALLOCATION_FAILURE
;
695 if (is_group_key_set
)
697 for (size_t x
= 0; x
< number_of_keys
; x
++)
699 hash
[x
]= master_server_key
;
704 for (size_t x
= 0; x
< number_of_keys
; x
++)
706 hash
[x
]= memcached_generate_hash_with_redistribution(ptr
, keys
[x
], key_length
[x
]);
710 memcached_return_t rc
= replication_binary_mget(ptr
, hash
, dead_servers
, keys
,
711 key_length
, number_of_keys
);
713 WATCHPOINT_IFERROR(rc
);
714 libmemcached_free(ptr
, hash
);
715 libmemcached_free(ptr
, dead_servers
);
717 return MEMCACHED_SUCCESS
;