Fix detangle, minor style cleanups.
[awesomized/libmemcached] / libmemcached / get.cc
1 /* vim:expandtab:shiftwidth=2:tabstop=2:smarttab:
2 *
3 * Libmemcached library
4 *
5 * Copyright (C) 2011 Data Differential, http://datadifferential.com/
6 * Copyright (C) 2006-2009 Brian Aker All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions are
10 * met:
11 *
12 * * Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 *
15 * * Redistributions in binary form must reproduce the above
16 * copyright notice, this list of conditions and the following disclaimer
17 * in the documentation and/or other materials provided with the
18 * distribution.
19 *
20 * * The names of its contributors may not be used to endorse or
21 * promote products derived from this software without specific prior
22 * written permission.
23 *
24 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
25 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
26 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
27 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
28 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
29 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
30 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
31 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
32 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
33 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
34 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
35 *
36 */
37
38 #include <libmemcached/common.h>
39
40 /*
41 What happens if no servers exist?
42 */
43 char *memcached_get(memcached_st *ptr, const char *key,
44 size_t key_length,
45 size_t *value_length,
46 uint32_t *flags,
47 memcached_return_t *error)
48 {
49 return memcached_get_by_key(ptr, NULL, 0, key, key_length, value_length,
50 flags, error);
51 }
52
53 static memcached_return_t memcached_mget_by_key_real(memcached_st *ptr,
54 const char *group_key,
55 size_t group_key_length,
56 const char * const *keys,
57 const size_t *key_length,
58 size_t number_of_keys,
59 bool mget_mode);
60
61 char *memcached_get_by_key(memcached_st *ptr,
62 const char *group_key,
63 size_t group_key_length,
64 const char *key, size_t key_length,
65 size_t *value_length,
66 uint32_t *flags,
67 memcached_return_t *error)
68 {
69 memcached_return_t unused;
70 if (error == NULL)
71 {
72 error= &unused;
73 }
74
75 if (ptr->flags.use_udp)
76 {
77 if (value_length)
78 {
79 *value_length= 0;
80 }
81
82 *error= memcached_set_error(*ptr, MEMCACHED_NOT_SUPPORTED, MEMCACHED_AT);
83 return NULL;
84 }
85
86 uint64_t query_id= ptr->query_id;
87 (void)query_id;
88
89 /* Request the key */
90 *error= memcached_mget_by_key_real(ptr, group_key, group_key_length,
91 (const char * const *)&key, &key_length,
92 1, false);
93 assert_msg(ptr->query_id == query_id +1, "Programmer error, the query_id was not incremented.");
94
95
96 if (memcached_failed(*error))
97 {
98 if (memcached_has_current_error(*ptr)) // Find the most accurate error
99 {
100 *error= memcached_last_error(ptr);
101 }
102
103 if (value_length)
104 *value_length= 0;
105
106 return NULL;
107 }
108
109 char *value= memcached_fetch(ptr, NULL, NULL,
110 value_length, flags, error);
111 assert_msg(ptr->query_id == query_id +1, "Programmer error, the query_id was not incremented.");
112
113 /* This is for historical reasons */
114 if (*error == MEMCACHED_END)
115 *error= MEMCACHED_NOTFOUND;
116
117 if (value == NULL)
118 {
119 if (ptr->get_key_failure && *error == MEMCACHED_NOTFOUND)
120 {
121 memcached_result_reset(&ptr->result);
122 memcached_return_t rc= ptr->get_key_failure(ptr, key, key_length, &ptr->result);
123
124 /* On all failure drop to returning NULL */
125 if (rc == MEMCACHED_SUCCESS || rc == MEMCACHED_BUFFERED)
126 {
127 if (rc == MEMCACHED_BUFFERED)
128 {
129 uint64_t latch; /* We use latch to track the state of the original socket */
130 latch= memcached_behavior_get(ptr, MEMCACHED_BEHAVIOR_BUFFER_REQUESTS);
131 if (latch == 0)
132 memcached_behavior_set(ptr, MEMCACHED_BEHAVIOR_BUFFER_REQUESTS, 1);
133
134 rc= memcached_set(ptr, key, key_length,
135 (memcached_result_value(&ptr->result)),
136 (memcached_result_length(&ptr->result)),
137 0,
138 (memcached_result_flags(&ptr->result)));
139
140 if (rc == MEMCACHED_BUFFERED && latch == 0)
141 {
142 memcached_behavior_set(ptr, MEMCACHED_BEHAVIOR_BUFFER_REQUESTS, 0);
143 }
144 }
145 else
146 {
147 rc= memcached_set(ptr, key, key_length,
148 (memcached_result_value(&ptr->result)),
149 (memcached_result_length(&ptr->result)),
150 0,
151 (memcached_result_flags(&ptr->result)));
152 }
153
154 if (rc == MEMCACHED_SUCCESS || rc == MEMCACHED_BUFFERED)
155 {
156 *error= rc;
157 *value_length= memcached_result_length(&ptr->result);
158 *flags= memcached_result_flags(&ptr->result);
159 return memcached_string_take_value(&ptr->result.value);
160 }
161 }
162 }
163 assert_msg(ptr->query_id == query_id +1, "Programmer error, the query_id was not incremented.");
164
165 return NULL;
166 }
167
168 size_t dummy_length;
169 uint32_t dummy_flags;
170 memcached_return_t dummy_error;
171
172 char *dummy_value= memcached_fetch(ptr, NULL, NULL,
173 &dummy_length, &dummy_flags,
174 &dummy_error);
175 assert_msg(dummy_value == 0, "memcached_fetch() returned additional values beyond the single get it expected");
176 assert_msg(dummy_length == 0, "memcached_fetch() returned additional values beyond the single get it expected");
177 assert_msg(ptr->query_id == query_id +1, "Programmer error, the query_id was not incremented.");
178
179 return value;
180 }
181
182 memcached_return_t memcached_mget(memcached_st *ptr,
183 const char * const *keys,
184 const size_t *key_length,
185 size_t number_of_keys)
186 {
187 return memcached_mget_by_key(ptr, NULL, 0, keys, key_length, number_of_keys);
188 }
189
190 static memcached_return_t binary_mget_by_key(memcached_st *ptr,
191 uint32_t master_server_key,
192 bool is_group_key_set,
193 const char * const *keys,
194 const size_t *key_length,
195 size_t number_of_keys,
196 bool mget_mode);
197
198 static memcached_return_t memcached_mget_by_key_real(memcached_st *ptr,
199 const char *group_key,
200 size_t group_key_length,
201 const char * const *keys,
202 const size_t *key_length,
203 size_t number_of_keys,
204 bool mget_mode)
205 {
206 bool failures_occured_in_sending= false;
207 const char *get_command= "get ";
208 uint8_t get_command_length= 4;
209 unsigned int master_server_key= (unsigned int)-1; /* 0 is a valid server id! */
210
211 memcached_return_t rc;
212 if (memcached_failed(rc= initialize_query(ptr)))
213 {
214 return rc;
215 }
216
217 if (ptr->flags.use_udp)
218 {
219 return memcached_set_error(*ptr, MEMCACHED_NOT_SUPPORTED, MEMCACHED_AT);
220 }
221
222 LIBMEMCACHED_MEMCACHED_MGET_START();
223
224 if (number_of_keys == 0)
225 {
226 return memcached_set_error(*ptr, MEMCACHED_NOTFOUND, MEMCACHED_AT, memcached_literal_param("number_of_keys was zero"));
227 }
228
229 if (memcached_failed(memcached_key_test(*ptr, keys, key_length, number_of_keys)))
230 {
231 return memcached_set_error(*ptr, MEMCACHED_BAD_KEY_PROVIDED, MEMCACHED_AT, memcached_literal_param("A bad key value was provided"));
232 }
233
234 bool is_group_key_set= false;
235 if (group_key and group_key_length)
236 {
237 if (memcached_failed(memcached_key_test(*ptr, (const char * const *)&group_key, &group_key_length, 1)))
238 {
239 return memcached_set_error(*ptr, MEMCACHED_BAD_KEY_PROVIDED, MEMCACHED_AT, memcached_literal_param("A bad group key was provided."));
240 }
241
242 master_server_key= memcached_generate_hash_with_redistribution(ptr, group_key, group_key_length);
243 is_group_key_set= true;
244 }
245
246 /*
247 Here is where we pay for the non-block API. We need to remove any data sitting
248 in the queue before we start our get.
249
250 It might be optimum to bounce the connection if count > some number.
251 */
252 for (uint32_t x= 0; x < memcached_server_count(ptr); x++)
253 {
254 memcached_server_write_instance_st instance=
255 memcached_server_instance_fetch(ptr, x);
256
257 if (memcached_server_response_count(instance))
258 {
259 char buffer[MEMCACHED_DEFAULT_COMMAND_SIZE];
260
261 if (ptr->flags.no_block)
262 {
263 (void)memcached_io_write(instance, NULL, 0, true);
264 }
265
266 while(memcached_server_response_count(instance))
267 (void)memcached_response(instance, buffer, MEMCACHED_DEFAULT_COMMAND_SIZE, &ptr->result);
268 }
269 }
270
271 if (ptr->flags.binary_protocol)
272 {
273 return binary_mget_by_key(ptr, master_server_key, is_group_key_set, keys,
274 key_length, number_of_keys, mget_mode);
275 }
276
277 if (ptr->flags.support_cas)
278 {
279 get_command= "gets ";
280 get_command_length= 5;
281 }
282
283 /*
284 If a server fails we warn about errors and start all over with sending keys
285 to the server.
286 */
287 WATCHPOINT_ASSERT(rc == MEMCACHED_SUCCESS);
288 size_t hosts_connected= 0;
289 for (uint32_t x= 0; x < number_of_keys; x++)
290 {
291 memcached_server_write_instance_st instance;
292 uint32_t server_key;
293
294 if (is_group_key_set)
295 {
296 server_key= master_server_key;
297 }
298 else
299 {
300 server_key= memcached_generate_hash_with_redistribution(ptr, keys[x], key_length[x]);
301 }
302
303 instance= memcached_server_instance_fetch(ptr, server_key);
304
305 struct libmemcached_io_vector_st vector[]=
306 {
307 { get_command, get_command_length },
308 { memcached_array_string(ptr->_namespace), memcached_array_size(ptr->_namespace) },
309 { keys[x], key_length[x] },
310 { memcached_literal_param(" ") }
311 };
312
313
314 if (memcached_server_response_count(instance) == 0)
315 {
316 rc= memcached_connect(instance);
317
318 if (memcached_failed(rc))
319 {
320 memcached_set_error(*instance, rc, MEMCACHED_AT);
321 continue;
322 }
323 hosts_connected++;
324
325 if ((memcached_io_writev(instance, vector, 4, false)) == -1)
326 {
327 failures_occured_in_sending= true;
328 continue;
329 }
330 WATCHPOINT_ASSERT(instance->cursor_active == 0);
331 memcached_server_response_increment(instance);
332 WATCHPOINT_ASSERT(instance->cursor_active == 1);
333 }
334 else
335 {
336 if ((memcached_io_writev(instance, (vector + 1), 3, false)) == -1)
337 {
338 memcached_server_response_reset(instance);
339 failures_occured_in_sending= true;
340 continue;
341 }
342 }
343 }
344
345 if (hosts_connected == 0)
346 {
347 LIBMEMCACHED_MEMCACHED_MGET_END();
348
349 if (memcached_failed(rc))
350 return rc;
351
352 return memcached_set_error(*ptr, MEMCACHED_NO_SERVERS, MEMCACHED_AT);
353 }
354
355
356 /*
357 Should we muddle on if some servers are dead?
358 */
359 bool success_happened= false;
360 for (uint32_t x= 0; x < memcached_server_count(ptr); x++)
361 {
362 memcached_server_write_instance_st instance=
363 memcached_server_instance_fetch(ptr, x);
364
365 if (memcached_server_response_count(instance))
366 {
367 /* We need to do something about non-connnected hosts in the future */
368 if ((memcached_io_write(instance, "\r\n", 2, true)) == -1)
369 {
370 failures_occured_in_sending= true;
371 }
372 else
373 {
374 success_happened= true;
375 }
376 }
377 }
378
379 LIBMEMCACHED_MEMCACHED_MGET_END();
380
381 if (failures_occured_in_sending && success_happened)
382 {
383 return MEMCACHED_SOME_ERRORS;
384 }
385
386 if (success_happened)
387 return MEMCACHED_SUCCESS;
388
389 return MEMCACHED_FAILURE; // Complete failure occurred
390 }
391
392 memcached_return_t memcached_mget_by_key(memcached_st *ptr,
393 const char *group_key,
394 size_t group_key_length,
395 const char * const *keys,
396 const size_t *key_length,
397 size_t number_of_keys)
398 {
399 return memcached_mget_by_key_real(ptr, group_key, group_key_length, keys,
400 key_length, number_of_keys, true);
401 }
402
403 memcached_return_t memcached_mget_execute(memcached_st *ptr,
404 const char * const *keys,
405 const size_t *key_length,
406 size_t number_of_keys,
407 memcached_execute_fn *callback,
408 void *context,
409 unsigned int number_of_callbacks)
410 {
411 return memcached_mget_execute_by_key(ptr, NULL, 0, keys, key_length,
412 number_of_keys, callback,
413 context, number_of_callbacks);
414 }
415
416 memcached_return_t memcached_mget_execute_by_key(memcached_st *ptr,
417 const char *group_key,
418 size_t group_key_length,
419 const char * const *keys,
420 const size_t *key_length,
421 size_t number_of_keys,
422 memcached_execute_fn *callback,
423 void *context,
424 unsigned int number_of_callbacks)
425 {
426 if ((ptr->flags.binary_protocol) == 0)
427 {
428 return MEMCACHED_NOT_SUPPORTED;
429 }
430
431 memcached_return_t rc;
432 memcached_callback_st *original_callbacks= ptr->callbacks;
433 memcached_callback_st cb= {
434 callback,
435 context,
436 number_of_callbacks
437 };
438
439 ptr->callbacks= &cb;
440 rc= memcached_mget_by_key(ptr, group_key, group_key_length, keys,
441 key_length, number_of_keys);
442 ptr->callbacks= original_callbacks;
443 return rc;
444 }
445
446 static memcached_return_t simple_binary_mget(memcached_st *ptr,
447 uint32_t master_server_key,
448 bool is_group_key_set,
449 const char * const *keys,
450 const size_t *key_length,
451 size_t number_of_keys, bool mget_mode)
452 {
453 memcached_return_t rc= MEMCACHED_NOTFOUND;
454
455 bool flush= (number_of_keys == 1);
456
457 /*
458 If a server fails we warn about errors and start all over with sending keys
459 to the server.
460 */
461 for (uint32_t x= 0; x < number_of_keys; ++x)
462 {
463 uint32_t server_key;
464
465 if (is_group_key_set)
466 {
467 server_key= master_server_key;
468 }
469 else
470 {
471 server_key= memcached_generate_hash_with_redistribution(ptr, keys[x], key_length[x]);
472 }
473
474 memcached_server_write_instance_st instance= memcached_server_instance_fetch(ptr, server_key);
475
476 if (memcached_server_response_count(instance) == 0)
477 {
478 rc= memcached_connect(instance);
479 if (memcached_failed(rc))
480 {
481 continue;
482 }
483 }
484
485 protocol_binary_request_getk request= { }; //= {.bytes= {0}};
486 request.message.header.request.magic= PROTOCOL_BINARY_REQ;
487 if (mget_mode)
488 request.message.header.request.opcode= PROTOCOL_BINARY_CMD_GETKQ;
489 else
490 request.message.header.request.opcode= PROTOCOL_BINARY_CMD_GETK;
491
492 memcached_return_t vk;
493 vk= memcached_validate_key_length(key_length[x],
494 ptr->flags.binary_protocol);
495 unlikely (vk != MEMCACHED_SUCCESS)
496 {
497 if (x > 0)
498 {
499 memcached_io_reset(instance);
500 }
501
502 return vk;
503 }
504
505 request.message.header.request.keylen= htons((uint16_t)(key_length[x] + memcached_array_size(ptr->_namespace)));
506 request.message.header.request.datatype= PROTOCOL_BINARY_RAW_BYTES;
507 request.message.header.request.bodylen= htonl((uint32_t)( key_length[x] + memcached_array_size(ptr->_namespace)));
508
509 struct libmemcached_io_vector_st vector[]=
510 {
511 { request.bytes, sizeof(request.bytes) },
512 { memcached_array_string(ptr->_namespace), memcached_array_size(ptr->_namespace) },
513 { keys[x], key_length[x] }
514 };
515
516 if (memcached_io_writev(instance, vector, 3, flush) == -1)
517 {
518 memcached_server_response_reset(instance);
519 rc= MEMCACHED_SOME_ERRORS;
520 continue;
521 }
522
523 /* We just want one pending response per server */
524 memcached_server_response_reset(instance);
525 memcached_server_response_increment(instance);
526 if ((x > 0 && x == ptr->io_key_prefetch) && memcached_flush_buffers(ptr) != MEMCACHED_SUCCESS)
527 {
528 rc= MEMCACHED_SOME_ERRORS;
529 }
530 }
531
532 if (mget_mode)
533 {
534 /*
535 Send a noop command to flush the buffers
536 */
537 protocol_binary_request_noop request= {}; //= {.bytes= {0}};
538 request.message.header.request.magic= PROTOCOL_BINARY_REQ;
539 request.message.header.request.opcode= PROTOCOL_BINARY_CMD_NOOP;
540 request.message.header.request.datatype= PROTOCOL_BINARY_RAW_BYTES;
541
542 for (uint32_t x= 0; x < memcached_server_count(ptr); ++x)
543 {
544 memcached_server_write_instance_st instance=
545 memcached_server_instance_fetch(ptr, x);
546
547 if (memcached_server_response_count(instance))
548 {
549 if (memcached_io_write(instance, NULL, 0, true) == -1)
550 {
551 memcached_server_response_reset(instance);
552 memcached_io_reset(instance);
553 rc= MEMCACHED_SOME_ERRORS;
554 }
555
556 if (memcached_io_write(instance, request.bytes,
557 sizeof(request.bytes), true) == -1)
558 {
559 memcached_server_response_reset(instance);
560 memcached_io_reset(instance);
561 rc= MEMCACHED_SOME_ERRORS;
562 }
563 }
564 }
565 }
566
567
568 return rc;
569 }
570
571 static memcached_return_t replication_binary_mget(memcached_st *ptr,
572 uint32_t* hash,
573 bool* dead_servers,
574 const char *const *keys,
575 const size_t *key_length,
576 size_t number_of_keys)
577 {
578 memcached_return_t rc= MEMCACHED_NOTFOUND;
579 uint32_t start= 0;
580 uint64_t randomize_read= memcached_behavior_get(ptr, MEMCACHED_BEHAVIOR_RANDOMIZE_REPLICA_READ);
581
582 if (randomize_read)
583 start= (uint32_t)random() % (uint32_t)(ptr->number_of_replicas + 1);
584
585 /* Loop for each replica */
586 for (uint32_t replica= 0; replica <= ptr->number_of_replicas; ++replica)
587 {
588 bool success= true;
589
590 for (uint32_t x= 0; x < number_of_keys; ++x)
591 {
592 if (hash[x] == memcached_server_count(ptr))
593 continue; /* Already successfully sent */
594
595 uint32_t server= hash[x] + replica;
596
597 /* In case of randomized reads */
598 if (randomize_read && ((server + start) <= (hash[x] + ptr->number_of_replicas)))
599 server += start;
600
601 while (server >= memcached_server_count(ptr))
602 {
603 server -= memcached_server_count(ptr);
604 }
605
606 if (dead_servers[server])
607 {
608 continue;
609 }
610
611 memcached_server_write_instance_st instance= memcached_server_instance_fetch(ptr, server);
612
613 if (memcached_server_response_count(instance) == 0)
614 {
615 rc= memcached_connect(instance);
616
617 if (memcached_failed(rc))
618 {
619 memcached_io_reset(instance);
620 dead_servers[server]= true;
621 success= false;
622 continue;
623 }
624 }
625
626 protocol_binary_request_getk request= {};
627 request.message.header.request.magic= PROTOCOL_BINARY_REQ;
628 request.message.header.request.opcode= PROTOCOL_BINARY_CMD_GETK;
629 request.message.header.request.keylen= htons((uint16_t)(key_length[x] + memcached_array_size(ptr->_namespace)));
630 request.message.header.request.datatype= PROTOCOL_BINARY_RAW_BYTES;
631 request.message.header.request.bodylen= htonl((uint32_t)(key_length[x] + memcached_array_size(ptr->_namespace)));
632
633 /*
634 * We need to disable buffering to actually know that the request was
635 * successfully sent to the server (so that we should expect a result
636 * back). It would be nice to do this in buffered mode, but then it
637 * would be complex to handle all error situations if we got to send
638 * some of the messages, and then we failed on writing out some others
639 * and we used the callback interface from memcached_mget_execute so
640 * that we might have processed some of the responses etc. For now,
641 * just make sure we work _correctly_
642 */
643 struct libmemcached_io_vector_st vector[]=
644 {
645 { request.bytes, sizeof(request.bytes) },
646 { memcached_array_string(ptr->_namespace), memcached_array_size(ptr->_namespace) },
647 { keys[x], key_length[x] }
648 };
649
650 if (memcached_io_writev(instance, vector, 3, true) == -1)
651 {
652 memcached_io_reset(instance);
653 dead_servers[server]= true;
654 success= false;
655 continue;
656 }
657
658 memcached_server_response_increment(instance);
659 hash[x]= memcached_server_count(ptr);
660 }
661
662 if (success)
663 {
664 break;
665 }
666 }
667
668 return rc;
669 }
670
671 static memcached_return_t binary_mget_by_key(memcached_st *ptr,
672 uint32_t master_server_key,
673 bool is_group_key_set,
674 const char * const *keys,
675 const size_t *key_length,
676 size_t number_of_keys,
677 bool mget_mode)
678 {
679 if (ptr->number_of_replicas == 0)
680 {
681 return simple_binary_mget(ptr, master_server_key, is_group_key_set,
682 keys, key_length, number_of_keys, mget_mode);
683 }
684
685 uint32_t* hash= libmemcached_xvalloc(ptr, number_of_keys, uint32_t);
686 bool* dead_servers= libmemcached_xcalloc(ptr, memcached_server_count(ptr), bool);
687
688 if (hash == NULL || dead_servers == NULL)
689 {
690 libmemcached_free(ptr, hash);
691 libmemcached_free(ptr, dead_servers);
692 return MEMCACHED_MEMORY_ALLOCATION_FAILURE;
693 }
694
695 if (is_group_key_set)
696 {
697 for (size_t x= 0; x < number_of_keys; x++)
698 {
699 hash[x]= master_server_key;
700 }
701 }
702 else
703 {
704 for (size_t x= 0; x < number_of_keys; x++)
705 {
706 hash[x]= memcached_generate_hash_with_redistribution(ptr, keys[x], key_length[x]);
707 }
708 }
709
710 memcached_return_t rc= replication_binary_mget(ptr, hash, dead_servers, keys,
711 key_length, number_of_keys);
712
713 WATCHPOINT_IFERROR(rc);
714 libmemcached_free(ptr, hash);
715 libmemcached_free(ptr, dead_servers);
716
717 return MEMCACHED_SUCCESS;
718 }