projects / m6w6 / libmemcached / blob
? search:
summary | shortlog | log | commit | commitdiff | tree | github
history | raw | HEAD
Style cleanup
[m6w6/libmemcached] / libmemcached / get.cc
1 /* vim:expandtab:shiftwidth=2:tabstop=2:smarttab:
2 *
3 * Libmemcached library
4 *
5 * Copyright (C) 2011 Data Differential, http://datadifferential.com/
6 * Copyright (C) 2006-2009 Brian Aker All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions are
10 * met:
11 *
12 * * Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 *
15 * * Redistributions in binary form must reproduce the above
16 * copyright notice, this list of conditions and the following disclaimer
17 * in the documentation and/or other materials provided with the
18 * distribution.
19 *
20 * * The names of its contributors may not be used to endorse or
21 * promote products derived from this software without specific prior
22 * written permission.
23 *
24 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
25 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
26 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
27 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
28 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
29 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
30 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
31 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
32 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
33 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
34 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
35 *
36 */
37
38 #include "libmemcached/common.h"
39 #include <iostream>
40
41 /*
42 What happens if no servers exist?
43 */
44 char *memcached_get(memcached_st *ptr, const char *key,
45 size_t key_length,
46 size_t *value_length,
47 uint32_t *flags,
48 memcached_return_t *error)
49 {
50 return memcached_get_by_key(ptr, NULL, 0, key, key_length, value_length,
51 flags, error);
52 }
53
54 static memcached_return_t memcached_mget_by_key_real(memcached_st *ptr,
55 const char *group_key,
56 size_t group_key_length,
57 const char * const *keys,
58 const size_t *key_length,
59 size_t number_of_keys,
60 bool mget_mode);
61
62 char *memcached_get_by_key(memcached_st *ptr,
63 const char *group_key,
64 size_t group_key_length,
65 const char *key, size_t key_length,
66 size_t *value_length,
67 uint32_t *flags,
68 memcached_return_t *error)
69 {
70 unlikely (ptr->flags.use_udp)
71 {
72 *error= MEMCACHED_NOT_SUPPORTED;
73 return NULL;
74 }
75
76 /* Request the key */
77 *error= memcached_mget_by_key_real(ptr, group_key, group_key_length,
78 (const char * const *)&key,
79 &key_length, 1, false);
80
81 char *value= memcached_fetch(ptr, NULL, NULL,
82 value_length, flags, error);
83 /* This is for historical reasons */
84 if (*error == MEMCACHED_END)
85 *error= MEMCACHED_NOTFOUND;
86
87 if (value == NULL)
88 {
89 if (ptr->get_key_failure && *error == MEMCACHED_NOTFOUND)
90 {
91
92 memcached_result_reset(&ptr->result);
93 memcached_return_t rc= ptr->get_key_failure(ptr, key, key_length, &ptr->result);
94
95 /* On all failure drop to returning NULL */
96 if (rc == MEMCACHED_SUCCESS || rc == MEMCACHED_BUFFERED)
97 {
98 if (rc == MEMCACHED_BUFFERED)
99 {
100 uint64_t latch; /* We use latch to track the state of the original socket */
101 latch= memcached_behavior_get(ptr, MEMCACHED_BEHAVIOR_BUFFER_REQUESTS);
102 if (latch == 0)
103 memcached_behavior_set(ptr, MEMCACHED_BEHAVIOR_BUFFER_REQUESTS, 1);
104
105 rc= memcached_set(ptr, key, key_length,
106 (memcached_result_value(&ptr->result)),
107 (memcached_result_length(&ptr->result)),
108 0,
109 (memcached_result_flags(&ptr->result)));
110
111 if (rc == MEMCACHED_BUFFERED && latch == 0)
112 memcached_behavior_set(ptr, MEMCACHED_BEHAVIOR_BUFFER_REQUESTS, 0);
113 }
114 else
115 {
116 rc= memcached_set(ptr, key, key_length,
117 (memcached_result_value(&ptr->result)),
118 (memcached_result_length(&ptr->result)),
119 0,
120 (memcached_result_flags(&ptr->result)));
121 }
122
123 if (rc == MEMCACHED_SUCCESS || rc == MEMCACHED_BUFFERED)
124 {
125 *error= rc;
126 *value_length= memcached_result_length(&ptr->result);
127 *flags= memcached_result_flags(&ptr->result);
128 return memcached_string_c_copy(&ptr->result.value);
129 }
130 }
131 }
132
133 return NULL;
134 }
135
136 size_t dummy_length;
137 uint32_t dummy_flags;
138 memcached_return_t dummy_error;
139
140 char *dummy_value= memcached_fetch(ptr, NULL, NULL,
141 &dummy_length, &dummy_flags,
142 &dummy_error);
143 WATCHPOINT_ASSERT(dummy_length == 0);
144 WATCHPOINT_ASSERT(dummy_value == 0);
145
146 return value;
147 }
148
149 memcached_return_t memcached_mget(memcached_st *ptr,
150 const char * const *keys,
151 const size_t *key_length,
152 size_t number_of_keys)
153 {
154 return memcached_mget_by_key(ptr, NULL, 0, keys, key_length, number_of_keys);
155 }
156
157 static memcached_return_t binary_mget_by_key(memcached_st *ptr,
158 uint32_t master_server_key,
159 bool is_group_key_set,
160 const char * const *keys,
161 const size_t *key_length,
162 size_t number_of_keys,
163 bool mget_mode);
164
165 static memcached_return_t memcached_mget_by_key_real(memcached_st *ptr,
166 const char *group_key,
167 size_t group_key_length,
168 const char * const *keys,
169 const size_t *key_length,
170 size_t number_of_keys,
171 bool mget_mode)
172 {
173 bool failures_occured_in_sending= false;
174 const char *get_command= "get ";
175 uint8_t get_command_length= 4;
176 unsigned int master_server_key= (unsigned int)-1; /* 0 is a valid server id! */
177 bool is_group_key_set= false;
178
179 memcached_return_t rc;
180 if (memcached_failed(rc= initialize_query(ptr)))
181 {
182 return rc;
183 }
184
185 unlikely (ptr->flags.use_udp)
186 return MEMCACHED_NOT_SUPPORTED;
187
188 LIBMEMCACHED_MEMCACHED_MGET_START();
189
190 if (number_of_keys == 0)
191 return MEMCACHED_NOTFOUND;
192
193 if (ptr->flags.verify_key && (memcached_key_test(keys, key_length, number_of_keys) == MEMCACHED_BAD_KEY_PROVIDED))
194 {
195 return MEMCACHED_BAD_KEY_PROVIDED;
196 }
197
198 if (group_key && group_key_length)
199 {
200 if (ptr->flags.verify_key and (memcached_key_test((const char * const *)&group_key, &group_key_length, 1) == MEMCACHED_BAD_KEY_PROVIDED))
201 return MEMCACHED_BAD_KEY_PROVIDED;
202
203 master_server_key= memcached_generate_hash_with_redistribution(ptr, group_key, group_key_length);
204 is_group_key_set= true;
205 }
206
207 /*
208 Here is where we pay for the non-block API. We need to remove any data sitting
209 in the queue before we start our get.
210
211 It might be optimum to bounce the connection if count > some number.
212 */
213 for (uint32_t x= 0; x < memcached_server_count(ptr); x++)
214 {
215 memcached_server_write_instance_st instance=
216 memcached_server_instance_fetch(ptr, x);
217
218 if (memcached_server_response_count(instance))
219 {
220 char buffer[MEMCACHED_DEFAULT_COMMAND_SIZE];
221
222 if (ptr->flags.no_block)
223 (void)memcached_io_write(instance, NULL, 0, true);
224
225 while(memcached_server_response_count(instance))
226 (void)memcached_response(instance, buffer, MEMCACHED_DEFAULT_COMMAND_SIZE, &ptr->result);
227 }
228 }
229
230 if (ptr->flags.binary_protocol)
231 {
232 return binary_mget_by_key(ptr, master_server_key, is_group_key_set, keys,
233 key_length, number_of_keys, mget_mode);
234 }
235
236 if (ptr->flags.support_cas)
237 {
238 get_command= "gets ";
239 get_command_length= 5;
240 }
241
242 /*
243 If a server fails we warn about errors and start all over with sending keys
244 to the server.
245 */
246 WATCHPOINT_ASSERT(rc == MEMCACHED_SUCCESS);
247 size_t hosts_connected= 0;
248 for (uint32_t x= 0; x < number_of_keys; x++)
249 {
250 memcached_server_write_instance_st instance;
251 uint32_t server_key;
252
253 if (is_group_key_set)
254 {
255 server_key= master_server_key;
256 }
257 else
258 {
259 server_key= memcached_generate_hash_with_redistribution(ptr, keys[x], key_length[x]);
260 }
261
262 instance= memcached_server_instance_fetch(ptr, server_key);
263
264 struct libmemcached_io_vector_st vector[]=
265 {
266 { get_command_length, get_command },
267 { memcached_array_size(ptr->prefix_key), memcached_array_string(ptr->prefix_key) },
268 { key_length[x], keys[x] },
269 { 1, " " }
270 };
271
272
273 if (memcached_server_response_count(instance) == 0)
274 {
275 rc= memcached_connect(instance);
276
277 if (rc != MEMCACHED_SUCCESS)
278 {
279 continue;
280 }
281 hosts_connected++;
282
283 if ((memcached_io_writev(instance, vector, 4, false)) == -1)
284 {
285 failures_occured_in_sending= true;
286 continue;
287 }
288 WATCHPOINT_ASSERT(instance->cursor_active == 0);
289 memcached_server_response_increment(instance);
290 WATCHPOINT_ASSERT(instance->cursor_active == 1);
291 }
292 else
293 {
294 if ((memcached_io_writev(instance, (vector + 1), 3, false)) == -1)
295 {
296 memcached_server_response_reset(instance);
297 failures_occured_in_sending= true;
298 continue;
299 }
300 }
301 }
302
303 if (hosts_connected == 0)
304 {
305 LIBMEMCACHED_MEMCACHED_MGET_END();
306
307 if (rc != MEMCACHED_SUCCESS)
308 return rc;
309
310 return MEMCACHED_NO_SERVERS;
311 }
312
313
314 /*
315 Should we muddle on if some servers are dead?
316 */
317 bool success_happened= false;
318 for (uint32_t x= 0; x < memcached_server_count(ptr); x++)
319 {
320 memcached_server_write_instance_st instance=
321 memcached_server_instance_fetch(ptr, x);
322
323 if (memcached_server_response_count(instance))
324 {
325 /* We need to do something about non-connnected hosts in the future */
326 if ((memcached_io_write(instance, "\r\n", 2, true)) == -1)
327 {
328 failures_occured_in_sending= true;
329 }
330 else
331 {
332 success_happened= true;
333 }
334 }
335 }
336
337 LIBMEMCACHED_MEMCACHED_MGET_END();
338
339 if (failures_occured_in_sending && success_happened)
340 return MEMCACHED_SOME_ERRORS;
341
342 if (success_happened)
343 return MEMCACHED_SUCCESS;
344
345 return MEMCACHED_FAILURE;
346 }
347
348 memcached_return_t memcached_mget_by_key(memcached_st *ptr,
349 const char *group_key,
350 size_t group_key_length,
351 const char * const *keys,
352 const size_t *key_length,
353 size_t number_of_keys)
354 {
355 return memcached_mget_by_key_real(ptr, group_key, group_key_length, keys,
356 key_length, number_of_keys, true);
357 }
358
359 memcached_return_t memcached_mget_execute(memcached_st *ptr,
360 const char * const *keys,
361 const size_t *key_length,
362 size_t number_of_keys,
363 memcached_execute_fn *callback,
364 void *context,
365 unsigned int number_of_callbacks)
366 {
367 return memcached_mget_execute_by_key(ptr, NULL, 0, keys, key_length,
368 number_of_keys, callback,
369 context, number_of_callbacks);
370 }
371
372 memcached_return_t memcached_mget_execute_by_key(memcached_st *ptr,
373 const char *group_key,
374 size_t group_key_length,
375 const char * const *keys,
376 const size_t *key_length,
377 size_t number_of_keys,
378 memcached_execute_fn *callback,
379 void *context,
380 unsigned int number_of_callbacks)
381 {
382 if ((ptr->flags.binary_protocol) == 0)
383 return MEMCACHED_NOT_SUPPORTED;
384
385 memcached_return_t rc;
386 memcached_callback_st *original_callbacks= ptr->callbacks;
387 memcached_callback_st cb= {
388 callback,
389 context,
390 number_of_callbacks
391 };
392
393 ptr->callbacks= &cb;
394 rc= memcached_mget_by_key(ptr, group_key, group_key_length, keys,
395 key_length, number_of_keys);
396 ptr->callbacks= original_callbacks;
397 return rc;
398 }
399
400 static memcached_return_t simple_binary_mget(memcached_st *ptr,
401 uint32_t master_server_key,
402 bool is_group_key_set,
403 const char * const *keys,
404 const size_t *key_length,
405 size_t number_of_keys, bool mget_mode)
406 {
407 memcached_return_t rc= MEMCACHED_NOTFOUND;
408
409 bool flush= (number_of_keys == 1);
410
411 /*
412 If a server fails we warn about errors and start all over with sending keys
413 to the server.
414 */
415 for (uint32_t x= 0; x < number_of_keys; ++x)
416 {
417 uint32_t server_key;
418 memcached_server_write_instance_st instance;
419
420 if (is_group_key_set)
421 {
422 server_key= master_server_key;
423 }
424 else
425 {
426 server_key= memcached_generate_hash_with_redistribution(ptr, keys[x], key_length[x]);
427 }
428
429 instance= memcached_server_instance_fetch(ptr, server_key);
430
431 if (memcached_server_response_count(instance) == 0)
432 {
433 rc= memcached_connect(instance);
434 if (rc != MEMCACHED_SUCCESS)
435 continue;
436 }
437
438 protocol_binary_request_getk request= { }; //= {.bytes= {0}};
439 request.message.header.request.magic= PROTOCOL_BINARY_REQ;
440 if (mget_mode)
441 request.message.header.request.opcode= PROTOCOL_BINARY_CMD_GETKQ;
442 else
443 request.message.header.request.opcode= PROTOCOL_BINARY_CMD_GETK;
444
445 memcached_return_t vk;
446 vk= memcached_validate_key_length(key_length[x],
447 ptr->flags.binary_protocol);
448 unlikely (vk != MEMCACHED_SUCCESS)
449 {
450 if (x > 0)
451 {
452 memcached_io_reset(instance);
453 }
454
455 return vk;
456 }
457
458 request.message.header.request.keylen= htons((uint16_t)(key_length[x] + memcached_array_size(ptr->prefix_key)));
459 request.message.header.request.datatype= PROTOCOL_BINARY_RAW_BYTES;
460 request.message.header.request.bodylen= htonl((uint32_t)( key_length[x] + memcached_array_size(ptr->prefix_key)));
461
462 struct libmemcached_io_vector_st vector[]=
463 {
464 { sizeof(request.bytes), request.bytes },
465 { memcached_array_size(ptr->prefix_key), memcached_array_string(ptr->prefix_key) },
466 { key_length[x], keys[x] }
467 };
468
469 if (memcached_io_writev(instance, vector, 3, flush) == -1)
470 {
471 memcached_server_response_reset(instance);
472 rc= MEMCACHED_SOME_ERRORS;
473 continue;
474 }
475
476 /* We just want one pending response per server */
477 memcached_server_response_reset(instance);
478 memcached_server_response_increment(instance);
479 if ((x > 0 && x == ptr->io_key_prefetch) && memcached_flush_buffers(ptr) != MEMCACHED_SUCCESS)
480 {
481 rc= MEMCACHED_SOME_ERRORS;
482 }
483 }
484
485 if (mget_mode)
486 {
487 /*
488 Send a noop command to flush the buffers
489 */
490 protocol_binary_request_noop request= {}; //= {.bytes= {0}};
491 request.message.header.request.magic= PROTOCOL_BINARY_REQ;
492 request.message.header.request.opcode= PROTOCOL_BINARY_CMD_NOOP;
493 request.message.header.request.datatype= PROTOCOL_BINARY_RAW_BYTES;
494
495 for (uint32_t x= 0; x < memcached_server_count(ptr); ++x)
496 {
497 memcached_server_write_instance_st instance=
498 memcached_server_instance_fetch(ptr, x);
499
500 if (memcached_server_response_count(instance))
501 {
502 if (memcached_io_write(instance, NULL, 0, true) == -1)
503 {
504 memcached_server_response_reset(instance);
505 memcached_io_reset(instance);
506 rc= MEMCACHED_SOME_ERRORS;
507 }
508
509 if (memcached_io_write(instance, request.bytes,
510 sizeof(request.bytes), true) == -1)
511 {
512 memcached_server_response_reset(instance);
513 memcached_io_reset(instance);
514 rc= MEMCACHED_SOME_ERRORS;
515 }
516 }
517 }
518 }
519
520
521 return rc;
522 }
523
524 static memcached_return_t replication_binary_mget(memcached_st *ptr,
525 uint32_t* hash,
526 bool* dead_servers,
527 const char *const *keys,
528 const size_t *key_length,
529 size_t number_of_keys)
530 {
531 memcached_return_t rc= MEMCACHED_NOTFOUND;
532 uint32_t start= 0;
533 uint64_t randomize_read= memcached_behavior_get(ptr, MEMCACHED_BEHAVIOR_RANDOMIZE_REPLICA_READ);
534
535 if (randomize_read)
536 start= (uint32_t)random() % (uint32_t)(ptr->number_of_replicas + 1);
537
538 /* Loop for each replica */
539 for (uint32_t replica= 0; replica <= ptr->number_of_replicas; ++replica)
540 {
541 bool success= true;
542
543 for (uint32_t x= 0; x < number_of_keys; ++x)
544 {
545 if (hash[x] == memcached_server_count(ptr))
546 continue; /* Already successfully sent */
547
548 uint32_t server= hash[x] + replica;
549
550 /* In case of randomized reads */
551 if (randomize_read && ((server + start) <= (hash[x] + ptr->number_of_replicas)))
552 server += start;
553
554 while (server >= memcached_server_count(ptr))
555 server -= memcached_server_count(ptr);
556
557 if (dead_servers[server])
558 continue;
559
560 memcached_server_write_instance_st instance= memcached_server_instance_fetch(ptr, server);
561
562 if (memcached_server_response_count(instance) == 0)
563 {
564 rc= memcached_connect(instance);
565 if (memcached_failed(rc))
566 {
567 memcached_io_reset(instance);
568 dead_servers[server]= true;
569 success= false;
570 continue;
571 }
572 }
573
574 protocol_binary_request_getk request= {};
575 request.message.header.request.magic= PROTOCOL_BINARY_REQ;
576 request.message.header.request.opcode= PROTOCOL_BINARY_CMD_GETK;
577 request.message.header.request.keylen= htons((uint16_t)(key_length[x] + memcached_array_size(ptr->prefix_key)));
578 request.message.header.request.datatype= PROTOCOL_BINARY_RAW_BYTES;
579 request.message.header.request.bodylen= htonl((uint32_t)(key_length[x] + memcached_array_size(ptr->prefix_key)));
580
581 /*
582 * We need to disable buffering to actually know that the request was
583 * successfully sent to the server (so that we should expect a result
584 * back). It would be nice to do this in buffered mode, but then it
585 * would be complex to handle all error situations if we got to send
586 * some of the messages, and then we failed on writing out some others
587 * and we used the callback interface from memcached_mget_execute so
588 * that we might have processed some of the responses etc. For now,
589 * just make sure we work _correctly_
590 */
591 struct libmemcached_io_vector_st vector[]=
592 {
593 { sizeof(request.bytes), request.bytes },
594 { memcached_array_size(ptr->prefix_key), memcached_array_string(ptr->prefix_key) },
595 { key_length[x], keys[x] }
596 };
597
598 if (memcached_io_writev(instance, vector, 3, true) == -1)
599 {
600 memcached_io_reset(instance);
601 dead_servers[server]= true;
602 success= false;
603 continue;
604 }
605
606 memcached_server_response_increment(instance);
607 hash[x]= memcached_server_count(ptr);
608 }
609
610 if (success)
611 break;
612 }
613
614 return rc;
615 }
616
617 static memcached_return_t binary_mget_by_key(memcached_st *ptr,
618 uint32_t master_server_key,
619 bool is_group_key_set,
620 const char * const *keys,
621 const size_t *key_length,
622 size_t number_of_keys,
623 bool mget_mode)
624 {
625 if (ptr->number_of_replicas == 0)
626 {
627 return simple_binary_mget(ptr, master_server_key, is_group_key_set,
628 keys, key_length, number_of_keys, mget_mode);
629 }
630
631 uint32_t* hash= static_cast<uint32_t*>(libmemcached_malloc(ptr, sizeof(uint32_t) * number_of_keys));
632 bool* dead_servers= static_cast<bool*>(libmemcached_calloc(ptr, memcached_server_count(ptr), sizeof(bool)));
633
634 if (hash == NULL || dead_servers == NULL)
635 {
636 libmemcached_free(ptr, hash);
637 libmemcached_free(ptr, dead_servers);
638 return MEMCACHED_MEMORY_ALLOCATION_FAILURE;
639 }
640
641 if (is_group_key_set)
642 {
643 for (size_t x= 0; x < number_of_keys; x++)
644 {
645 hash[x]= master_server_key;
646 }
647 }
648 else
649 {
650 for (size_t x= 0; x < number_of_keys; x++)
651 {
652 hash[x]= memcached_generate_hash_with_redistribution(ptr, keys[x], key_length[x]);
653 }
654 }
655
656 memcached_return_t rc= replication_binary_mget(ptr, hash, dead_servers, keys,
657 key_length, number_of_keys);
658
659 WATCHPOINT_IFERROR(rc);
660 libmemcached_free(ptr, hash);
661 libmemcached_free(ptr, dead_servers);
662
663 return MEMCACHED_SUCCESS;
664 }
Resurrection of libmemcached https://launchpad.net/libmemcached