Mass rename to simplify names.
[m6w6/libmemcached] / libmemcached / get.c
1 /* LibMemcached
2 * Copyright (C) 2006-2009 Brian Aker
3 * All rights reserved.
4 *
5 * Use and distribution licensed under the BSD license. See
6 * the COPYING file in the parent directory for full text.
7 *
8 * Summary: Get functions for libmemcached
9 *
10 */
11
12 #include "common.h"
13
14 /*
15 What happens if no servers exist?
16 */
17 char *memcached_get(memcached_st *ptr, const char *key,
18 size_t key_length,
19 size_t *value_length,
20 uint32_t *flags,
21 memcached_return_t *error)
22 {
23 return memcached_get_by_key(ptr, NULL, 0, key, key_length, value_length,
24 flags, error);
25 }
26
27 static memcached_return_t memcached_mget_by_key_real(memcached_st *ptr,
28 const char *master_key,
29 size_t master_key_length,
30 const char * const *keys,
31 const size_t *key_length,
32 size_t number_of_keys,
33 bool mget_mode);
34
35 char *memcached_get_by_key(memcached_st *ptr,
36 const char *master_key,
37 size_t master_key_length,
38 const char *key, size_t key_length,
39 size_t *value_length,
40 uint32_t *flags,
41 memcached_return_t *error)
42 {
43 char *value;
44 size_t dummy_length;
45 uint32_t dummy_flags;
46 memcached_return_t dummy_error;
47
48 unlikely (ptr->flags.use_udp)
49 {
50 *error= MEMCACHED_NOT_SUPPORTED;
51 return NULL;
52 }
53
54 /* Request the key */
55 *error= memcached_mget_by_key_real(ptr, master_key, master_key_length,
56 (const char * const *)&key,
57 &key_length, 1, false);
58
59 value= memcached_fetch(ptr, NULL, NULL,
60 value_length, flags, error);
61 /* This is for historical reasons */
62 if (*error == MEMCACHED_END)
63 *error= MEMCACHED_NOTFOUND;
64
65 if (value == NULL)
66 {
67 if (ptr->get_key_failure && *error == MEMCACHED_NOTFOUND)
68 {
69 memcached_return_t rc;
70
71 memcached_result_reset(&ptr->result);
72 rc= ptr->get_key_failure(ptr, key, key_length, &ptr->result);
73
74 /* On all failure drop to returning NULL */
75 if (rc == MEMCACHED_SUCCESS || rc == MEMCACHED_BUFFERED)
76 {
77 if (rc == MEMCACHED_BUFFERED)
78 {
79 uint64_t latch; /* We use latch to track the state of the original socket */
80 latch= memcached_behavior_get(ptr, MEMCACHED_BEHAVIOR_BUFFER_REQUESTS);
81 if (latch == 0)
82 memcached_behavior_set(ptr, MEMCACHED_BEHAVIOR_BUFFER_REQUESTS, 1);
83
84 rc= memcached_set(ptr, key, key_length,
85 memcached_result_value(&ptr->result),
86 memcached_result_length(&ptr->result),
87 0, memcached_result_flags(&ptr->result));
88
89 if (rc == MEMCACHED_BUFFERED && latch == 0)
90 memcached_behavior_set(ptr, MEMCACHED_BEHAVIOR_BUFFER_REQUESTS, 0);
91 }
92 else
93 {
94 rc= memcached_set(ptr, key, key_length,
95 memcached_result_value(&ptr->result),
96 memcached_result_length(&ptr->result),
97 0, memcached_result_flags(&ptr->result));
98 }
99
100 if (rc == MEMCACHED_SUCCESS || rc == MEMCACHED_BUFFERED)
101 {
102 *error= rc;
103 *value_length= memcached_result_length(&ptr->result);
104 *flags= memcached_result_flags(&ptr->result);
105 return memcached_string_c_copy(&ptr->result.value);
106 }
107 }
108 }
109
110 return NULL;
111 }
112
113 (void)memcached_fetch(ptr, NULL, NULL,
114 &dummy_length, &dummy_flags,
115 &dummy_error);
116 WATCHPOINT_ASSERT(dummy_length == 0);
117
118 return value;
119 }
120
121 memcached_return_t memcached_mget(memcached_st *ptr,
122 const char * const *keys,
123 const size_t *key_length,
124 size_t number_of_keys)
125 {
126 return memcached_mget_by_key(ptr, NULL, 0, keys, key_length, number_of_keys);
127 }
128
129 static memcached_return_t binary_mget_by_key(memcached_st *ptr,
130 unsigned int master_server_key,
131 bool is_master_key_set,
132 const char * const *keys,
133 const size_t *key_length,
134 size_t number_of_keys,
135 bool mget_mode);
136
137 static memcached_return_t memcached_mget_by_key_real(memcached_st *ptr,
138 const char *master_key,
139 size_t master_key_length,
140 const char * const *keys,
141 const size_t *key_length,
142 size_t number_of_keys,
143 bool mget_mode)
144 {
145 unsigned int x;
146 memcached_return_t rc= MEMCACHED_NOTFOUND;
147 const char *get_command= "get ";
148 uint8_t get_command_length= 4;
149 unsigned int master_server_key= (unsigned int)-1; /* 0 is a valid server id! */
150 bool is_master_key_set= false;
151
152 unlikely (ptr->flags.use_udp)
153 return MEMCACHED_NOT_SUPPORTED;
154
155 LIBMEMCACHED_MEMCACHED_MGET_START();
156 ptr->cursor_server= 0;
157
158 if (number_of_keys == 0)
159 return MEMCACHED_NOTFOUND;
160
161 if (ptr->number_of_hosts == 0)
162 return MEMCACHED_NO_SERVERS;
163
164 if (ptr->flags.verify_key && (memcached_key_test(keys, key_length, number_of_keys) == MEMCACHED_BAD_KEY_PROVIDED))
165 return MEMCACHED_BAD_KEY_PROVIDED;
166
167 if (master_key && master_key_length)
168 {
169 if (ptr->flags.verify_key && (memcached_key_test((const char * const *)&master_key, &master_key_length, 1) == MEMCACHED_BAD_KEY_PROVIDED))
170 return MEMCACHED_BAD_KEY_PROVIDED;
171 master_server_key= memcached_generate_hash(ptr, master_key, master_key_length);
172 is_master_key_set= true;
173 }
174
175 /*
176 Here is where we pay for the non-block API. We need to remove any data sitting
177 in the queue before we start our get.
178
179 It might be optimum to bounce the connection if count > some number.
180 */
181 for (x= 0; x < ptr->number_of_hosts; x++)
182 {
183 if (memcached_server_response_count(&ptr->hosts[x]))
184 {
185 char buffer[MEMCACHED_DEFAULT_COMMAND_SIZE];
186
187 if (ptr->flags.no_block)
188 (void)memcached_io_write(&ptr->hosts[x], NULL, 0, 1);
189
190 while(memcached_server_response_count(&ptr->hosts[x]))
191 (void)memcached_response(&ptr->hosts[x], buffer, MEMCACHED_DEFAULT_COMMAND_SIZE, &ptr->result);
192 }
193 }
194
195 if (ptr->flags.binary_protocol)
196 return binary_mget_by_key(ptr, master_server_key, is_master_key_set, keys,
197 key_length, number_of_keys, mget_mode);
198
199 if (ptr->flags.support_cas)
200 {
201 get_command= "gets ";
202 get_command_length= 5;
203 }
204
205 /*
206 If a server fails we warn about errors and start all over with sending keys
207 to the server.
208 */
209 for (x= 0; x < number_of_keys; x++)
210 {
211 unsigned int server_key;
212
213 if (is_master_key_set)
214 server_key= master_server_key;
215 else
216 server_key= memcached_generate_hash(ptr, keys[x], key_length[x]);
217
218 if (memcached_server_response_count(&ptr->hosts[server_key]) == 0)
219 {
220 rc= memcached_connect(&ptr->hosts[server_key]);
221
222 if (rc != MEMCACHED_SUCCESS)
223 continue;
224
225 if ((memcached_io_write(&ptr->hosts[server_key], get_command, get_command_length, 0)) == -1)
226 {
227 rc= MEMCACHED_SOME_ERRORS;
228 continue;
229 }
230 WATCHPOINT_ASSERT(ptr->hosts[server_key].cursor_active == 0);
231 memcached_server_response_increment(&ptr->hosts[server_key]);
232 WATCHPOINT_ASSERT(ptr->hosts[server_key].cursor_active == 1);
233 }
234
235 /* Only called when we have a prefix key */
236 if (ptr->prefix_key[0] != 0)
237 {
238 if ((memcached_io_write(&ptr->hosts[server_key], ptr->prefix_key, ptr->prefix_key_length, 0)) == -1)
239 {
240 memcached_server_response_reset(&ptr->hosts[server_key]);
241 rc= MEMCACHED_SOME_ERRORS;
242 continue;
243 }
244 }
245
246 if ((memcached_io_write(&ptr->hosts[server_key], keys[x], key_length[x], 0)) == -1)
247 {
248 memcached_server_response_reset(&ptr->hosts[server_key]);
249 rc= MEMCACHED_SOME_ERRORS;
250 continue;
251 }
252
253 if ((memcached_io_write(&ptr->hosts[server_key], " ", 1, 0)) == -1)
254 {
255 memcached_server_response_reset(&ptr->hosts[server_key]);
256 rc= MEMCACHED_SOME_ERRORS;
257 continue;
258 }
259 }
260
261 /*
262 Should we muddle on if some servers are dead?
263 */
264 for (x= 0; x < ptr->number_of_hosts; x++)
265 {
266 if (memcached_server_response_count(&ptr->hosts[x]))
267 {
268 /* We need to do something about non-connnected hosts in the future */
269 if ((memcached_io_write(&ptr->hosts[x], "\r\n", 2, 1)) == -1)
270 {
271 rc= MEMCACHED_SOME_ERRORS;
272 }
273 }
274 }
275
276 LIBMEMCACHED_MEMCACHED_MGET_END();
277 return rc;
278 }
279
280 memcached_return_t memcached_mget_by_key(memcached_st *ptr,
281 const char *master_key,
282 size_t master_key_length,
283 const char * const *keys,
284 const size_t *key_length,
285 size_t number_of_keys)
286 {
287 return memcached_mget_by_key_real(ptr, master_key, master_key_length, keys,
288 key_length, number_of_keys, true);
289 }
290
291 memcached_return_t memcached_mget_execute(memcached_st *ptr,
292 const char * const *keys,
293 const size_t *key_length,
294 size_t number_of_keys,
295 memcached_execute_fn *callback,
296 void *context,
297 unsigned int number_of_callbacks)
298 {
299 return memcached_mget_execute_by_key(ptr, NULL, 0, keys, key_length,
300 number_of_keys, callback,
301 context, number_of_callbacks);
302 }
303
304 memcached_return_t memcached_mget_execute_by_key(memcached_st *ptr,
305 const char *master_key,
306 size_t master_key_length,
307 const char * const *keys,
308 const size_t *key_length,
309 size_t number_of_keys,
310 memcached_execute_fn *callback,
311 void *context,
312 unsigned int number_of_callbacks)
313 {
314 if ((ptr->flags.binary_protocol) == 0)
315 return MEMCACHED_NOT_SUPPORTED;
316
317 memcached_return_t rc;
318 memcached_callback_st *original_callbacks= ptr->callbacks;
319 memcached_callback_st cb= {
320 .callback= callback,
321 .context= context,
322 .number_of_callback= number_of_callbacks
323 };
324
325 ptr->callbacks= &cb;
326 rc= memcached_mget_by_key(ptr, master_key, master_key_length, keys,
327 key_length, number_of_keys);
328 ptr->callbacks= original_callbacks;
329 return rc;
330 }
331
332 static memcached_return_t simple_binary_mget(memcached_st *ptr,
333 unsigned int master_server_key,
334 bool is_master_key_set,
335 const char * const *keys,
336 const size_t *key_length,
337 size_t number_of_keys, bool mget_mode)
338 {
339 memcached_return_t rc= MEMCACHED_NOTFOUND;
340 uint32_t x;
341
342 int flush= number_of_keys == 1;
343
344 /*
345 If a server fails we warn about errors and start all over with sending keys
346 to the server.
347 */
348 for (x= 0; x < number_of_keys; x++)
349 {
350 unsigned int server_key;
351
352 if (is_master_key_set)
353 server_key= master_server_key;
354 else
355 server_key= memcached_generate_hash(ptr, keys[x], key_length[x]);
356
357 if (memcached_server_response_count(&ptr->hosts[server_key]) == 0)
358 {
359 rc= memcached_connect(&ptr->hosts[server_key]);
360 if (rc != MEMCACHED_SUCCESS)
361 continue;
362 }
363
364 protocol_binary_request_getk request= {.bytes= {0}};
365 request.message.header.request.magic= PROTOCOL_BINARY_REQ;
366 if (mget_mode)
367 request.message.header.request.opcode= PROTOCOL_BINARY_CMD_GETKQ;
368 else
369 request.message.header.request.opcode= PROTOCOL_BINARY_CMD_GETK;
370
371 memcached_return_t vk;
372 vk= memcached_validate_key_length(key_length[x],
373 ptr->flags.binary_protocol);
374 unlikely (vk != MEMCACHED_SUCCESS)
375 {
376 if (x > 0)
377 memcached_io_reset(&ptr->hosts[server_key]);
378 return vk;
379 }
380
381 request.message.header.request.keylen= htons((uint16_t)key_length[x]);
382 request.message.header.request.datatype= PROTOCOL_BINARY_RAW_BYTES;
383 request.message.header.request.bodylen= htonl((uint32_t) key_length[x]);
384
385 if ((memcached_io_write(&ptr->hosts[server_key], request.bytes,
386 sizeof(request.bytes), 0) == -1) ||
387 (memcached_io_write(&ptr->hosts[server_key], keys[x],
388 key_length[x], (char) flush) == -1))
389 {
390 memcached_server_response_reset(&ptr->hosts[server_key]);
391 rc= MEMCACHED_SOME_ERRORS;
392 continue;
393 }
394
395 /* We just want one pending response per server */
396 memcached_server_response_reset(&ptr->hosts[server_key]);
397 memcached_server_response_increment(&ptr->hosts[server_key]);
398 if ((x > 0 && x == ptr->io_key_prefetch) &&
399 memcached_flush_buffers(ptr) != MEMCACHED_SUCCESS)
400 rc= MEMCACHED_SOME_ERRORS;
401 }
402
403 if (mget_mode)
404 {
405 /*
406 * Send a noop command to flush the buffers
407 */
408 protocol_binary_request_noop request= {.bytes= {0}};
409 request.message.header.request.magic= PROTOCOL_BINARY_REQ;
410 request.message.header.request.opcode= PROTOCOL_BINARY_CMD_NOOP;
411 request.message.header.request.datatype= PROTOCOL_BINARY_RAW_BYTES;
412
413 for (x= 0; x < ptr->number_of_hosts; x++)
414 if (memcached_server_response_count(&ptr->hosts[x]))
415 {
416 if (memcached_io_write(&ptr->hosts[x], NULL, 0, 1) == -1)
417 {
418 memcached_server_response_reset(&ptr->hosts[x]);
419 memcached_io_reset(&ptr->hosts[x]);
420 rc= MEMCACHED_SOME_ERRORS;
421 }
422
423 if (memcached_io_write(&ptr->hosts[x], request.bytes,
424 sizeof(request.bytes), 1) == -1)
425 {
426 memcached_server_response_reset(&ptr->hosts[x]);
427 memcached_io_reset(&ptr->hosts[x]);
428 rc= MEMCACHED_SOME_ERRORS;
429 }
430 }
431 }
432
433
434 return rc;
435 }
436
437 static memcached_return_t replication_binary_mget(memcached_st *ptr,
438 uint32_t* hash,
439 bool* dead_servers,
440 const char *const *keys,
441 const size_t *key_length,
442 size_t number_of_keys)
443 {
444 memcached_return_t rc= MEMCACHED_NOTFOUND;
445 uint32_t x, start= 0;
446 uint64_t randomize_read= memcached_behavior_get(ptr, MEMCACHED_BEHAVIOR_RANDOMIZE_REPLICA_READ);
447
448 if (randomize_read)
449 start= (uint32_t)random() % (uint32_t)(ptr->number_of_replicas + 1);
450
451 /* Loop for each replica */
452 for (uint32_t replica= 0; replica <= ptr->number_of_replicas; ++replica)
453 {
454 bool success= true;
455
456 for (x= 0; x < number_of_keys; ++x)
457 {
458 if (hash[x] == ptr->number_of_hosts)
459 continue; /* Already successfully sent */
460
461 uint32_t server= hash[x] + replica;
462
463 /* In case of randomized reads */
464 if (randomize_read && ((server + start) <= (hash[x] + ptr->number_of_replicas)))
465 server += start;
466
467 while (server >= ptr->number_of_hosts)
468 server -= ptr->number_of_hosts;
469
470 if (dead_servers[server])
471 continue;
472
473 if (memcached_server_response_count(&ptr->hosts[server]) == 0)
474 {
475 rc= memcached_connect(&ptr->hosts[server]);
476 if (rc != MEMCACHED_SUCCESS)
477 {
478 memcached_io_reset(&ptr->hosts[server]);
479 dead_servers[server]= true;
480 success= false;
481 continue;
482 }
483 }
484
485 protocol_binary_request_getk request= {
486 .message.header.request= {
487 .magic= PROTOCOL_BINARY_REQ,
488 .opcode= PROTOCOL_BINARY_CMD_GETK,
489 .keylen= htons((uint16_t)key_length[x]),
490 .datatype= PROTOCOL_BINARY_RAW_BYTES,
491 .bodylen= htonl((uint32_t)key_length[x])
492 }
493 };
494
495 /*
496 * We need to disable buffering to actually know that the request was
497 * successfully sent to the server (so that we should expect a result
498 * back). It would be nice to do this in buffered mode, but then it
499 * would be complex to handle all error situations if we got to send
500 * some of the messages, and then we failed on writing out some others
501 * and we used the callback interface from memcached_mget_execute so
502 * that we might have processed some of the responses etc. For now,
503 * just make sure we work _correctly_
504 */
505 if ((memcached_io_write(&ptr->hosts[server], request.bytes,
506 sizeof(request.bytes), 0) == -1) ||
507 (memcached_io_write(&ptr->hosts[server], keys[x],
508 key_length[x], 1) == -1))
509 {
510 memcached_io_reset(&ptr->hosts[server]);
511 dead_servers[server]= true;
512 success= false;
513 continue;
514 }
515
516 memcached_server_response_increment(&ptr->hosts[server]);
517 hash[x]= ptr->number_of_hosts;
518 }
519
520 if (success)
521 break;
522 }
523
524 return rc;
525 }
526
527 static memcached_return_t binary_mget_by_key(memcached_st *ptr,
528 unsigned int master_server_key,
529 bool is_master_key_set,
530 const char * const *keys,
531 const size_t *key_length,
532 size_t number_of_keys,
533 bool mget_mode)
534 {
535 memcached_return_t rc;
536
537 if (ptr->number_of_replicas == 0)
538 {
539 rc= simple_binary_mget(ptr, master_server_key, is_master_key_set,
540 keys, key_length, number_of_keys, mget_mode);
541 }
542 else
543 {
544 uint32_t* hash;
545 bool* dead_servers;
546
547 hash= ptr->call_malloc(ptr, sizeof(uint32_t) * number_of_keys);
548 dead_servers= ptr->call_calloc(ptr, ptr->number_of_hosts, sizeof(bool));
549
550 if (hash == NULL || dead_servers == NULL)
551 {
552 ptr->call_free(ptr, hash);
553 ptr->call_free(ptr, dead_servers);
554 return MEMCACHED_MEMORY_ALLOCATION_FAILURE;
555 }
556
557 if (is_master_key_set)
558 for (unsigned int x= 0; x < number_of_keys; x++)
559 hash[x]= master_server_key;
560 else
561 for (unsigned int x= 0; x < number_of_keys; x++)
562 hash[x]= memcached_generate_hash(ptr, keys[x], key_length[x]);
563
564 rc= replication_binary_mget(ptr, hash, dead_servers, keys,
565 key_length, number_of_keys);
566
567 ptr->call_free(ptr, hash);
568 ptr->call_free(ptr, dead_servers);
569
570 return MEMCACHED_SUCCESS;
571 }
572
573 return rc;
574 }