projects / m6w6 / libmemcached / blob
? search:


39247287666d55fed37f5b5f72242057fa5f462d
[m6w6/libmemcached] / src / libmemcached / hosts.cc
1 /*
2 +--------------------------------------------------------------------+
3 | libmemcached - C/C++ Client Library for memcached |
4 +--------------------------------------------------------------------+
5 | Redistribution and use in source and binary forms, with or without |
6 | modification, are permitted under the terms of the BSD license. |
7 | You should have received a copy of the license in a bundled file |
8 | named LICENSE; in case you did not receive a copy you can review |
9 | the terms online at: https://opensource.org/licenses/BSD-3-Clause |
10 +--------------------------------------------------------------------+
11 | Copyright (c) 2006-2014 Brian Aker https://datadifferential.com/ |
12 | Copyright (c) 2020 Michael Wallner <mike@php.net> |
13 +--------------------------------------------------------------------+
14 */
15
16 #include "libmemcached/common.h"
17 #include "libmemcached/assert.hpp"
18
19 #include <cmath>
20 #if HAVE_SYS_TIME_H
21 # include <sys/time.h>
22 #endif
23 #include <time.h>
24
25 /* Protoypes (static) */
26 static memcached_return_t update_continuum(Memcached *ptr);
27
28 static int compare_servers(const void *p1, const void *p2) {
29 const memcached_instance_st *a = (const memcached_instance_st *) p1;
30 const memcached_instance_st *b = (const memcached_instance_st *) p2;
31
32 int return_value = strcmp(a->_hostname, b->_hostname);
33
34 if (return_value == 0) {
35 return_value = int(a->port() - b->port());
36 }
37
38 return return_value;
39 }
40
41 static void sort_hosts(Memcached *ptr) {
42 if (memcached_server_count(ptr)) {
43 qsort(memcached_instance_list(ptr), memcached_server_count(ptr), sizeof(memcached_instance_st),
44 compare_servers);
45 }
46 }
47
48 memcached_return_t run_distribution(Memcached *ptr) {
49 if (ptr->flags.use_sort_hosts) {
50 sort_hosts(ptr);
51 }
52
53 switch (ptr->distribution) {
54 case MEMCACHED_DISTRIBUTION_CONSISTENT:
55 case MEMCACHED_DISTRIBUTION_CONSISTENT_KETAMA:
56 case MEMCACHED_DISTRIBUTION_CONSISTENT_KETAMA_SPY:
57 case MEMCACHED_DISTRIBUTION_CONSISTENT_WEIGHTED:
58 return update_continuum(ptr);
59
60 case MEMCACHED_DISTRIBUTION_VIRTUAL_BUCKET:
61 case MEMCACHED_DISTRIBUTION_MODULA:
62 break;
63
64 case MEMCACHED_DISTRIBUTION_RANDOM:
65 srandom((uint32_t) time(NULL));
66 break;
67
68 case MEMCACHED_DISTRIBUTION_CONSISTENT_MAX:
69 default:
70 assert_msg(0, "Invalid distribution type passed to run_distribution()");
71 }
72
73 return MEMCACHED_SUCCESS;
74 }
75
76 static uint32_t ketama_server_hash(const char *key, size_t key_length, uint32_t alignment) {
77 unsigned char results[16];
78
79 libhashkit_md5_signature((unsigned char *) key, key_length, results);
80
81 return ((uint32_t)(results[3 + alignment * 4] & 0xFF) << 24)
82 | ((uint32_t)(results[2 + alignment * 4] & 0xFF) << 16)
83 | ((uint32_t)(results[1 + alignment * 4] & 0xFF) << 8) | (results[0 + alignment * 4] & 0xFF);
84 }
85
86 static int continuum_item_cmp(const void *t1, const void *t2) {
87 memcached_continuum_item_st *ct1 = (memcached_continuum_item_st *) t1;
88 memcached_continuum_item_st *ct2 = (memcached_continuum_item_st *) t2;
89
90 /* Why 153? Hmmm... */
91 WATCHPOINT_ASSERT(ct1->value != 153);
92 if (ct1->value == ct2->value) {
93 if (ct1->index == ct2->index) {
94 return 0;
95 } else if (ct1->index > ct2->index) {
96 return 1;
97 } else {
98 return -1;
99 }
100 } else if (ct1->value > ct2->value) {
101 return 1;
102 } else {
103 return -1;
104 }
105 }
106
107 static memcached_return_t update_continuum(Memcached *ptr) {
108 uint32_t continuum_index = 0;
109 uint32_t pointer_counter = 0;
110 uint32_t pointer_per_server = MEMCACHED_POINTS_PER_SERVER;
111 uint32_t pointer_per_hash = 1;
112 uint32_t live_servers = 0;
113 struct timeval now;
114
115 if (gettimeofday(&now, NULL)) {
116 return memcached_set_errno(*ptr, errno, MEMCACHED_AT);
117 }
118
119 memcached_instance_st *list = memcached_instance_list(ptr);
120
121 /* count live servers (those without a retry delay set) */
122 bool is_auto_ejecting = _is_auto_eject_host(ptr);
123 if (is_auto_ejecting) {
124 live_servers = 0;
125 ptr->ketama.next_distribution_rebuild = 0;
126 for (uint32_t host_index = 0; host_index < memcached_server_count(ptr); ++host_index) {
127 if (list[host_index].next_retry <= now.tv_sec) {
128 live_servers++;
129 } else {
130 if (ptr->ketama.next_distribution_rebuild == 0
131 or list[host_index].next_retry < ptr->ketama.next_distribution_rebuild)
132 {
133 ptr->ketama.next_distribution_rebuild = list[host_index].next_retry;
134 }
135 }
136 }
137 } else {
138 live_servers = memcached_server_count(ptr);
139 }
140
141 if (live_servers == 0) {
142 return MEMCACHED_SUCCESS;
143 }
144
145 uint32_t points_per_server =
146 (uint32_t)(memcached_is_weighted_ketama(ptr) ? MEMCACHED_POINTS_PER_SERVER_KETAMA
147 : MEMCACHED_POINTS_PER_SERVER);
148 uint32_t continuum_limit = live_servers * points_per_server;
149 uint32_t continuum_extra = MEMCACHED_CONTINUUM_ADDITION * points_per_server;
150
151 if (continuum_limit > ptr->ketama.continuum_count) {
152 memcached_continuum_item_st *new_ptr;
153
154 new_ptr = libmemcached_xrealloc(ptr, ptr->ketama.continuum, continuum_limit + continuum_extra,
155 memcached_continuum_item_st);
156
157 if (new_ptr == 0) {
158 return MEMCACHED_MEMORY_ALLOCATION_FAILURE;
159 }
160
161 ptr->ketama.continuum = new_ptr;
162 ptr->ketama.continuum_count = continuum_limit + continuum_extra;
163 }
164 assert_msg(ptr->ketama.continuum, "Programmer Error, empty ketama continuum");
165
166 uint64_t total_weight = 0;
167 if (memcached_is_weighted_ketama(ptr)) {
168 for (uint32_t host_index = 0; host_index < memcached_server_count(ptr); ++host_index) {
169 if (is_auto_ejecting == false or list[host_index].next_retry <= now.tv_sec) {
170 total_weight += list[host_index].weight;
171 }
172 }
173 }
174
175 for (uint32_t host_index = 0; host_index < memcached_server_count(ptr); ++host_index) {
176 if (is_auto_ejecting and list[host_index].next_retry > now.tv_sec) {
177 continue;
178 }
179
180 if (memcached_is_weighted_ketama(ptr)) {
181 float pct = (float) list[host_index].weight / (float) total_weight;
182 pointer_per_server = (uint32_t)(
183 (::floor((float) (pct * MEMCACHED_POINTS_PER_SERVER_KETAMA / 4 * (float) live_servers
184 + 0.0000000001F)))
185 * 4);
186 pointer_per_hash = 4;
187 if (0 && DEBUG) {
188 printf("ketama_weighted:%s|%d|%llu|%u\n", list[host_index]._hostname,
189 list[host_index].port(), (unsigned long long) list[host_index].weight,
190 pointer_per_server);
191 }
192 }
193
194 if (ptr->distribution == MEMCACHED_DISTRIBUTION_CONSISTENT_KETAMA_SPY) {
195 for (uint32_t pointer_index = 0; pointer_index < pointer_per_server / pointer_per_hash;
196 pointer_index++)
197 {
198 char sort_host[1 + MEMCACHED_NI_MAXHOST + 1 + MEMCACHED_NI_MAXSERV + 1
199 + MEMCACHED_NI_MAXSERV] = "";
200 int sort_host_length;
201
202 // Spymemcached ketema key format is: hostname/ip:port-index
203 // If hostname is not available then: /ip:port-index
204 sort_host_length =
205 snprintf(sort_host, sizeof(sort_host), "/%s:%u-%u", list[host_index]._hostname,
206 (uint32_t) list[host_index].port(), pointer_index);
207
208 if (size_t(sort_host_length) >= sizeof(sort_host) or sort_host_length < 0) {
209 return memcached_set_error(*ptr, MEMCACHED_MEMORY_ALLOCATION_FAILURE, MEMCACHED_AT,
210 memcached_literal_param("snprintf(sizeof(sort_host))"));
211 }
212
213 if (0 && DEBUG) {
214 fprintf(stdout, "update_continuum: key is %s\n", sort_host);
215 }
216
217 if (memcached_is_weighted_ketama(ptr)) {
218 for (uint32_t x = 0; x < pointer_per_hash; x++) {
219 uint32_t value = ketama_server_hash(sort_host, (size_t) sort_host_length, x);
220 ptr->ketama.continuum[continuum_index].index = host_index;
221 ptr->ketama.continuum[continuum_index++].value = value;
222 }
223 } else {
224 uint32_t value = hashkit_digest(&ptr->hashkit, sort_host, (size_t) sort_host_length);
225 ptr->ketama.continuum[continuum_index].index = host_index;
226 ptr->ketama.continuum[continuum_index++].value = value;
227 }
228 }
229 } else {
230 for (uint32_t pointer_index = 1; pointer_index <= pointer_per_server / pointer_per_hash;
231 pointer_index++)
232 {
233 char sort_host[MEMCACHED_NI_MAXHOST + 1 + MEMCACHED_NI_MAXSERV + 1 + MEMCACHED_NI_MAXSERV] =
234 "";
235 int sort_host_length;
236
237 if (list[host_index].port() == MEMCACHED_DEFAULT_PORT) {
238 sort_host_length = snprintf(sort_host, sizeof(sort_host), "%s-%u",
239 list[host_index]._hostname, pointer_index - 1);
240 } else {
241 sort_host_length =
242 snprintf(sort_host, sizeof(sort_host), "%s:%u-%u", list[host_index]._hostname,
243 (uint32_t) list[host_index].port(), pointer_index - 1);
244 }
245
246 if (size_t(sort_host_length) >= sizeof(sort_host) or sort_host_length < 0) {
247 return memcached_set_error(*ptr, MEMCACHED_MEMORY_ALLOCATION_FAILURE, MEMCACHED_AT,
248 memcached_literal_param("snprintf(sizeof(sort_host)))"));
249 }
250
251 if (memcached_is_weighted_ketama(ptr)) {
252 for (uint32_t x = 0; x < pointer_per_hash; x++) {
253 uint32_t value = ketama_server_hash(sort_host, (size_t) sort_host_length, x);
254 ptr->ketama.continuum[continuum_index].index = host_index;
255 ptr->ketama.continuum[continuum_index++].value = value;
256 }
257 } else {
258 uint32_t value = hashkit_digest(&ptr->hashkit, sort_host, (size_t) sort_host_length);
259 ptr->ketama.continuum[continuum_index].index = host_index;
260 ptr->ketama.continuum[continuum_index++].value = value;
261 }
262 }
263 }
264
265 pointer_counter += pointer_per_server;
266 }
267
268 assert_msg(ptr, "Programmer Error, no valid ptr");
269 assert_msg(ptr->ketama.continuum, "Programmer Error, empty ketama continuum");
270 assert_msg(memcached_server_count(ptr) * MEMCACHED_POINTS_PER_SERVER <= MEMCACHED_CONTINUUM_SIZE,
271 "invalid size information being given to qsort()");
272 ptr->ketama.continuum_points_counter = pointer_counter;
273 qsort(ptr->ketama.continuum, ptr->ketama.continuum_points_counter,
274 sizeof(memcached_continuum_item_st), continuum_item_cmp);
275
276 if (DEBUG) {
277 for (uint32_t pointer_index = 0; memcached_server_count(ptr)
278 && pointer_index < ((live_servers * MEMCACHED_POINTS_PER_SERVER) - 1);
279 pointer_index++)
280 {
281 WATCHPOINT_ASSERT(ptr->ketama.continuum[pointer_index].value
282 <= ptr->ketama.continuum[pointer_index + 1].value);
283 }
284 }
285
286 return MEMCACHED_SUCCESS;
287 }
288
289 static memcached_return_t server_add(Memcached *memc, const memcached_string_t &hostname,
290 in_port_t port, uint32_t weight, memcached_connection_t type) {
291 assert_msg(memc, "Programmer mistake, somehow server_add() was passed a NULL memcached_st");
292
293 if (memc->number_of_hosts) {
294 assert(memcached_instance_list(memc));
295 }
296
297 if (memcached_instance_list(memc)) {
298 assert(memc->number_of_hosts);
299 }
300
301 uint32_t host_list_size = memc->number_of_hosts + 1;
302 memcached_instance_st *new_host_list = libmemcached_xrealloc(
303 memc, memcached_instance_list(memc), host_list_size, memcached_instance_st);
304
305 if (new_host_list == NULL) {
306 return memcached_set_error(*memc, MEMCACHED_MEMORY_ALLOCATION_FAILURE, MEMCACHED_AT);
307 }
308
309 memcached_instance_set(memc, new_host_list, host_list_size);
310 assert(memc->number_of_hosts == host_list_size);
311
312 /* TODO: Check return type */
313 memcached_instance_st *instance =
314 memcached_instance_fetch(memc, memcached_server_count(memc) - 1);
315
316 if (__instance_create_with(memc, instance, hostname, port, weight, type) == NULL) {
317 return memcached_set_error(*memc, MEMCACHED_MEMORY_ALLOCATION_FAILURE, MEMCACHED_AT);
318 }
319
320 if (weight > 1) {
321 if (memcached_is_consistent_distribution(memc)) {
322 memcached_set_weighted_ketama(memc, true);
323 }
324 }
325
326 return run_distribution(memc);
327 }
328
329 memcached_return_t memcached_server_push(memcached_st *shell, const memcached_server_list_st list) {
330 if (list == NULL) {
331 return MEMCACHED_SUCCESS;
332 }
333
334 Memcached *ptr = memcached2Memcached(shell);
335 if (ptr) {
336 uint32_t original_host_size = memcached_server_count(ptr);
337 uint32_t count = memcached_server_list_count(list);
338 uint32_t host_list_size = count + original_host_size;
339
340 memcached_instance_st *new_host_list = libmemcached_xrealloc(
341 ptr, memcached_instance_list(ptr), host_list_size, memcached_instance_st);
342
343 if (new_host_list == NULL) {
344 return MEMCACHED_MEMORY_ALLOCATION_FAILURE;
345 }
346
347 memcached_instance_set(ptr, new_host_list, host_list_size);
348
349 ptr->state.is_parsing = true;
350 for (uint32_t x = 0; x < count; ++x, ++original_host_size) {
351 WATCHPOINT_ASSERT(list[x].hostname[0]);
352
353 // We have extended the array, and now we will find it, and use it.
354 memcached_instance_st *instance = memcached_instance_fetch(ptr, original_host_size);
355 WATCHPOINT_ASSERT(instance);
356
357 memcached_string_t hostname = {memcached_string_make_from_cstr(list[x].hostname)};
358 if (__instance_create_with(ptr, instance, hostname, list[x].port, list[x].weight,
359 list[x].type)
360 == NULL)
361 {
362 ptr->state.is_parsing = false;
363 return memcached_set_error(*ptr, MEMCACHED_MEMORY_ALLOCATION_FAILURE, MEMCACHED_AT);
364 }
365
366 if (list[x].weight > 1) {
367 memcached_set_weighted_ketama(ptr, true);
368 }
369 }
370 ptr->state.is_parsing = false;
371
372 return run_distribution(ptr);
373 }
374
375 return MEMCACHED_INVALID_ARGUMENTS;
376 }
377
378 memcached_return_t memcached_instance_push(memcached_st *ptr,
379 const struct memcached_instance_st *list,
380 uint32_t number_of_hosts) {
381 if (list == NULL) {
382 return MEMCACHED_SUCCESS;
383 }
384
385 uint32_t original_host_size = memcached_server_count(ptr);
386 uint32_t host_list_size = number_of_hosts + original_host_size;
387 memcached_instance_st *new_host_list = libmemcached_xrealloc(
388 ptr, memcached_instance_list(ptr), host_list_size, memcached_instance_st);
389
390 if (new_host_list == NULL) {
391 return MEMCACHED_MEMORY_ALLOCATION_FAILURE;
392 }
393
394 memcached_instance_set(ptr, new_host_list, host_list_size);
395
396 // We don't bother with lookups for this operation
397 ptr->state.is_parsing = true;
398
399 // We use original_host_size since size will now point to the first new
400 // instance allocated.
401 for (uint32_t x = 0; x < number_of_hosts; ++x, ++original_host_size) {
402 WATCHPOINT_ASSERT(list[x]._hostname[0]);
403
404 // We have extended the array, and now we will find it, and use it.
405 memcached_instance_st *instance = memcached_instance_fetch(ptr, original_host_size);
406 WATCHPOINT_ASSERT(instance);
407
408 memcached_string_t hostname = {memcached_string_make_from_cstr(list[x]._hostname)};
409 if (__instance_create_with(ptr, instance, hostname, list[x].port(), list[x].weight,
410 list[x].type)
411 == NULL)
412 {
413 ptr->state.is_parsing = false;
414 return memcached_set_error(*ptr, MEMCACHED_MEMORY_ALLOCATION_FAILURE, MEMCACHED_AT);
415 }
416
417 if (list[x].weight > 1) {
418 memcached_set_weighted_ketama(ptr, true);
419 }
420 }
421 ptr->state.is_parsing = false;
422
423 return run_distribution(ptr);
424 }
425
426 memcached_return_t memcached_server_add_unix_socket(memcached_st *ptr, const char *filename) {
427 return memcached_server_add_unix_socket_with_weight(ptr, filename, 0);
428 }
429
430 memcached_return_t memcached_server_add_unix_socket_with_weight(memcached_st *shell,
431 const char *filename,
432 uint32_t weight) {
433 Memcached *ptr = memcached2Memcached(shell);
434 if (ptr) {
435 memcached_string_t _filename = {memcached_string_make_from_cstr(filename)};
436 if (memcached_is_valid_filename(_filename) == false) {
437 return memcached_set_error(*ptr, MEMCACHED_INVALID_ARGUMENTS, MEMCACHED_AT,
438 memcached_literal_param("Invalid filename for socket provided"));
439 }
440
441 return server_add(ptr, _filename, 0, weight, MEMCACHED_CONNECTION_UNIX_SOCKET);
442 }
443
444 return MEMCACHED_FAILURE;
445 }
446
447 memcached_return_t memcached_server_add_udp(memcached_st *ptr, const char *hostname,
448 in_port_t port) {
449 return memcached_server_add_udp_with_weight(ptr, hostname, port, 0);
450 }
451
452 memcached_return_t memcached_server_add_udp_with_weight(memcached_st *shell, const char *,
453 in_port_t, uint32_t) {
454 Memcached *self = memcached2Memcached(shell);
455 if (self) {
456 return memcached_set_error(*self, MEMCACHED_DEPRECATED, MEMCACHED_AT);
457 }
458
459 return MEMCACHED_INVALID_ARGUMENTS;
460 }
461
462 memcached_return_t memcached_server_add(memcached_st *shell, const char *hostname, in_port_t port) {
463 return memcached_server_add_with_weight(shell, hostname, port, 0);
464 }
465
466 memcached_return_t memcached_server_add_with_weight(memcached_st *shell, const char *hostname,
467 in_port_t port, uint32_t weight) {
468 Memcached *ptr = memcached2Memcached(shell);
469 if (ptr == NULL) {
470 return MEMCACHED_INVALID_ARGUMENTS;
471 }
472
473 if (port == 0) {
474 port = MEMCACHED_DEFAULT_PORT;
475 }
476
477 size_t hostname_length = hostname ? strlen(hostname) : 0;
478 if (hostname_length == 0) {
479 hostname = "localhost";
480 hostname_length = memcached_literal_param_size("localhost");
481 }
482
483 memcached_string_t _hostname = {hostname, hostname_length};
484
485 if (memcached_is_valid_servername(_hostname) == false) {
486 return memcached_set_error(*ptr, MEMCACHED_INVALID_ARGUMENTS, MEMCACHED_AT,
487 memcached_literal_param("Invalid hostname provided"));
488 }
489
490 return server_add(ptr, _hostname, port, weight,
491 _hostname.c_str[0] == '/' ? MEMCACHED_CONNECTION_UNIX_SOCKET
492 : MEMCACHED_CONNECTION_TCP);
493 }
494
495 memcached_return_t memcached_server_add_parsed(memcached_st *ptr, const char *hostname,
496 size_t hostname_length, in_port_t port,
497 uint32_t weight) {
498 char buffer[MEMCACHED_NI_MAXHOST] = {0};
499
500 memcpy(buffer, hostname, hostname_length);
501 buffer[hostname_length] = 0;
502
503 memcached_string_t _hostname = {buffer, hostname_length};
504
505 return server_add(ptr, _hostname, port, weight, MEMCACHED_CONNECTION_TCP);
506 }
Resurrection of libmemcached https://launchpad.net/libmemcached