projects / m6w6 / libmemcached / blob
? search:


b478881ce7731ac7faa7413e2ba589768e69e2e7
[m6w6/libmemcached] / src / libmemcached / hosts.cc
1 /*
2 +--------------------------------------------------------------------+
3 | libmemcached - C/C++ Client Library for memcached |
4 +--------------------------------------------------------------------+
5 | Redistribution and use in source and binary forms, with or without |
6 | modification, are permitted under the terms of the BSD license. |
7 | You should have received a copy of the license in a bundled file |
8 | named LICENSE; in case you did not receive a copy you can review |
9 | the terms online at: https://opensource.org/licenses/BSD-3-Clause |
10 +--------------------------------------------------------------------+
11 | Copyright (c) 2006-2014 Brian Aker https://datadifferential.com/ |
12 | Copyright (c) 2020 Michael Wallner <mike@php.net> |
13 +--------------------------------------------------------------------+
14 */
15
16 #include "libmemcached/common.h"
17 #include "libmemcached/assert.hpp"
18
19 #include <cmath>
20 #include <sys/time.h>
21
22 /* Protoypes (static) */
23 static memcached_return_t update_continuum(Memcached *ptr);
24
25 static int compare_servers(const void *p1, const void *p2) {
26 const memcached_instance_st *a = (const memcached_instance_st *) p1;
27 const memcached_instance_st *b = (const memcached_instance_st *) p2;
28
29 int return_value = strcmp(a->_hostname, b->_hostname);
30
31 if (return_value == 0) {
32 return_value = int(a->port() - b->port());
33 }
34
35 return return_value;
36 }
37
38 static void sort_hosts(Memcached *ptr) {
39 if (memcached_server_count(ptr)) {
40 qsort(memcached_instance_list(ptr), memcached_server_count(ptr), sizeof(memcached_instance_st),
41 compare_servers);
42 }
43 }
44
45 memcached_return_t run_distribution(Memcached *ptr) {
46 if (ptr->flags.use_sort_hosts) {
47 sort_hosts(ptr);
48 }
49
50 switch (ptr->distribution) {
51 case MEMCACHED_DISTRIBUTION_CONSISTENT:
52 case MEMCACHED_DISTRIBUTION_CONSISTENT_KETAMA:
53 case MEMCACHED_DISTRIBUTION_CONSISTENT_KETAMA_SPY:
54 case MEMCACHED_DISTRIBUTION_CONSISTENT_WEIGHTED:
55 return update_continuum(ptr);
56
57 case MEMCACHED_DISTRIBUTION_VIRTUAL_BUCKET:
58 case MEMCACHED_DISTRIBUTION_MODULA:
59 break;
60
61 case MEMCACHED_DISTRIBUTION_RANDOM:
62 srandom((uint32_t) time(NULL));
63 break;
64
65 case MEMCACHED_DISTRIBUTION_CONSISTENT_MAX:
66 default:
67 assert_msg(0, "Invalid distribution type passed to run_distribution()");
68 }
69
70 return MEMCACHED_SUCCESS;
71 }
72
73 static uint32_t ketama_server_hash(const char *key, size_t key_length, uint32_t alignment) {
74 unsigned char results[16];
75
76 libhashkit_md5_signature((unsigned char *) key, key_length, results);
77
78 return ((uint32_t)(results[3 + alignment * 4] & 0xFF) << 24)
79 | ((uint32_t)(results[2 + alignment * 4] & 0xFF) << 16)
80 | ((uint32_t)(results[1 + alignment * 4] & 0xFF) << 8) | (results[0 + alignment * 4] & 0xFF);
81 }
82
83 static int continuum_item_cmp(const void *t1, const void *t2) {
84 memcached_continuum_item_st *ct1 = (memcached_continuum_item_st *) t1;
85 memcached_continuum_item_st *ct2 = (memcached_continuum_item_st *) t2;
86
87 /* Why 153? Hmmm... */
88 WATCHPOINT_ASSERT(ct1->value != 153);
89 if (ct1->value == ct2->value) {
90 if (ct1->index == ct2->index) {
91 return 0;
92 } else if (ct1->index > ct2->index) {
93 return 1;
94 } else {
95 return -1;
96 }
97 } else if (ct1->value > ct2->value) {
98 return 1;
99 } else {
100 return -1;
101 }
102 }
103
104 static memcached_return_t update_continuum(Memcached *ptr) {
105 uint32_t continuum_index = 0;
106 uint32_t pointer_counter = 0;
107 uint32_t pointer_per_server = MEMCACHED_POINTS_PER_SERVER;
108 uint32_t pointer_per_hash = 1;
109 uint32_t live_servers = 0;
110 struct timeval now;
111
112 if (gettimeofday(&now, NULL)) {
113 return memcached_set_errno(*ptr, errno, MEMCACHED_AT);
114 }
115
116 memcached_instance_st *list = memcached_instance_list(ptr);
117
118 /* count live servers (those without a retry delay set) */
119 bool is_auto_ejecting = _is_auto_eject_host(ptr);
120 if (is_auto_ejecting) {
121 live_servers = 0;
122 ptr->ketama.next_distribution_rebuild = 0;
123 for (uint32_t host_index = 0; host_index < memcached_server_count(ptr); ++host_index) {
124 if (list[host_index].next_retry <= now.tv_sec) {
125 live_servers++;
126 } else {
127 if (ptr->ketama.next_distribution_rebuild == 0
128 or list[host_index].next_retry < ptr->ketama.next_distribution_rebuild)
129 {
130 ptr->ketama.next_distribution_rebuild = list[host_index].next_retry;
131 }
132 }
133 }
134 } else {
135 live_servers = memcached_server_count(ptr);
136 }
137
138 if (live_servers == 0) {
139 return MEMCACHED_SUCCESS;
140 }
141
142 uint32_t points_per_server =
143 (uint32_t)(memcached_is_weighted_ketama(ptr) ? MEMCACHED_POINTS_PER_SERVER_KETAMA
144 : MEMCACHED_POINTS_PER_SERVER);
145 uint32_t continuum_limit = live_servers * points_per_server;
146 uint32_t continuum_extra = MEMCACHED_CONTINUUM_ADDITION * points_per_server;
147
148 if (continuum_limit > ptr->ketama.continuum_count) {
149 memcached_continuum_item_st *new_ptr;
150
151 new_ptr = libmemcached_xrealloc(ptr, ptr->ketama.continuum, continuum_limit + continuum_extra,
152 memcached_continuum_item_st);
153
154 if (new_ptr == 0) {
155 return MEMCACHED_MEMORY_ALLOCATION_FAILURE;
156 }
157
158 ptr->ketama.continuum = new_ptr;
159 ptr->ketama.continuum_count = continuum_limit + continuum_extra;
160 }
161 assert_msg(ptr->ketama.continuum, "Programmer Error, empty ketama continuum");
162
163 uint64_t total_weight = 0;
164 if (memcached_is_weighted_ketama(ptr)) {
165 for (uint32_t host_index = 0; host_index < memcached_server_count(ptr); ++host_index) {
166 if (is_auto_ejecting == false or list[host_index].next_retry <= now.tv_sec) {
167 total_weight += list[host_index].weight;
168 }
169 }
170 }
171
172 for (uint32_t host_index = 0; host_index < memcached_server_count(ptr); ++host_index) {
173 if (is_auto_ejecting and list[host_index].next_retry > now.tv_sec) {
174 continue;
175 }
176
177 if (memcached_is_weighted_ketama(ptr)) {
178 float pct = (float) list[host_index].weight / (float) total_weight;
179 pointer_per_server = (uint32_t)(
180 (::floor((float) (pct * MEMCACHED_POINTS_PER_SERVER_KETAMA / 4 * (float) live_servers
181 + 0.0000000001F)))
182 * 4);
183 pointer_per_hash = 4;
184 if (0 && DEBUG) {
185 printf("ketama_weighted:%s|%d|%llu|%u\n", list[host_index]._hostname,
186 list[host_index].port(), (unsigned long long) list[host_index].weight,
187 pointer_per_server);
188 }
189 }
190
191 if (ptr->distribution == MEMCACHED_DISTRIBUTION_CONSISTENT_KETAMA_SPY) {
192 for (uint32_t pointer_index = 0; pointer_index < pointer_per_server / pointer_per_hash;
193 pointer_index++)
194 {
195 char sort_host[1 + MEMCACHED_NI_MAXHOST + 1 + MEMCACHED_NI_MAXSERV + 1
196 + MEMCACHED_NI_MAXSERV] = "";
197 int sort_host_length;
198
199 // Spymemcached ketema key format is: hostname/ip:port-index
200 // If hostname is not available then: /ip:port-index
201 sort_host_length =
202 snprintf(sort_host, sizeof(sort_host), "/%s:%u-%u", list[host_index]._hostname,
203 (uint32_t) list[host_index].port(), pointer_index);
204
205 if (size_t(sort_host_length) >= sizeof(sort_host) or sort_host_length < 0) {
206 return memcached_set_error(*ptr, MEMCACHED_MEMORY_ALLOCATION_FAILURE, MEMCACHED_AT,
207 memcached_literal_param("snprintf(sizeof(sort_host))"));
208 }
209
210 if (0 && DEBUG) {
211 fprintf(stdout, "update_continuum: key is %s\n", sort_host);
212 }
213
214 if (memcached_is_weighted_ketama(ptr)) {
215 for (uint32_t x = 0; x < pointer_per_hash; x++) {
216 uint32_t value = ketama_server_hash(sort_host, (size_t) sort_host_length, x);
217 ptr->ketama.continuum[continuum_index].index = host_index;
218 ptr->ketama.continuum[continuum_index++].value = value;
219 }
220 } else {
221 uint32_t value = hashkit_digest(&ptr->hashkit, sort_host, (size_t) sort_host_length);
222 ptr->ketama.continuum[continuum_index].index = host_index;
223 ptr->ketama.continuum[continuum_index++].value = value;
224 }
225 }
226 } else {
227 for (uint32_t pointer_index = 1; pointer_index <= pointer_per_server / pointer_per_hash;
228 pointer_index++)
229 {
230 char sort_host[MEMCACHED_NI_MAXHOST + 1 + MEMCACHED_NI_MAXSERV + 1 + MEMCACHED_NI_MAXSERV] =
231 "";
232 int sort_host_length;
233
234 if (list[host_index].port() == MEMCACHED_DEFAULT_PORT) {
235 sort_host_length = snprintf(sort_host, sizeof(sort_host), "%s-%u",
236 list[host_index]._hostname, pointer_index - 1);
237 } else {
238 sort_host_length =
239 snprintf(sort_host, sizeof(sort_host), "%s:%u-%u", list[host_index]._hostname,
240 (uint32_t) list[host_index].port(), pointer_index - 1);
241 }
242
243 if (size_t(sort_host_length) >= sizeof(sort_host) or sort_host_length < 0) {
244 return memcached_set_error(*ptr, MEMCACHED_MEMORY_ALLOCATION_FAILURE, MEMCACHED_AT,
245 memcached_literal_param("snprintf(sizeof(sort_host)))"));
246 }
247
248 if (memcached_is_weighted_ketama(ptr)) {
249 for (uint32_t x = 0; x < pointer_per_hash; x++) {
250 uint32_t value = ketama_server_hash(sort_host, (size_t) sort_host_length, x);
251 ptr->ketama.continuum[continuum_index].index = host_index;
252 ptr->ketama.continuum[continuum_index++].value = value;
253 }
254 } else {
255 uint32_t value = hashkit_digest(&ptr->hashkit, sort_host, (size_t) sort_host_length);
256 ptr->ketama.continuum[continuum_index].index = host_index;
257 ptr->ketama.continuum[continuum_index++].value = value;
258 }
259 }
260 }
261
262 pointer_counter += pointer_per_server;
263 }
264
265 assert_msg(ptr, "Programmer Error, no valid ptr");
266 assert_msg(ptr->ketama.continuum, "Programmer Error, empty ketama continuum");
267 assert_msg(memcached_server_count(ptr) * MEMCACHED_POINTS_PER_SERVER <= MEMCACHED_CONTINUUM_SIZE,
268 "invalid size information being given to qsort()");
269 ptr->ketama.continuum_points_counter = pointer_counter;
270 qsort(ptr->ketama.continuum, ptr->ketama.continuum_points_counter,
271 sizeof(memcached_continuum_item_st), continuum_item_cmp);
272
273 if (DEBUG) {
274 for (uint32_t pointer_index = 0; memcached_server_count(ptr)
275 && pointer_index < ((live_servers * MEMCACHED_POINTS_PER_SERVER) - 1);
276 pointer_index++)
277 {
278 WATCHPOINT_ASSERT(ptr->ketama.continuum[pointer_index].value
279 <= ptr->ketama.continuum[pointer_index + 1].value);
280 }
281 }
282
283 return MEMCACHED_SUCCESS;
284 }
285
286 static memcached_return_t server_add(Memcached *memc, const memcached_string_t &hostname,
287 in_port_t port, uint32_t weight, memcached_connection_t type) {
288 assert_msg(memc, "Programmer mistake, somehow server_add() was passed a NULL memcached_st");
289
290 if (memc->number_of_hosts) {
291 assert(memcached_instance_list(memc));
292 }
293
294 if (memcached_instance_list(memc)) {
295 assert(memc->number_of_hosts);
296 }
297
298 uint32_t host_list_size = memc->number_of_hosts + 1;
299 memcached_instance_st *new_host_list = libmemcached_xrealloc(
300 memc, memcached_instance_list(memc), host_list_size, memcached_instance_st);
301
302 if (new_host_list == NULL) {
303 return memcached_set_error(*memc, MEMCACHED_MEMORY_ALLOCATION_FAILURE, MEMCACHED_AT);
304 }
305
306 memcached_instance_set(memc, new_host_list, host_list_size);
307 assert(memc->number_of_hosts == host_list_size);
308
309 /* TODO: Check return type */
310 memcached_instance_st *instance =
311 memcached_instance_fetch(memc, memcached_server_count(memc) - 1);
312
313 if (__instance_create_with(memc, instance, hostname, port, weight, type) == NULL) {
314 return memcached_set_error(*memc, MEMCACHED_MEMORY_ALLOCATION_FAILURE, MEMCACHED_AT);
315 }
316
317 if (weight > 1) {
318 if (memcached_is_consistent_distribution(memc)) {
319 memcached_set_weighted_ketama(memc, true);
320 }
321 }
322
323 return run_distribution(memc);
324 }
325
326 memcached_return_t memcached_server_push(memcached_st *shell, const memcached_server_list_st list) {
327 if (list == NULL) {
328 return MEMCACHED_SUCCESS;
329 }
330
331 Memcached *ptr = memcached2Memcached(shell);
332 if (ptr) {
333 uint32_t original_host_size = memcached_server_count(ptr);
334 uint32_t count = memcached_server_list_count(list);
335 uint32_t host_list_size = count + original_host_size;
336
337 memcached_instance_st *new_host_list = libmemcached_xrealloc(
338 ptr, memcached_instance_list(ptr), host_list_size, memcached_instance_st);
339
340 if (new_host_list == NULL) {
341 return MEMCACHED_MEMORY_ALLOCATION_FAILURE;
342 }
343
344 memcached_instance_set(ptr, new_host_list, host_list_size);
345
346 ptr->state.is_parsing = true;
347 for (uint32_t x = 0; x < count; ++x, ++original_host_size) {
348 WATCHPOINT_ASSERT(list[x].hostname[0]);
349
350 // We have extended the array, and now we will find it, and use it.
351 memcached_instance_st *instance = memcached_instance_fetch(ptr, original_host_size);
352 WATCHPOINT_ASSERT(instance);
353
354 memcached_string_t hostname = {memcached_string_make_from_cstr(list[x].hostname)};
355 if (__instance_create_with(ptr, instance, hostname, list[x].port, list[x].weight,
356 list[x].type)
357 == NULL)
358 {
359 ptr->state.is_parsing = false;
360 return memcached_set_error(*ptr, MEMCACHED_MEMORY_ALLOCATION_FAILURE, MEMCACHED_AT);
361 }
362
363 if (list[x].weight > 1) {
364 memcached_set_weighted_ketama(ptr, true);
365 }
366 }
367 ptr->state.is_parsing = false;
368
369 return run_distribution(ptr);
370 }
371
372 return MEMCACHED_INVALID_ARGUMENTS;
373 }
374
375 memcached_return_t memcached_instance_push(memcached_st *ptr,
376 const struct memcached_instance_st *list,
377 uint32_t number_of_hosts) {
378 if (list == NULL) {
379 return MEMCACHED_SUCCESS;
380 }
381
382 uint32_t original_host_size = memcached_server_count(ptr);
383 uint32_t host_list_size = number_of_hosts + original_host_size;
384 memcached_instance_st *new_host_list = libmemcached_xrealloc(
385 ptr, memcached_instance_list(ptr), host_list_size, memcached_instance_st);
386
387 if (new_host_list == NULL) {
388 return MEMCACHED_MEMORY_ALLOCATION_FAILURE;
389 }
390
391 memcached_instance_set(ptr, new_host_list, host_list_size);
392
393 // We don't bother with lookups for this operation
394 ptr->state.is_parsing = true;
395
396 // We use original_host_size since size will now point to the first new
397 // instance allocated.
398 for (uint32_t x = 0; x < number_of_hosts; ++x, ++original_host_size) {
399 WATCHPOINT_ASSERT(list[x]._hostname[0]);
400
401 // We have extended the array, and now we will find it, and use it.
402 memcached_instance_st *instance = memcached_instance_fetch(ptr, original_host_size);
403 WATCHPOINT_ASSERT(instance);
404
405 memcached_string_t hostname = {memcached_string_make_from_cstr(list[x]._hostname)};
406 if (__instance_create_with(ptr, instance, hostname, list[x].port(), list[x].weight,
407 list[x].type)
408 == NULL)
409 {
410 ptr->state.is_parsing = false;
411 return memcached_set_error(*ptr, MEMCACHED_MEMORY_ALLOCATION_FAILURE, MEMCACHED_AT);
412 }
413
414 if (list[x].weight > 1) {
415 memcached_set_weighted_ketama(ptr, true);
416 }
417 }
418 ptr->state.is_parsing = false;
419
420 return run_distribution(ptr);
421 }
422
423 memcached_return_t memcached_server_add_unix_socket(memcached_st *ptr, const char *filename) {
424 return memcached_server_add_unix_socket_with_weight(ptr, filename, 0);
425 }
426
427 memcached_return_t memcached_server_add_unix_socket_with_weight(memcached_st *shell,
428 const char *filename,
429 uint32_t weight) {
430 Memcached *ptr = memcached2Memcached(shell);
431 if (ptr) {
432 memcached_string_t _filename = {memcached_string_make_from_cstr(filename)};
433 if (memcached_is_valid_filename(_filename) == false) {
434 return memcached_set_error(*ptr, MEMCACHED_INVALID_ARGUMENTS, MEMCACHED_AT,
435 memcached_literal_param("Invalid filename for socket provided"));
436 }
437
438 return server_add(ptr, _filename, 0, weight, MEMCACHED_CONNECTION_UNIX_SOCKET);
439 }
440
441 return MEMCACHED_FAILURE;
442 }
443
444 memcached_return_t memcached_server_add_udp(memcached_st *ptr, const char *hostname,
445 in_port_t port) {
446 return memcached_server_add_udp_with_weight(ptr, hostname, port, 0);
447 }
448
449 memcached_return_t memcached_server_add_udp_with_weight(memcached_st *shell, const char *,
450 in_port_t, uint32_t) {
451 Memcached *self = memcached2Memcached(shell);
452 if (self) {
453 return memcached_set_error(*self, MEMCACHED_DEPRECATED, MEMCACHED_AT);
454 }
455
456 return MEMCACHED_INVALID_ARGUMENTS;
457 }
458
459 memcached_return_t memcached_server_add(memcached_st *shell, const char *hostname, in_port_t port) {
460 return memcached_server_add_with_weight(shell, hostname, port, 0);
461 }
462
463 memcached_return_t memcached_server_add_with_weight(memcached_st *shell, const char *hostname,
464 in_port_t port, uint32_t weight) {
465 Memcached *ptr = memcached2Memcached(shell);
466 if (ptr == NULL) {
467 return MEMCACHED_INVALID_ARGUMENTS;
468 }
469
470 if (port == 0) {
471 port = MEMCACHED_DEFAULT_PORT;
472 }
473
474 size_t hostname_length = hostname ? strlen(hostname) : 0;
475 if (hostname_length == 0) {
476 hostname = "localhost";
477 hostname_length = memcached_literal_param_size("localhost");
478 }
479
480 memcached_string_t _hostname = {hostname, hostname_length};
481
482 if (memcached_is_valid_servername(_hostname) == false) {
483 return memcached_set_error(*ptr, MEMCACHED_INVALID_ARGUMENTS, MEMCACHED_AT,
484 memcached_literal_param("Invalid hostname provided"));
485 }
486
487 return server_add(ptr, _hostname, port, weight,
488 _hostname.c_str[0] == '/' ? MEMCACHED_CONNECTION_UNIX_SOCKET
489 : MEMCACHED_CONNECTION_TCP);
490 }
491
492 memcached_return_t memcached_server_add_parsed(memcached_st *ptr, const char *hostname,
493 size_t hostname_length, in_port_t port,
494 uint32_t weight) {
495 char buffer[MEMCACHED_NI_MAXHOST] = {0};
496
497 memcpy(buffer, hostname, hostname_length);
498 buffer[hostname_length] = 0;
499
500 memcached_string_t _hostname = {buffer, hostname_length};
501
502 return server_add(ptr, _hostname, port, weight, MEMCACHED_CONNECTION_TCP);
503 }
Resurrection of libmemcached https://launchpad.net/libmemcached