2 +--------------------------------------------------------------------+
3 | libmemcached - C/C++ Client Library for memcached |
4 +--------------------------------------------------------------------+
5 | Redistribution and use in source and binary forms, with or without |
6 | modification, are permitted under the terms of the BSD license. |
7 | You should have received a copy of the license in a bundled file |
8 | named LICENSE; in case you did not receive a copy you can review |
9 | the terms online at: https://opensource.org/licenses/BSD-3-Clause |
10 +--------------------------------------------------------------------+
11 | Copyright (c) 2006-2014 Brian Aker https://datadifferential.com/ |
12 | Copyright (c) 2020 Michael Wallner <mike@php.net> |
13 +--------------------------------------------------------------------+
16 #include "libmemcached/common.h"
17 #include "p9y/socket.hpp"
18 #include "p9y/poll.hpp"
23 static memcached_return_t
connect_poll(memcached_instance_st
*server
, const int connection_error
) {
25 fds
[0].fd
= server
->fd
;
26 fds
[0].events
= server
->events();
31 if (server
->root
->connect_timeout
== 0) {
32 return memcached_set_error(
33 *server
, MEMCACHED_TIMEOUT
, MEMCACHED_AT
,
34 memcached_literal_param("The time to wait for a connection to be established was set to "
35 "zero which produces a timeout to every call to poll()."));
38 while (--loop_max
) // Should only loop on cases of ERESTART or EINTR
41 if ((number_of
= poll(fds
, 1, server
->root
->connect_timeout
)) == SOCKET_ERROR
) {
42 int local_errno
= get_socket_errno(); // We cache in case closesocket() modifies errno
43 switch (local_errno
) {
52 return memcached_set_error(*server
, MEMCACHED_MEMORY_ALLOCATION_FAILURE
, MEMCACHED_AT
);
55 return memcached_set_error(
56 *server
, MEMCACHED_MEMORY_ALLOCATION_FAILURE
, MEMCACHED_AT
,
57 memcached_literal_param(
58 "RLIMIT_NOFILE exceeded, or if OSX the timeout value was invalid"));
60 default: // This should not happen
64 assert_msg(server
->fd
!= INVALID_SOCKET
, "poll() was passed an invalid file descriptor");
65 server
->reset_socket();
66 server
->state
= MEMCACHED_SERVER_STATE_NEW
;
68 return memcached_set_errno(*server
, local_errno
, MEMCACHED_AT
);
72 if (connection_error
!= EALREADY
) {
74 socklen_t len
= sizeof(err
);
75 if (getsockopt(server
->fd
, SOL_SOCKET
, SO_ERROR
, (char *) &err
, &len
) == -1) {
76 return memcached_set_errno(
77 *server
, errno
, MEMCACHED_AT
,
78 memcached_literal_param(
79 "getsockopt() error'ed while looking for error connect_poll(EINPROGRESS)"));
82 // If Zero, my hero, we just fail to a generic MEMCACHED_TIMEOUT error
84 return memcached_set_errno(
85 *server
, err
, MEMCACHED_AT
,
86 memcached_literal_param("getsockopt() found the error from poll() after connect() "
87 "returned EINPROGRESS."));
91 return memcached_set_error(*server
, MEMCACHED_TIMEOUT
, MEMCACHED_AT
,
92 memcached_literal_param("(number_of == 0)"));
95 assert(number_of
== 1);
97 if (fds
[0].revents
& POLLERR
or fds
[0].revents
& POLLHUP
or fds
[0].revents
& POLLNVAL
) {
99 socklen_t len
= sizeof(err
);
100 if (getsockopt(fds
[0].fd
, SOL_SOCKET
, SO_ERROR
, (char *) &err
, &len
) == -1) {
101 return memcached_set_errno(
102 *server
, errno
, MEMCACHED_AT
,
103 memcached_literal_param(
104 "getsockopt() errored while looking up error state from poll()"));
107 // We check the value to see what happened with the socket.
108 if (err
== 0) // Should not happen
110 return MEMCACHED_SUCCESS
;
114 return memcached_set_errno(
115 *server
, err
, MEMCACHED_AT
,
116 memcached_literal_param("getsockopt() found the error from poll() during connect."));
118 assert(fds
[0].revents
& POLLOUT
);
120 if (fds
[0].revents
& POLLOUT
and connection_error
!= EALREADY
) {
122 socklen_t len
= sizeof(err
);
123 if (getsockopt(server
->fd
, SOL_SOCKET
, SO_ERROR
, (char *) &err
, &len
) == -1) {
124 return memcached_set_errno(*server
, errno
, MEMCACHED_AT
);
128 return MEMCACHED_SUCCESS
;
131 return memcached_set_errno(
132 *server
, err
, MEMCACHED_AT
,
133 memcached_literal_param(
134 "getsockopt() found the error from poll() after connect() returned EINPROGRESS."));
137 break; // We only have the loop setup for errno types that require restart
140 // This should only be possible from ERESTART or EINTR;
141 return memcached_set_errno(*server
, connection_error
, MEMCACHED_AT
,
142 memcached_literal_param("connect_poll() was exhausted"));
145 static memcached_return_t
set_hostinfo(memcached_instance_st
*server
) {
146 assert(server
->type
!= MEMCACHED_CONNECTION_UNIX_SOCKET
);
147 server
->clear_addrinfo();
149 char str_port
[MEMCACHED_NI_MAXSERV
] = {0};
151 int length
= snprintf(str_port
, MEMCACHED_NI_MAXSERV
, "%u", uint32_t(server
->port()));
152 if (length
>= MEMCACHED_NI_MAXSERV
or length
<= 0 or errno
) {
153 return memcached_set_error(*server
, MEMCACHED_MEMORY_ALLOCATION_FAILURE
, MEMCACHED_AT
,
154 memcached_literal_param("snprintf(NI_MAXSERV)"));
157 struct addrinfo hints
;
158 memset(&hints
, 0, sizeof(struct addrinfo
));
160 hints
.ai_family
= AF_UNSPEC
;
161 if (memcached_is_udp(server
->root
)) {
162 hints
.ai_protocol
= IPPROTO_UDP
;
163 hints
.ai_socktype
= SOCK_DGRAM
;
165 hints
.ai_socktype
= SOCK_STREAM
;
166 hints
.ai_protocol
= IPPROTO_TCP
;
169 assert(server
->address_info
== NULL
);
170 assert(server
->address_info_next
== NULL
);
172 assert(server
->hostname());
173 switch (errcode
= getaddrinfo(server
->hostname(), str_port
, &hints
, &server
->address_info
)) {
175 server
->address_info_next
= server
->address_info
;
176 server
->state
= MEMCACHED_SERVER_STATE_ADDRINFO
;
180 return memcached_set_error(*server
, MEMCACHED_TIMEOUT
, MEMCACHED_AT
,
181 memcached_string_make_from_cstr(gai_strerror(errcode
)));
184 server
->clear_addrinfo();
185 return memcached_set_errno(*server
, errno
, MEMCACHED_AT
,
186 memcached_literal_param("getaddrinfo(EAI_SYSTEM)"));
189 server
->clear_addrinfo();
190 return memcached_set_error(*server
, MEMCACHED_INVALID_ARGUMENTS
, MEMCACHED_AT
,
191 memcached_literal_param("getaddrinfo(EAI_BADFLAGS)"));
194 server
->clear_addrinfo();
195 return memcached_set_error(*server
, MEMCACHED_MEMORY_ALLOCATION_FAILURE
, MEMCACHED_AT
,
196 memcached_literal_param("getaddrinfo(EAI_MEMORY)"));
199 server
->clear_addrinfo();
200 return memcached_set_error(*server
, MEMCACHED_HOST_LOOKUP_FAILURE
, MEMCACHED_AT
,
201 memcached_string_make_from_cstr(gai_strerror(errcode
)));
205 return MEMCACHED_SUCCESS
;
208 static inline void set_socket_nonblocking(memcached_instance_st
*server
) {
211 if (ioctlsocket(server
->fd
, FIONBIO
, &arg
) == SOCKET_ERROR
) {
212 memcached_set_errno(*server
, get_socket_errno(), NULL
);
217 if (SOCK_NONBLOCK
== 0) {
219 flags
= fcntl(server
->fd
, F_GETFL
, 0);
220 } while (flags
== -1 && (errno
== EINTR
|| errno
== EAGAIN
));
223 memcached_set_errno(*server
, errno
, NULL
);
224 } else if ((flags
& O_NONBLOCK
) == 0) {
228 rval
= fcntl(server
->fd
, F_SETFL
, flags
| O_NONBLOCK
);
229 } while (rval
== -1 && (errno
== EINTR
or errno
== EAGAIN
));
232 memcached_set_errno(*server
, errno
, NULL
);
239 static bool set_socket_options(memcached_instance_st
*server
) {
240 assert_msg(server
->fd
!= INVALID_SOCKET
, "invalid socket was passed to set_socket_options()");
243 // If SOCK_CLOEXEC exists then we don't need to call the following
244 if (SOCK_CLOEXEC
== 0) {
248 flags
= fcntl(server
->fd
, F_GETFD
, 0);
249 } while (flags
== -1 and (errno
== EINTR
or errno
== EAGAIN
));
254 rval
= fcntl(server
->fd
, F_SETFD
, flags
| FD_CLOEXEC
);
255 } while (rval
== -1 && (errno
== EINTR
or errno
== EAGAIN
));
256 // we currently ignore the case where rval is -1
262 if (memcached_is_udp(server
->root
)) {
266 #ifdef HAVE_SO_SNDTIMEO
267 if (server
->root
->snd_timeout
> 0) {
268 struct timeval waittime
;
270 waittime
.tv_sec
= server
->root
->snd_timeout
/ 1000000;
271 waittime
.tv_usec
= server
->root
->snd_timeout
% 1000000;
273 int error
= setsockopt(server
->fd
, SOL_SOCKET
, SO_SNDTIMEO
, (char *) &waittime
,
274 (socklen_t
) sizeof(struct timeval
));
280 #ifdef HAVE_SO_RCVTIMEO
281 if (server
->root
->rcv_timeout
> 0) {
282 struct timeval waittime
;
284 waittime
.tv_sec
= server
->root
->rcv_timeout
/ 1000000;
285 waittime
.tv_usec
= server
->root
->rcv_timeout
% 1000000;
287 int error
= setsockopt(server
->fd
, SOL_SOCKET
, SO_RCVTIMEO
, (char *) &waittime
,
288 (socklen_t
) sizeof(struct timeval
));
296 # if defined(SO_NOSIGPIPE)
299 int error
= setsockopt(server
->fd
, SOL_SOCKET
, SO_NOSIGPIPE
, (void *) &set
, sizeof(int));
303 // This is not considered a fatal error
306 perror("setsockopt(SO_NOSIGPIPE)");
310 # endif // SO_NOSIGPIPE
313 if (server
->root
->flags
.no_block
) {
314 struct linger linger
;
317 linger
.l_linger
= 0; /* By default on close() just drop the socket */
318 int error
= setsockopt(server
->fd
, SOL_SOCKET
, SO_LINGER
, (char *) &linger
,
319 (socklen_t
) sizeof(struct linger
));
325 if (server
->root
->flags
.tcp_nodelay
) {
329 setsockopt(server
->fd
, IPPROTO_TCP
, TCP_NODELAY
, (char *) &flag
, (socklen_t
) sizeof(int));
335 if (server
->root
->flags
.tcp_keepalive
) {
339 setsockopt(server
->fd
, SOL_SOCKET
, SO_KEEPALIVE
, (char *) &flag
, (socklen_t
) sizeof(int));
345 if (server
->root
->tcp_keepidle
> 0) {
346 int error
= setsockopt(server
->fd
, IPPROTO_TCP
, TCP_KEEPIDLE
,
347 (char *) &server
->root
->tcp_keepidle
, (socklen_t
) sizeof(int));
353 if (server
->root
->send_size
> 0) {
354 int error
= setsockopt(server
->fd
, SOL_SOCKET
, SO_SNDBUF
, (char *) &server
->root
->send_size
,
355 (socklen_t
) sizeof(int));
360 if (server
->root
->recv_size
> 0) {
361 int error
= setsockopt(server
->fd
, SOL_SOCKET
, SO_RCVBUF
, (char *) &server
->root
->recv_size
,
362 (socklen_t
) sizeof(int));
367 /* libmemcached will always use nonblocking IO to avoid write deadlocks */
368 set_socket_nonblocking(server
);
373 static memcached_return_t
unix_socket_connect(memcached_instance_st
*server
) {
375 WATCHPOINT_ASSERT(server
->fd
== INVALID_SOCKET
);
378 int type
= SOCK_STREAM
;
380 type
|= SOCK_CLOEXEC
;
384 type
|= SOCK_NONBLOCK
;
387 if ((server
->fd
= socket(AF_UNIX
, type
, 0)) == -1) {
388 return memcached_set_errno(*server
, errno
, NULL
);
391 struct sockaddr_un servAddr
;
393 memset(&servAddr
, 0, sizeof(struct sockaddr_un
));
394 servAddr
.sun_family
= AF_UNIX
;
395 if (strlen(server
->hostname()) >= sizeof(servAddr
.sun_path
)) {
396 return memcached_set_error(*server
, MEMCACHED_FAIL_UNIX_SOCKET
, MEMCACHED_AT
);
398 strncpy(servAddr
.sun_path
, server
->hostname(),
399 sizeof(servAddr
.sun_path
) - 1); /* Copy filename */
401 if (connect(server
->fd
, (struct sockaddr
*) &servAddr
, sizeof(servAddr
)) == -1) {
406 server
->events(POLLOUT
);
410 server
->reset_socket();
413 case EISCONN
: /* We were spinning waiting on connect */
415 assert(0); // Programmer error
416 server
->reset_socket();
421 WATCHPOINT_ERRNO(errno
);
422 server
->reset_socket();
423 return memcached_set_errno(*server
, errno
, MEMCACHED_AT
);
427 server
->state
= MEMCACHED_SERVER_STATE_CONNECTED
;
429 WATCHPOINT_ASSERT(server
->fd
!= INVALID_SOCKET
);
431 return MEMCACHED_SUCCESS
;
434 return MEMCACHED_NOT_SUPPORTED
;
438 static memcached_return_t
network_connect(memcached_instance_st
*server
) {
439 bool timeout_error_occured
= false;
441 WATCHPOINT_ASSERT(server
->fd
== INVALID_SOCKET
);
442 WATCHPOINT_ASSERT(server
->cursor_active_
== 0);
445 We want to check both of these because if address_info_next has been fully tried, we want to do
446 a new lookup to make sure we have picked up on any new DNS information.
448 if (server
->address_info
== NULL
or server
->address_info_next
== NULL
) {
449 WATCHPOINT_ASSERT(server
->state
== MEMCACHED_SERVER_STATE_NEW
);
450 server
->address_info_next
= NULL
;
451 memcached_return_t rc
= set_hostinfo(server
);
453 if (memcached_failed(rc
)) {
458 assert(server
->address_info_next
);
459 assert(server
->address_info
);
461 /* Create the socket */
462 while (server
->address_info_next
and server
->fd
== INVALID_SOCKET
) {
463 int type
= server
->address_info_next
->ai_socktype
;
465 type
|= SOCK_CLOEXEC
;
469 type
|= SOCK_NONBLOCK
;
473 socket(server
->address_info_next
->ai_family
, type
, server
->address_info_next
->ai_protocol
);
475 if (int(server
->fd
) == SOCKET_ERROR
) {
476 return memcached_set_errno(*server
, get_socket_errno(), NULL
);
479 if (set_socket_options(server
) == false) {
480 server
->reset_socket();
481 return MEMCACHED_CONNECTION_FAILURE
;
484 /* connect to server */
485 if ((connect(server
->fd
, server
->address_info_next
->ai_addr
,
486 server
->address_info_next
->ai_addrlen
)
489 server
->state
= MEMCACHED_SERVER_STATE_CONNECTED
;
490 return MEMCACHED_SUCCESS
;
493 /* An error occurred */
494 int local_error
= get_socket_errno();
495 switch (local_error
) {
497 timeout_error_occured
= true;
500 #if EWOULDBLOCK != EAGAIN
504 case EINPROGRESS
: // nonblocking mode - first return
505 case EALREADY
: // nonblocking mode - subsequent returns
507 server
->events(POLLOUT
);
508 server
->state
= MEMCACHED_SERVER_STATE_IN_PROGRESS
;
509 memcached_return_t rc
= connect_poll(server
, local_error
);
511 if (memcached_success(rc
)) {
512 server
->state
= MEMCACHED_SERVER_STATE_CONNECTED
;
513 return MEMCACHED_SUCCESS
;
516 // A timeout here is treated as an error, we will not retry
517 if (rc
== MEMCACHED_TIMEOUT
) {
518 timeout_error_occured
= true;
522 case EISCONN
: // we are connected :-)
523 WATCHPOINT_ASSERT(0); // This is a programmer's error
526 case EINTR
: // Special case, we retry ai_addr
527 WATCHPOINT_ASSERT(server
->fd
!= INVALID_SOCKET
);
528 server
->reset_socket();
532 // Probably not running service
535 memcached_set_errno(*server
, local_error
, MEMCACHED_AT
);
539 WATCHPOINT_ASSERT(server
->fd
!= INVALID_SOCKET
);
540 server
->reset_socket();
541 server
->address_info_next
= server
->address_info_next
->ai_next
;
544 WATCHPOINT_ASSERT(server
->fd
== INVALID_SOCKET
);
546 if (timeout_error_occured
) {
547 server
->reset_socket();
550 WATCHPOINT_STRING("Never got a good file descriptor");
552 if (memcached_has_current_error(*server
)) {
553 return memcached_instance_error_return(server
);
556 if (timeout_error_occured
and server
->state
< MEMCACHED_SERVER_STATE_IN_PROGRESS
) {
557 return memcached_set_error(
558 *server
, MEMCACHED_TIMEOUT
, MEMCACHED_AT
,
559 memcached_literal_param(
560 "if (timeout_error_occured and server->state < MEMCACHED_SERVER_STATE_IN_PROGRESS)"));
563 return memcached_set_error(*server
, MEMCACHED_CONNECTION_FAILURE
,
564 MEMCACHED_AT
); /* The last error should be from connect() */
570 Based on time/failure count fail the connect without trying. This prevents waiting in a state
571 where we get caught spending cycles just waiting.
573 static memcached_return_t
backoff_handling(memcached_instance_st
*server
, bool &in_timeout
) {
574 struct timeval curr_time
;
575 bool _gettime_success
= (gettimeofday(&curr_time
, NULL
) == 0);
578 If we hit server_failure_limit then something is completely wrong about the server.
580 1) If autoeject is enabled we do that.
581 2) If not? We go into timeout again, there is much else to do :(
583 if (server
->server_failure_counter
>= server
->root
->server_failure_limit
) {
585 We just auto_eject if we hit this point
587 if (_is_auto_eject_host(server
->root
)) {
588 set_last_disconnected_host(server
);
590 // Retry dead servers if requested
591 if (_gettime_success
and server
->root
->dead_timeout
> 0) {
592 server
->next_retry
= curr_time
.tv_sec
+ server
->root
->dead_timeout
;
594 // We only retry dead servers once before assuming failure again
595 server
->server_failure_counter
= server
->root
->server_failure_limit
- 1;
598 memcached_return_t rc
;
599 if (memcached_failed(rc
= run_distribution((memcached_st
*) server
->root
))) {
600 return memcached_set_error(
601 *server
, rc
, MEMCACHED_AT
,
602 memcached_literal_param("Backoff handling failed during run_distribution"));
605 return memcached_set_error(*server
, MEMCACHED_SERVER_MARKED_DEAD
, MEMCACHED_AT
);
608 server
->state
= MEMCACHED_SERVER_STATE_IN_TIMEOUT
;
610 // Sanity check/setting
611 if (server
->next_retry
== 0) {
612 server
->next_retry
= 1;
616 if (server
->state
== MEMCACHED_SERVER_STATE_IN_TIMEOUT
) {
618 If next_retry is less then our current time, then we reset and try everything again.
620 if (_gettime_success
and server
->next_retry
< curr_time
.tv_sec
) {
621 server
->state
= MEMCACHED_SERVER_STATE_NEW
;
622 server
->server_timeout_counter
= 0;
624 return memcached_set_error(*server
, MEMCACHED_SERVER_TEMPORARILY_DISABLED
, MEMCACHED_AT
);
630 return MEMCACHED_SUCCESS
;
633 static memcached_return_t
_memcached_connect(memcached_instance_st
*server
,
634 const bool set_last_disconnected
) {
636 if (server
->fd
!= INVALID_SOCKET
) {
637 return MEMCACHED_SUCCESS
;
640 LIBMEMCACHED_MEMCACHED_CONNECT_START();
642 bool in_timeout
= false;
643 memcached_return_t rc
;
644 if (memcached_failed(rc
= backoff_handling(server
, in_timeout
))) {
645 set_last_disconnected_host(server
);
649 if (LIBMEMCACHED_WITH_SASL_SUPPORT
and server
->root
->sasl
.callbacks
650 and memcached_is_udp(server
->root
))
652 return memcached_set_error(
653 *server
, MEMCACHED_INVALID_HOST_PROTOCOL
, MEMCACHED_AT
,
654 memcached_literal_param("SASL is not supported for UDP connections"));
657 if (server
->hostname()[0] == '/') {
658 server
->type
= MEMCACHED_CONNECTION_UNIX_SOCKET
;
661 /* We need to clean up the multi startup piece */
662 switch (server
->type
) {
663 case MEMCACHED_CONNECTION_UDP
:
664 case MEMCACHED_CONNECTION_TCP
:
665 rc
= network_connect(server
);
667 #if defined(LIBMEMCACHED_WITH_SASL_SUPPORT)
668 if (LIBMEMCACHED_WITH_SASL_SUPPORT
) {
669 if (server
->fd
!= INVALID_SOCKET
and server
->root
->sasl
.callbacks
) {
670 rc
= memcached_sasl_authenticate_connection(server
);
671 if (memcached_failed(rc
) and server
->fd
!= INVALID_SOCKET
) {
672 WATCHPOINT_ASSERT(server
->fd
!= INVALID_SOCKET
);
673 server
->reset_socket();
680 case MEMCACHED_CONNECTION_UNIX_SOCKET
:
681 rc
= unix_socket_connect(server
);
685 if (memcached_success(rc
)) {
686 server
->mark_server_as_clean();
687 memcached_version_instance(server
);
689 } else if (set_last_disconnected
) {
690 set_last_disconnected_host(server
);
691 if (memcached_has_current_error(*server
)) {
692 memcached_mark_server_for_timeout(server
);
693 assert(memcached_failed(memcached_instance_error_return(server
)));
695 memcached_set_error(*server
, rc
, MEMCACHED_AT
);
696 memcached_mark_server_for_timeout(server
);
699 LIBMEMCACHED_MEMCACHED_CONNECT_END();
703 int snprintf_length
=
704 snprintf(buffer
, sizeof(buffer
), "%s:%d", server
->hostname(), int(server
->port()));
705 return memcached_set_error(*server
, MEMCACHED_SERVER_TEMPORARILY_DISABLED
, MEMCACHED_AT
,
706 buffer
, snprintf_length
);
713 memcached_return_t
memcached_connect(memcached_instance_st
*server
) {
714 return _memcached_connect(server
, true);