From e058a9fe5370577c4b8d722dde6bb1f4063def3c Mon Sep 17 00:00:00 2001 From: Aperence Date: Fri, 6 Sep 2024 20:27:36 +0200 Subject: [PATCH 1/4] Core: added socket protocol Multipath TCP (MPTCP), standardized in RFC8684 [1], is a TCP extension that enables a TCP connection to use different paths. Multipath TCP has been used for several use cases. On smartphones, MPTCP enables seamless handovers between cellular and Wi-Fi networks while preserving Established connections. This use-case is what pushed Apple to use MPTCP since 2013 in multiple applications [2]. On dual-stack hosts, Multipath TCP enables the TCP connection to automatically use the best performing path, either IPv4 or IPv6. If one path fails, MPTCP automatically uses the other path. The benefit from MPTCP, both the client and the server have to support it. Multipath TCP is a backward-compatible TCP extension that is enabled by default on recent Linux distributions (Debian, Ubuntu, Redhat, ...). Multipath TCP is included in the Linux kernel since version 5.6 [3]. To use it on Linux, an application must explicitly enable it when creating the socket. No need to change anything else in the application. Even if MPTCP is supported by different OS, only Linux supports the `IPPROTO_MPTCP` protocol, which is why this feature is currently limited to Linux only. This patch updates the creation of listening sockets to use a new field of the `ngx_listening_s` structure. The `protocol` field can be used in conjunction with the `type` to specify the protocol to be used. Modules will then be able to specify a different protocol, e.g. IPPROTO_MPTCP. Co-authored-by: Maxime Dourov Link: https://www.rfc-editor.org/rfc/rfc8684.html [1] Link: https://www.tessares.net/apples-mptcp-story-so-far/ [2] Link: https://www.mptcp.dev [3] --- src/core/ngx_connection.c | 3 ++- src/core/ngx_connection.h | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/core/ngx_connection.c b/src/core/ngx_connection.c index 75809d9ad..eb6b6af5c 100644 --- a/src/core/ngx_connection.c +++ b/src/core/ngx_connection.c @@ -487,7 +487,8 @@ ngx_open_listening_sockets(ngx_cycle_t *cycle) continue; } - s = ngx_socket(ls[i].sockaddr->sa_family, ls[i].type, 0); + s = ngx_socket(ls[i].sockaddr->sa_family, ls[i].type, ++ ls[i].protocol); if (s == (ngx_socket_t) -1) { ngx_log_error(NGX_LOG_EMERG, log, ngx_socket_errno, diff --git a/src/core/ngx_connection.h b/src/core/ngx_connection.h index 84dd80442..80caaf741 100644 --- a/src/core/ngx_connection.h +++ b/src/core/ngx_connection.h @@ -24,6 +24,7 @@ struct ngx_listening_s { ngx_str_t addr_text; int type; + int protocol; int backlog; int rcvbuf; From f33492028ea1bc89d72371eb04d91f1477306b48 Mon Sep 17 00:00:00 2001 From: Aperence Date: Fri, 6 Sep 2024 20:29:32 +0200 Subject: [PATCH 2/4] HTTP: added MPTCP support. Multipath TCP (MPTCP), standardized in RFC8684 [1], is a TCP extension that enables a TCP connection to use different paths. Multipath TCP has been used for several use cases. On smartphones, MPTCP enables seamless handovers between cellular and Wi-Fi networks while preserving Established connections. This use-case is what pushed Apple to use MPTCP since 2013 in multiple applications [2]. On dual-stack hosts, Multipath TCP enables the TCP connection to automatically use the best performing path, either IPv4 or IPv6. If one path fails, MPTCP automatically uses the other path. The benefit from MPTCP, both the client and the server have to support it. Multipath TCP is a backward-compatible TCP extension that is enabled by default on recent Linux distributions (Debian, Ubuntu, Redhat, ...). Multipath TCP is included in the Linux kernel since version 5.6 [3]. To use it on Linux, an application must explicitly enable it when creating the socket. No need to change anything else in the application. Even if MPTCP is supported by different OS, only Linux supports the `IPPROTO_MPTCP` protocol, which is why this feature is currently limited to Linux only. This patch adds a new parameter 'multipath' to the 'listen' directive in the HTTP module. This new parameter is only compatible with TCP if IPPROTO_MPTCP is defined, not with QUIC so far. Co-authored-by: Maxime Dourov Link: https://www.rfc-editor.org/rfc/rfc8684.html [1] Link: https://www.tessares.net/apples-mptcp-story-so-far/ [2] Link: https://www.mptcp.dev [3] --- contrib/vim/syntax/nginx.vim | 2 +- src/http/ngx_http.c | 1 + src/http/ngx_http_core_module.c | 13 +++++++++++++ src/http/ngx_http_core_module.h | 1 + 4 files changed, 16 insertions(+), 1 deletion(-) diff --git a/contrib/vim/syntax/nginx.vim b/contrib/vim/syntax/nginx.vim index 29eef7a23..ea7c58464 100644 --- a/contrib/vim/syntax/nginx.vim +++ b/contrib/vim/syntax/nginx.vim @@ -65,7 +65,7 @@ syn match ngxListenComment '#.*$' \ contained \ nextgroup=@ngxListenParams skipwhite skipempty syn keyword ngxListenOptions contained - \ default_server ssl quic proxy_protocol + \ default_server ssl quic proxy_protocol multipath \ setfib fastopen backlog rcvbuf sndbuf accept_filter deferred bind \ ipv6only reuseport so_keepalive \ nextgroup=@ngxListenParams skipwhite skipempty diff --git a/src/http/ngx_http.c b/src/http/ngx_http.c index d835f896e..c00b2e27b 100644 --- a/src/http/ngx_http.c +++ b/src/http/ngx_http.c @@ -1845,6 +1845,7 @@ ngx_http_add_listening(ngx_conf_t *cf, ngx_http_conf_addr_t *addr) #endif ls->type = addr->opt.type; + ls->protocol = addr->opt.protocol; ls->backlog = addr->opt.backlog; ls->rcvbuf = addr->opt.rcvbuf; ls->sndbuf = addr->opt.sndbuf; diff --git a/src/http/ngx_http_core_module.c b/src/http/ngx_http_core_module.c index 033a3bf64..9c539310a 100644 --- a/src/http/ngx_http_core_module.c +++ b/src/http/ngx_http_core_module.c @@ -4049,6 +4049,13 @@ ngx_http_core_listen(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) } #endif +#ifdef IPPROTO_MPTCP + if (ngx_strcmp(value[n].data, "multipath") == 0) { + lsopt.protocol = IPPROTO_MPTCP; + continue; + } +#endif + if (ngx_strncmp(value[n].data, "backlog=", 8) == 0) { lsopt.backlog = ngx_atoi(value[n].data + 8, value[n].len - 8); lsopt.set = 1; @@ -4338,6 +4345,12 @@ ngx_http_core_listen(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) } #endif +#ifdef IPPROTO_MPTCP + if (lsopt.protocol == IPPROTO_MPTCP) { + return "\"multipath\" parameter is incompatible with \"quic\""; + } +#endif + #if (NGX_HTTP_V2) if (lsopt.http2) { return "\"http2\" parameter is incompatible with \"quic\""; diff --git a/src/http/ngx_http_core_module.h b/src/http/ngx_http_core_module.h index 765e7ff60..9862eaa27 100644 --- a/src/http/ngx_http_core_module.h +++ b/src/http/ngx_http_core_module.h @@ -88,6 +88,7 @@ typedef struct { int rcvbuf; int sndbuf; int type; + int protocol; #if (NGX_HAVE_SETFIB) int setfib; #endif From 8b8e2ed3ef2029df400be185edcefc20be7652ff Mon Sep 17 00:00:00 2001 From: Aperence Date: Fri, 6 Sep 2024 20:31:31 +0200 Subject: [PATCH 3/4] Mail: added MPTCP support. Multipath TCP (MPTCP), standardized in RFC8684 [1], is a TCP extension that enables a TCP connection to use different paths. Multipath TCP has been used for several use cases. On smartphones, MPTCP enables seamless handovers between cellular and Wi-Fi networks while preserving Established connections. This use-case is what pushed Apple to use MPTCP since 2013 in multiple applications [2]. On dual-stack hosts, Multipath TCP enables the TCP connection to automatically use the best performing path, either IPv4 or IPv6. If one path fails, MPTCP automatically uses the other path. The benefit from MPTCP, both the client and the server have to support it. Multipath TCP is a backward-compatible TCP extension that is enabled by default on recent Linux distributions (Debian, Ubuntu, Redhat, ...). Multipath TCP is included in the Linux kernel since version 5.6 [3]. To use it on Linux, an application must explicitly enable it when creating the socket. No need to change anything else in the application. Even if MPTCP is supported by different OS, only Linux supports the `IPPROTO_MPTCP` protocol, which is why this feature is currently limited to Linux only. This patch adds a new parameter 'multipath' to the 'listen' directive in the Mail module. Co-authored-by: Maxime Dourov Link: https://www.rfc-editor.org/rfc/rfc8684.html [1] Link: https://www.tessares.net/apples-mptcp-story-so-far/ [2] Link: https://www.mptcp.dev [3] --- src/mail/ngx_mail.c | 1 + src/mail/ngx_mail.h | 1 + src/mail/ngx_mail_core_module.c | 7 +++++++ 3 files changed, 9 insertions(+) diff --git a/src/mail/ngx_mail.c b/src/mail/ngx_mail.c index 890d8153a..cc34cb365 100644 --- a/src/mail/ngx_mail.c +++ b/src/mail/ngx_mail.c @@ -332,6 +332,7 @@ ngx_mail_optimize_servers(ngx_conf_t *cf, ngx_array_t *ports) ls->log.data = &ls->addr_text; ls->log.handler = ngx_accept_log_error; + ls->protocol = addr[i].opt.protocol; ls->backlog = addr[i].opt.backlog; ls->rcvbuf = addr[i].opt.rcvbuf; ls->sndbuf = addr[i].opt.sndbuf; diff --git a/src/mail/ngx_mail.h b/src/mail/ngx_mail.h index e0c62b7ab..221faf67d 100644 --- a/src/mail/ngx_mail.h +++ b/src/mail/ngx_mail.h @@ -47,6 +47,7 @@ typedef struct { int tcp_keepintvl; int tcp_keepcnt; #endif + int protocol; int backlog; int rcvbuf; int sndbuf; diff --git a/src/mail/ngx_mail_core_module.c b/src/mail/ngx_mail_core_module.c index 228a8d0a8..7b6fea1a3 100644 --- a/src/mail/ngx_mail_core_module.c +++ b/src/mail/ngx_mail_core_module.c @@ -468,6 +468,13 @@ ngx_mail_core_listen(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) #endif } +#ifdef IPPROTO_MPTCP + if (ngx_strcmp(value[i].data, "multipath") == 0) { + ls->protocol = IPPROTO_MPTCP; + continue; + } +#endif + if (ngx_strncmp(value[i].data, "so_keepalive=", 13) == 0) { if (ngx_strcmp(&value[i].data[13], "on") == 0) { From 90f6d089397506dd8df8304fc9d89481a88d2b76 Mon Sep 17 00:00:00 2001 From: Aperence Date: Fri, 6 Sep 2024 20:35:41 +0200 Subject: [PATCH 4/4] Stream: added MPTCP support. Multipath TCP (MPTCP), standardized in RFC8684 [1], is a TCP extension that enables a TCP connection to use different paths. Multipath TCP has been used for several use cases. On smartphones, MPTCP enables seamless handovers between cellular and Wi-Fi networks while preserving Established connections. This use-case is what pushed Apple to use MPTCP since 2013 in multiple applications [2]. On dual-stack hosts, Multipath TCP enables the TCP connection to automatically use the best performing path, either IPv4 or IPv6. If one path fails, MPTCP automatically uses the other path. The benefit from MPTCP, both the client and the server have to support it. Multipath TCP is a backward-compatible TCP extension that is enabled by default on recent Linux distributions (Debian, Ubuntu, Redhat, ...). Multipath TCP is included in the Linux kernel since version 5.6 [3]. To use it on Linux, an application must explicitly enable it when creating the socket. No need to change anything else in the application. Even if MPTCP is supported by different OS, only Linux supports the `IPPROTO_MPTCP` protocol, which is why this feature is currently limited to Linux only. This patch adds a new parameter 'multipath' to the 'listen' directive in the Stream module. This new parameter is only compatible with TCP if IPPROTO_MPTCP is defined, not with QUIC so far. Co-authored-by: Maxime Dourov Link: https://www.rfc-editor.org/rfc/rfc8684.html [1] Link: https://www.tessares.net/apples-mptcp-story-so-far/ [2] Link: https://www.mptcp.dev [3] --- src/stream/ngx_stream.c | 1 + src/stream/ngx_stream.h | 1 + src/stream/ngx_stream_core_module.c | 13 +++++++++++++ 3 files changed, 15 insertions(+) diff --git a/src/stream/ngx_stream.c b/src/stream/ngx_stream.c index b6eeb23af..0194d6095 100644 --- a/src/stream/ngx_stream.c +++ b/src/stream/ngx_stream.c @@ -1010,6 +1010,7 @@ ngx_stream_add_listening(ngx_conf_t *cf, ngx_stream_conf_addr_t *addr) ls->log.handler = ngx_accept_log_error; ls->type = addr->opt.type; + ls->protocol = addr[i].opt.protocol; ls->backlog = addr->opt.backlog; ls->rcvbuf = addr->opt.rcvbuf; ls->sndbuf = addr->opt.sndbuf; diff --git a/src/stream/ngx_stream.h b/src/stream/ngx_stream.h index dc05dc5ba..9bc689e99 100644 --- a/src/stream/ngx_stream.h +++ b/src/stream/ngx_stream.h @@ -62,6 +62,7 @@ typedef struct { int rcvbuf; int sndbuf; int type; + int protocol; #if (NGX_HAVE_SETFIB) int setfib; #endif diff --git a/src/stream/ngx_stream_core_module.c b/src/stream/ngx_stream_core_module.c index 40951c291..06d16f40c 100644 --- a/src/stream/ngx_stream_core_module.c +++ b/src/stream/ngx_stream_core_module.c @@ -984,6 +984,13 @@ ngx_stream_core_listen(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) } #endif +#ifdef IPPROTO_MPTCP + if (ngx_strcmp(value[i].data, "multipath") == 0) { + lsopt.protocol = IPPROTO_MPTCP; + continue; + } +#endif + if (ngx_strncmp(value[i].data, "backlog=", 8) == 0) { lsopt.backlog = ngx_atoi(value[i].data + 8, value[i].len - 8); lsopt.set = 1; @@ -1252,6 +1259,12 @@ ngx_stream_core_listen(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) } } +#ifdef IPPROTO_MPTCP + if (lsopt.protocol == IPPROTO_MPTCP) { + return "\"multipath\" parameter is incompatible with \"udp\""; + } +#endif + for (n = 0; n < u.naddrs; n++) { for (i = 0; i < n; i++) {