From fd4078d11c681335a4fd7a0949d49bd5513c1aae Mon Sep 17 00:00:00 2001 From: tangxiao187 Date: Sat, 8 Feb 2025 10:55:49 +0800 Subject: [PATCH] fix quic request block after reload issue:#425 After nginx reloading, old workers become shuttingdown if there are still in-flight requests, and old quic reuseport socket still open before all shuttingdown workers exit. New quic conn packet received on old quic socket binding to old workers will not be processed, thus new quic request blocked after reload. To fix this problem: 1. In ngx_close_listening_sockets, close old udp socket unrelated to current worker; 2. Shutingdown worker send retry packet with routable magic_cid when receive new quic conn; 3. Use quic_bpf to redirect packet with magic_cid to new worker. --- src/core/ngx_connection.c | 10 ++++++++- src/event/quic/ngx_event_quic.c | 15 ++++++++++--- src/event/quic/ngx_event_quic_bpf.c | 30 +++++++++++++++++--------- src/event/quic/ngx_event_quic_connid.h | 14 ++++++++++++ src/event/quic/ngx_event_quic_output.c | 9 +++++++- src/event/quic/ngx_event_quic_output.h | 2 +- 6 files changed, 64 insertions(+), 16 deletions(-) diff --git a/src/core/ngx_connection.c b/src/core/ngx_connection.c index 75809d9ad..ea584a55a 100644 --- a/src/core/ngx_connection.c +++ b/src/core/ngx_connection.c @@ -1111,7 +1111,15 @@ ngx_close_listening_sockets(ngx_cycle_t *cycle) #if (NGX_QUIC) if (ls[i].quic) { - continue; +#if (NGX_HAVE_REUSEPORT) + if (ls[i].reuseport) { + // close quic reuseport fd unrelated to current worker + if (ngx_process == NGX_PROCESS_WORKER && ls[i].worker == ngx_worker) { + continue; + } + } else +#endif + continue; } #endif diff --git a/src/event/quic/ngx_event_quic.c b/src/event/quic/ngx_event_quic.c index 308597e27..02306b84f 100644 --- a/src/event/quic/ngx_event_quic.c +++ b/src/event/quic/ngx_event_quic.c @@ -911,22 +911,31 @@ ngx_quic_handle_packet(ngx_connection_t *c, ngx_quic_conf_t *conf, "invalid address validation token"); } else if (conf->retry) { /* invalid NEW_TOKEN */ - return ngx_quic_send_retry(c, conf, pkt); + return ngx_quic_send_retry(c, conf, pkt, 0); } } /* NGX_OK */ } else if (conf->retry) { - return ngx_quic_send_retry(c, conf, pkt); + return ngx_quic_send_retry(c, conf, pkt, 0); } else { pkt->odcid = pkt->dcid; } if (ngx_terminate || ngx_exiting) { +#if (NGX_QUIC_BPF) + // fix quic request block after reload: + // 1. shutingdown worker send retry with magic cid when receive new quic conn + // 2. bpf redirect magic cid conn to new worker + ngx_quic_bpf_conf_t *bcf = ngx_quic_bpf_get_conf(ngx_cycle); + if (bcf->enabled) { + return ngx_quic_send_retry(c, conf, pkt, 1); + } +#endif if (conf->retry) { - return ngx_quic_send_retry(c, conf, pkt); + return ngx_quic_send_retry(c, conf, pkt, 0); } return NGX_ERROR; diff --git a/src/event/quic/ngx_event_quic_bpf.c b/src/event/quic/ngx_event_quic_bpf.c index ab024ad56..871fe6254 100644 --- a/src/event/quic/ngx_event_quic_bpf.c +++ b/src/event/quic/ngx_event_quic_bpf.c @@ -7,15 +7,13 @@ #include #include +#include #define NGX_QUIC_BPF_VARNAME "NGINX_BPF_MAPS" #define NGX_QUIC_BPF_VARSEP ';' #define NGX_QUIC_BPF_ADDRSEP '#' -#define ngx_quic_bpf_get_conf(cycle) \ - (ngx_quic_bpf_conf_t *) ngx_get_conf(cycle->conf_ctx, ngx_quic_bpf_module) - #define ngx_quic_bpf_get_old_conf(cycle) \ cycle->old_cycle->conf_ctx ? ngx_quic_bpf_get_conf(cycle->old_cycle) \ : NULL @@ -27,6 +25,7 @@ typedef struct { ngx_queue_t queue; int map_fd; + int sock_cnt; struct sockaddr *sockaddr; socklen_t socklen; @@ -34,13 +33,6 @@ typedef struct { } ngx_quic_sock_group_t; -typedef struct { - ngx_flag_t enabled; - ngx_uint_t map_size; - ngx_queue_t groups; /* of ngx_quic_sock_group_t */ -} ngx_quic_bpf_conf_t; - - static void *ngx_quic_bpf_create_conf(ngx_cycle_t *cycle); static ngx_int_t ngx_quic_bpf_module_init(ngx_cycle_t *cycle); @@ -114,6 +106,7 @@ ngx_quic_bpf_create_conf(ngx_cycle_t *cycle) bcf->enabled = NGX_CONF_UNSET; bcf->map_size = NGX_CONF_UNSET_UINT; + bcf->worker_map_fd = -1; ngx_queue_init(&bcf->groups); @@ -219,6 +212,10 @@ ngx_quic_bpf_cleanup(void *data) ngx_quic_bpf_close(ngx_cycle->log, grp->map_fd, "map"); } + + if (bcf->worker_map_fd != -1) { + ngx_quic_bpf_close(ngx_cycle->log, bcf->worker_map_fd, "worker map"); + } } @@ -303,6 +300,7 @@ ngx_quic_bpf_create_group(ngx_cycle_t *cycle, ngx_listening_t *ls) return NULL; } + // create BPF_MAP_TYPE_SOCKHASH with key len 64bit, need kernel >= 5.7 grp->map_fd = ngx_bpf_map_create(cycle->log, BPF_MAP_TYPE_SOCKHASH, sizeof(uint64_t), sizeof(uint64_t), bcf->map_size, 0); @@ -329,6 +327,9 @@ ngx_quic_bpf_create_group(ngx_cycle_t *cycle, ngx_listening_t *ls) ngx_bpf_program_link(&ngx_quic_reuseport_helper, "ngx_quic_sockmap", grp->map_fd); + + ngx_bpf_program_link(&ngx_quic_reuseport_helper, + "ngx_worker_map", bcf->worker_map_fd); progfd = ngx_bpf_load_program(cycle->log, &ngx_quic_reuseport_helper); if (progfd < 0) { @@ -448,12 +449,21 @@ ngx_quic_bpf_group_add_socket(ngx_cycle_t *cycle, ngx_listening_t *ls) return NGX_ERROR; } + /* map[magic_cid] = socket; for use in kernel helper */ + uint64_t tmp_key = REDIRECT_WORKER_CID_MAGIC + grp->sock_cnt; + if (ngx_bpf_map_update(grp->map_fd, &tmp_key, &ls->fd, BPF_ANY) == -1) { + ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno, + "quic bpf failed to update socket map key=%xL", tmp_key); + return NGX_ERROR; + } + ngx_log_debug4(NGX_LOG_DEBUG_EVENT, cycle->log, 0, "quic bpf sockmap fd:%d add socket:%d cookie:0x%xL worker:%ui", grp->map_fd, ls->fd, cookie, ls->worker); /* do not inherit this socket */ ls->ignore = 1; + grp->sock_cnt++; return NGX_OK; } diff --git a/src/event/quic/ngx_event_quic_connid.h b/src/event/quic/ngx_event_quic_connid.h index 33e9c65b9..55e6326df 100644 --- a/src/event/quic/ngx_event_quic_connid.h +++ b/src/event/quic/ngx_event_quic_connid.h @@ -11,6 +11,20 @@ #include #include +typedef struct { + ngx_flag_t enabled; + ngx_uint_t map_size; + int worker_map_fd; + ngx_queue_t groups; /* of ngx_quic_sock_group_t */ +} ngx_quic_bpf_conf_t; + +extern ngx_module_t ngx_quic_bpf_module; +#define ngx_quic_bpf_get_conf(cycle) \ + (ngx_quic_bpf_conf_t *) ngx_get_conf(cycle->conf_ctx, ngx_quic_bpf_module) + +// shuttingdown worker use magic_cid to redirect received new quic conn to new worker. +// 0x6e67696e78 refers to "nginx". +#define REDIRECT_WORKER_CID_MAGIC 0x6e67696e78000000ULL ngx_int_t ngx_quic_handle_retire_connection_id_frame(ngx_connection_t *c, ngx_quic_retire_cid_frame_t *f); diff --git a/src/event/quic/ngx_event_quic_output.c b/src/event/quic/ngx_event_quic_output.c index f087e2bfa..ae53de286 100644 --- a/src/event/quic/ngx_event_quic_output.c +++ b/src/event/quic/ngx_event_quic_output.c @@ -967,7 +967,7 @@ ngx_quic_send_early_cc(ngx_connection_t *c, ngx_quic_header_t *inpkt, ngx_int_t ngx_quic_send_retry(ngx_connection_t *c, ngx_quic_conf_t *conf, - ngx_quic_header_t *inpkt) + ngx_quic_header_t *inpkt, ngx_uint_t redirect_worker) { time_t expires; ssize_t len; @@ -1003,6 +1003,13 @@ ngx_quic_send_retry(ngx_connection_t *c, ngx_quic_conf_t *conf, return NGX_ERROR; } +#if (NGX_QUIC_BPF) + if (redirect_worker) { + uint64_t tmp_key = REDIRECT_WORKER_CID_MAGIC + ngx_worker; + ngx_quic_dcid_encode_key(dcid, tmp_key); + } +#endif + pkt.scid.len = NGX_QUIC_SERVER_CID_LEN; pkt.scid.data = dcid; diff --git a/src/event/quic/ngx_event_quic_output.h b/src/event/quic/ngx_event_quic_output.h index 52d8a374f..fc3b92711 100644 --- a/src/event/quic/ngx_event_quic_output.h +++ b/src/event/quic/ngx_event_quic_output.h @@ -24,7 +24,7 @@ ngx_int_t ngx_quic_send_early_cc(ngx_connection_t *c, ngx_quic_header_t *inpkt, ngx_uint_t err, const char *reason); ngx_int_t ngx_quic_send_retry(ngx_connection_t *c, - ngx_quic_conf_t *conf, ngx_quic_header_t *pkt); + ngx_quic_conf_t *conf, ngx_quic_header_t *pkt, ngx_uint_t redirect_worker); ngx_int_t ngx_quic_send_new_token(ngx_connection_t *c, ngx_quic_path_t *path); ngx_int_t ngx_quic_send_ack(ngx_connection_t *c,