fix quic request block after reload

issue:#425
After nginx reloading, old workers become shuttingdown if there are still in-flight requests,
and old quic reuseport socket still open before all shuttingdown workers exit.
New quic conn packet received on old quic socket binding to old workers will not be processed,
thus new quic request blocked after reload.

To fix this problem:
1. In ngx_close_listening_sockets, close old udp socket unrelated to current worker;
2. Shutingdown worker send retry packet with routable magic_cid when receive new quic conn;
3. Use quic_bpf to redirect packet with magic_cid to new worker.

src/core/ngx_connection.c

@@ -1111,7 +1111,15 @@ ngx_close_listening_sockets(ngx_cycle_t *cycle)
 
 #if (NGX_QUIC)
         if (ls[i].quic) {
-            continue;
+#if (NGX_HAVE_REUSEPORT)
+            if (ls[i].reuseport) {
+                // close quic reuseport fd unrelated to current worker
+                if (ngx_process == NGX_PROCESS_WORKER && ls[i].worker == ngx_worker) {
+                    continue;
+                }
+            } else
+#endif
+                continue;
         }
 #endif
 

src/event/quic/ngx_event_quic.c

@@ -911,22 +911,31 @@ ngx_quic_handle_packet(ngx_connection_t *c, ngx_quic_conf_t *conf,
                                           "invalid address validation token");
             } else if (conf->retry) {
                 /* invalid NEW_TOKEN */
-                return ngx_quic_send_retry(c, conf, pkt);
+                return ngx_quic_send_retry(c, conf, pkt, 0);
             }
         }
 
         /* NGX_OK */
 
     } else if (conf->retry) {
-        return ngx_quic_send_retry(c, conf, pkt);
+        return ngx_quic_send_retry(c, conf, pkt, 0);
 
     } else {
         pkt->odcid = pkt->dcid;
     }
 
     if (ngx_terminate || ngx_exiting) {
+#if (NGX_QUIC_BPF)
+        // fix quic request block after reload:
+        // 1. shutingdown worker send retry with magic cid when receive new quic conn
+        // 2. bpf redirect magic cid conn to new worker
+        ngx_quic_bpf_conf_t *bcf = ngx_quic_bpf_get_conf(ngx_cycle);
+        if (bcf->enabled) {
+            return ngx_quic_send_retry(c, conf, pkt, 1);
+        }
+#endif
         if (conf->retry) {
-            return ngx_quic_send_retry(c, conf, pkt);
+            return ngx_quic_send_retry(c, conf, pkt, 0);
         }
 
         return NGX_ERROR;

src/event/quic/ngx_event_quic_bpf.c

@@ -7,15 +7,13 @@
 #include <ngx_config.h>
 #include <ngx_core.h>
 
+#include <ngx_event_quic_connection.h>
 
 #define NGX_QUIC_BPF_VARNAME  "NGINX_BPF_MAPS"
 #define NGX_QUIC_BPF_VARSEP    ';'
 #define NGX_QUIC_BPF_ADDRSEP   '#'
 
 
-#define ngx_quic_bpf_get_conf(cycle)                                          \
-    (ngx_quic_bpf_conf_t *) ngx_get_conf(cycle->conf_ctx, ngx_quic_bpf_module)
-
 #define ngx_quic_bpf_get_old_conf(cycle)                                      \
     cycle->old_cycle->conf_ctx ? ngx_quic_bpf_get_conf(cycle->old_cycle)      \
                                : NULL
@@ -27,6 +25,7 @@
 typedef struct {
     ngx_queue_t           queue;
     int                   map_fd;
+    int                   sock_cnt;
 
     struct sockaddr      *sockaddr;
     socklen_t             socklen;
@@ -34,13 +33,6 @@ typedef struct {
 } ngx_quic_sock_group_t;
 
 
-typedef struct {
-    ngx_flag_t            enabled;
-    ngx_uint_t            map_size;
-    ngx_queue_t           groups;     /* of ngx_quic_sock_group_t */
-} ngx_quic_bpf_conf_t;
-
-
 static void *ngx_quic_bpf_create_conf(ngx_cycle_t *cycle);
 static ngx_int_t ngx_quic_bpf_module_init(ngx_cycle_t *cycle);
 
@@ -114,6 +106,7 @@ ngx_quic_bpf_create_conf(ngx_cycle_t *cycle)
 
     bcf->enabled = NGX_CONF_UNSET;
     bcf->map_size = NGX_CONF_UNSET_UINT;
+    bcf->worker_map_fd = -1;
 
     ngx_queue_init(&bcf->groups);
 
@@ -219,6 +212,10 @@ ngx_quic_bpf_cleanup(void *data)
 
         ngx_quic_bpf_close(ngx_cycle->log, grp->map_fd, "map");
     }
+
+    if (bcf->worker_map_fd != -1) {
+        ngx_quic_bpf_close(ngx_cycle->log, bcf->worker_map_fd, "worker map");
+    }
 }
 
 
@@ -303,6 +300,7 @@ ngx_quic_bpf_create_group(ngx_cycle_t *cycle, ngx_listening_t *ls)
         return NULL;
     }
 
+    // create BPF_MAP_TYPE_SOCKHASH with key len 64bit, need kernel >= 5.7
     grp->map_fd = ngx_bpf_map_create(cycle->log, BPF_MAP_TYPE_SOCKHASH,
                                      sizeof(uint64_t), sizeof(uint64_t),
                                      bcf->map_size, 0);
@@ -329,6 +327,9 @@ ngx_quic_bpf_create_group(ngx_cycle_t *cycle, ngx_listening_t *ls)
 
     ngx_bpf_program_link(&ngx_quic_reuseport_helper,
                          "ngx_quic_sockmap", grp->map_fd);
+    
+    ngx_bpf_program_link(&ngx_quic_reuseport_helper,
+                         "ngx_worker_map", bcf->worker_map_fd);
 
     progfd = ngx_bpf_load_program(cycle->log, &ngx_quic_reuseport_helper);
     if (progfd < 0) {
@@ -448,12 +449,21 @@ ngx_quic_bpf_group_add_socket(ngx_cycle_t *cycle,  ngx_listening_t *ls)
         return NGX_ERROR;
     }
 
+    /* map[magic_cid] = socket; for use in kernel helper */
+    uint64_t tmp_key = REDIRECT_WORKER_CID_MAGIC + grp->sock_cnt;
+    if (ngx_bpf_map_update(grp->map_fd, &tmp_key, &ls->fd, BPF_ANY) == -1) {
+        ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno,
+                      "quic bpf failed to update socket map key=%xL", tmp_key);
+        return NGX_ERROR;
+    }
+
     ngx_log_debug4(NGX_LOG_DEBUG_EVENT, cycle->log, 0,
                  "quic bpf sockmap fd:%d add socket:%d cookie:0x%xL worker:%ui",
                  grp->map_fd, ls->fd, cookie, ls->worker);
 
     /* do not inherit this socket */
     ls->ignore = 1;
+    grp->sock_cnt++;
 
     return NGX_OK;
 }

src/event/quic/ngx_event_quic_connid.h

@@ -11,6 +11,20 @@
 #include <ngx_config.h>
 #include <ngx_core.h>
 
+typedef struct {
+    ngx_flag_t            enabled;
+    ngx_uint_t            map_size;
+    int                   worker_map_fd;
+    ngx_queue_t           groups;     /* of ngx_quic_sock_group_t */
+} ngx_quic_bpf_conf_t;
+
+extern ngx_module_t ngx_quic_bpf_module;
+#define ngx_quic_bpf_get_conf(cycle)                                          \
+    (ngx_quic_bpf_conf_t *) ngx_get_conf(cycle->conf_ctx, ngx_quic_bpf_module)
+
+// shuttingdown worker use magic_cid to redirect received new quic conn to new worker.
+// 0x6e67696e78 refers to "nginx".
+#define REDIRECT_WORKER_CID_MAGIC 0x6e67696e78000000ULL
 
 ngx_int_t ngx_quic_handle_retire_connection_id_frame(ngx_connection_t *c,
     ngx_quic_retire_cid_frame_t *f);

src/event/quic/ngx_event_quic_output.c

@@ -967,7 +967,7 @@ ngx_quic_send_early_cc(ngx_connection_t *c, ngx_quic_header_t *inpkt,
 
 ngx_int_t
 ngx_quic_send_retry(ngx_connection_t *c, ngx_quic_conf_t *conf,
-    ngx_quic_header_t *inpkt)
+    ngx_quic_header_t *inpkt, ngx_uint_t redirect_worker)
 {
     time_t             expires;
     ssize_t            len;
@@ -1003,6 +1003,13 @@ ngx_quic_send_retry(ngx_connection_t *c, ngx_quic_conf_t *conf,
         return NGX_ERROR;
     }
 
+#if (NGX_QUIC_BPF)
+    if (redirect_worker) {
+        uint64_t tmp_key = REDIRECT_WORKER_CID_MAGIC + ngx_worker;
+        ngx_quic_dcid_encode_key(dcid, tmp_key);
+    }
+#endif
+
     pkt.scid.len = NGX_QUIC_SERVER_CID_LEN;
     pkt.scid.data = dcid;
 

src/event/quic/ngx_event_quic_output.h

@@ -24,7 +24,7 @@ ngx_int_t ngx_quic_send_early_cc(ngx_connection_t *c,
     ngx_quic_header_t *inpkt, ngx_uint_t err, const char *reason);
 
 ngx_int_t ngx_quic_send_retry(ngx_connection_t *c,
-    ngx_quic_conf_t *conf, ngx_quic_header_t *pkt);
+    ngx_quic_conf_t *conf, ngx_quic_header_t *pkt, ngx_uint_t redirect_worker);
 ngx_int_t ngx_quic_send_new_token(ngx_connection_t *c, ngx_quic_path_t *path);
 
 ngx_int_t ngx_quic_send_ack(ngx_connection_t *c,