SSL: ignored EIO after shutdown with kernel TLS

After SSL shutdown, lingering close reads the socket with recv().  With
kernel TLS receive offload, which OpenSSL enables for TLS 1.3 since
3.2.0, recv() fails with EIO on a record other than application data,
such as the "close notify" alert from the client, and this was logged
at the "alert" level.

Now ngx_ssl_shutdown() marks connections with kernel TLS receive
offload, and EIO on such connections is ignored, so the read ends as
at EOF.

Reported by soukichi on GitHub.

Closes: https://github.com/nginx/nginx/issues/651

src/core/ngx_connection.c

@@ -1619,6 +1619,12 @@ ngx_connection_error(ngx_connection_t *c, ngx_err_t err, char *text)
         return 0;
     }
 
+#if (NGX_LINUX)
+    if (err == NGX_EIO && c->ktls_recv) {
+        return 0;
+    }
+#endif
+
     if (err == 0
         || err == NGX_ECONNRESET
 #if (NGX_WIN32)

src/core/ngx_connection.h

@@ -198,6 +198,8 @@ struct ngx_connection_s {
     unsigned            busy_count:2;
 #endif
 
+    unsigned            ktls_recv:1;
+
 #if (NGX_THREADS || NGX_COMPAT)
     ngx_thread_task_t  *sendfile_task;
 #endif

src/event/ngx_event_openssl.c

@@ -3811,6 +3811,19 @@ failed:
 
 done:
 
+#if (defined BIO_get_ktls_recv && NGX_LINUX)
+
+    /*
+     * with kernel TLS, recv() fails with EIO on a record
+     * other than application data, such as "close notify"
+     */
+
+    if (BIO_get_ktls_recv(SSL_get_rbio(c->ssl->connection)) == 1) {
+        c->ktls_recv = 1;
+    }
+
+#endif
+
     if (c->ssl->shutdown_without_free) {
         c->ssl->shutdown_without_free = 0;
         c->recv = ngx_recv;

src/os/unix/ngx_errno.h

@@ -20,6 +20,7 @@ typedef int               ngx_err_t;
 #define NGX_ENOPATH       ENOENT
 #define NGX_ESRCH         ESRCH
 #define NGX_EINTR         EINTR
+#define NGX_EIO           EIO
 #define NGX_ECHILD        ECHILD
 #define NGX_ENOMEM        ENOMEM
 #define NGX_EACCES        EACCES