IntenseWebs/nginx
3
0
Fork 0
mirror of https://github.com/nginx/nginx.git synced 2024-12-18 21:23:36 -06:00
Code Issues Packages Projects Releases Wiki Activity

Generic subrequests in memory.

Browse Source 
Previously, only the upstream response body could be accessed with the
NGX_HTTP_SUBREQUEST_IN_MEMORY feature.  Now any response body from a subrequest
can be saved in a memory buffer.  It is available as a single buffer in r->out
and the buffer size is configured by the subrequest_output_buffer_size
directive.

Upstream, proxy and fastcgi code used to handle the old-style feature is
removed.
This commit is contained in:
Roman Arutyunyan 2018-02-28 16:56:58 +03:00
parent 2d9db482aa
commit 7c5c15a25d
7 changed files with 107 additions and 188 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
src/http/modules/ngx_http_fastcgi_module.c
View File

@ -2512,36 +2512,6 @@ ngx_http_fastcgi_non_buffered_filter(void *data, ssize_t bytes)
break;
}
/* provide continuous buffer for subrequests in memory */
if (r->subrequest_in_memory) {
cl = u->out_bufs;
if (cl) {
buf->pos = cl->buf->pos;
}
buf->last = buf->pos;
for (cl = u->out_bufs; cl; cl = cl->next) {
ngx_log_debug3(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
"http fastcgi in memory %p-%p %O",
cl->buf->pos, cl->buf->last, ngx_buf_size(cl->buf));
if (buf->last == cl->buf->pos) {
buf->last = cl->buf->last;
continue;
}
buf->last = ngx_movemem(buf->last, cl->buf->pos,
cl->buf->last - cl->buf->pos);
cl->buf->pos = buf->last - (cl->buf->last - cl->buf->pos);
cl->buf->last = buf->last;
}
}
return NGX_OK;
}

30
src/http/modules/ngx_http_proxy_module.c
View File

@ -2321,36 +2321,6 @@ ngx_http_proxy_non_buffered_chunked_filter(void *data, ssize_t bytes)
return NGX_ERROR;
}
/* provide continuous buffer for subrequests in memory */
if (r->subrequest_in_memory) {
cl = u->out_bufs;
if (cl) {
buf->pos = cl->buf->pos;
}
buf->last = buf->pos;
for (cl = u->out_bufs; cl; cl = cl->next) {
ngx_log_debug3(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
"http proxy in memory %p-%p %O",
cl->buf->pos, cl->buf->last, ngx_buf_size(cl->buf));
if (buf->last == cl->buf->pos) {
buf->last = cl->buf->last;
continue;
}
buf->last = ngx_movemem(buf->last, cl->buf->pos,
cl->buf->last - cl->buf->pos);
cl->buf->pos = buf->last - (cl->buf->last - cl->buf->pos);
cl->buf->last = buf->last;
}
}
return NGX_OK;
}

8
src/http/modules/ngx_http_ssi_filter_module.c
View File

@ -2231,9 +2231,11 @@ ngx_http_ssi_set_variable(ngx_http_request_t *r, void *data, ngx_int_t rc)
{
ngx_str_t *value = data;
if (r->upstream) {
value->len = r->upstream->buffer.last - r->upstream->buffer.pos;
value->data = r->upstream->buffer.pos;
if (r->headers_out.status < NGX_HTTP_SPECIAL_RESPONSE
&& r->out && r->out->buf)
{
value->len = r->out->buf->last - r->out->buf->pos;
value->data = r->out->buf->pos;
}
return rc;

21
src/http/ngx_http_core_module.c
View File

@ -399,6 +399,13 @@ static ngx_command_t ngx_http_core_commands[] = {
offsetof(ngx_http_core_loc_conf_t, sendfile_max_chunk),
NULL },
{ ngx_string("subrequest_output_buffer_size"),
NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1,
ngx_conf_set_size_slot,
NGX_HTTP_LOC_CONF_OFFSET,
offsetof(ngx_http_core_loc_conf_t, subrequest_output_buffer_size),
NULL },
{ ngx_string("aio"),
NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1,
ngx_http_core_set_aio,
@ -2237,6 +2244,12 @@ ngx_http_subrequest(ngx_http_request_t *r,
return NGX_ERROR;
}
if (r->subrequest_in_memory) {
ngx_log_error(NGX_LOG_ERR, r->connection->log, 0,
"nested in-memory subrequest \"%V\"", uri);
return NGX_ERROR;
}
sr = ngx_pcalloc(r->pool, sizeof(ngx_http_request_t));
if (sr == NULL) {
return NGX_ERROR;
@ -2318,6 +2331,10 @@ ngx_http_subrequest(ngx_http_request_t *r,
sr->log_handler = r->log_handler;
if (sr->subrequest_in_memory) {
sr->filter_need_in_memory = 1;
}
if (!sr->background) {
if (c->data == r && r->postponed == NULL) {
c->data = sr;
@ -3356,6 +3373,7 @@ ngx_http_core_create_loc_conf(ngx_conf_t *cf)
clcf->internal = NGX_CONF_UNSET;
clcf->sendfile = NGX_CONF_UNSET;
clcf->sendfile_max_chunk = NGX_CONF_UNSET_SIZE;
clcf->subrequest_output_buffer_size = NGX_CONF_UNSET_SIZE;
clcf->aio = NGX_CONF_UNSET;
clcf->aio_write = NGX_CONF_UNSET;
#if (NGX_THREADS)
@ -3578,6 +3596,9 @@ ngx_http_core_merge_loc_conf(ngx_conf_t *cf, void *parent, void *child)
ngx_conf_merge_value(conf->sendfile, prev->sendfile, 0);
ngx_conf_merge_size_value(conf->sendfile_max_chunk,
prev->sendfile_max_chunk, 0);
ngx_conf_merge_size_value(conf->subrequest_output_buffer_size,
prev->subrequest_output_buffer_size,
(size_t) ngx_pagesize);
ngx_conf_merge_value(conf->aio, prev->aio, NGX_HTTP_AIO_OFF);
ngx_conf_merge_value(conf->aio_write, prev->aio_write, 0);
#if (NGX_THREADS)

2
src/http/ngx_http_core_module.h
View File

@ -351,6 +351,8 @@ struct ngx_http_core_loc_conf_s {
size_t limit_rate_after; /* limit_rate_after */
size_t sendfile_max_chunk; /* sendfile_max_chunk */
size_t read_ahead; /* read_ahead */
size_t subrequest_output_buffer_size;
/* subrequest_output_buffer_size */
ngx_msec_t client_body_timeout; /* client_body_timeout */
ngx_msec_t send_timeout; /* send_timeout */

78
src/http/ngx_http_postpone_filter_module.c
View File

@ -12,6 +12,8 @@
static ngx_int_t ngx_http_postpone_filter_add(ngx_http_request_t *r,
ngx_chain_t *in);
static ngx_int_t ngx_http_postpone_filter_in_memory(ngx_http_request_t *r,
ngx_chain_t *in);
static ngx_int_t ngx_http_postpone_filter_init(ngx_conf_t *cf);
@ -60,6 +62,10 @@ ngx_http_postpone_filter(ngx_http_request_t *r, ngx_chain_t *in)
ngx_log_debug3(NGX_LOG_DEBUG_HTTP, c->log, 0,
"http postpone filter \"%V?%V\" %p", &r->uri, &r->args, in);
if (r->subrequest_in_memory) {
return ngx_http_postpone_filter_in_memory(r, in);
}
if (r != c->data) {
if (in) {
@ -171,6 +177,78 @@ found:
}
static ngx_int_t
ngx_http_postpone_filter_in_memory(ngx_http_request_t *r, ngx_chain_t *in)
{
size_t len;
ngx_buf_t *b;
ngx_connection_t *c;
ngx_http_core_loc_conf_t *clcf;
c = r->connection;
ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->log, 0,
"http postpone filter in memory");
if (r->out == NULL) {
r->out = ngx_alloc_chain_link(r->pool);
if (r->out == NULL) {
return NGX_ERROR;
}
clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module);
if (r->headers_out.content_length_n != -1) {
len = r->headers_out.content_length_n;
if (len > clcf->subrequest_output_buffer_size) {
ngx_log_error(NGX_LOG_ERR, c->log, 0,
"too big subrequest response: %uz", len);
return NGX_ERROR;
}
} else {
len = clcf->subrequest_output_buffer_size;
}
b = ngx_create_temp_buf(r->pool, len);
if (b == NULL) {
return NGX_ERROR;
}
b->last_buf = 1;
r->out->buf = b;
r->out->next = NULL;
}
b = r->out->buf;
for ( /* void */ ; in; in = in->next) {
if (ngx_buf_special(in->buf)) {
continue;
}
len = in->buf->last - in->buf->pos;
if (len > (size_t) (b->end - b->last)) {
ngx_log_error(NGX_LOG_ERR, c->log, 0,
"too big subrequest response");
return NGX_ERROR;
}
ngx_log_debug1(NGX_LOG_DEBUG_HTTP, c->log, 0,
"http postpone filter in memory %uz bytes", len);
b->last = ngx_cpymem(b->last, in->buf->pos, len);
in->buf->pos = in->buf->last;
}
return NGX_OK;
}
static ngx_int_t
ngx_http_postpone_filter_init(ngx_conf_t *cf)
{

126
src/http/ngx_http_upstream.c
View File

@ -55,8 +55,6 @@ static ngx_int_t ngx_http_upstream_intercept_errors(ngx_http_request_t *r,
static ngx_int_t ngx_http_upstream_test_connect(ngx_connection_t *c);
static ngx_int_t ngx_http_upstream_process_headers(ngx_http_request_t *r,
ngx_http_upstream_t *u);
static void ngx_http_upstream_process_body_in_memory(ngx_http_request_t *r,
ngx_http_upstream_t *u);
static void ngx_http_upstream_send_response(ngx_http_request_t *r,
ngx_http_upstream_t *u);
static void ngx_http_upstream_upgrade(ngx_http_request_t *r,
@ -2335,45 +2333,7 @@ ngx_http_upstream_process_header(ngx_http_request_t *r, ngx_http_upstream_t *u)
return;
}
if (!r->subrequest_in_memory) {
ngx_http_upstream_send_response(r, u);
return;
}
/* subrequest content in memory */
if (u->input_filter == NULL) {
u->input_filter_init = ngx_http_upstream_non_buffered_filter_init;
u->input_filter = ngx_http_upstream_non_buffered_filter;
u->input_filter_ctx = r;
}
if (u->input_filter_init(u->input_filter_ctx) == NGX_ERROR) {
ngx_http_upstream_finalize_request(r, u, NGX_ERROR);
return;
}
n = u->buffer.last - u->buffer.pos;
if (n) {
u->buffer.last = u->buffer.pos;
u->state->response_length += n;
if (u->input_filter(u->input_filter_ctx, n) == NGX_ERROR) {
ngx_http_upstream_finalize_request(r, u, NGX_ERROR);
return;
}
}
if (u->length == 0) {
ngx_http_upstream_finalize_request(r, u, 0);
return;
}
u->read_event_handler = ngx_http_upstream_process_body_in_memory;
ngx_http_upstream_process_body_in_memory(r, u);
ngx_http_upstream_send_response(r, u);
}
@ -2775,84 +2735,6 @@ ngx_http_upstream_process_headers(ngx_http_request_t *r, ngx_http_upstream_t *u)
}
static void
ngx_http_upstream_process_body_in_memory(ngx_http_request_t *r,
ngx_http_upstream_t *u)
{
size_t size;
ssize_t n;
ngx_buf_t *b;
ngx_event_t *rev;
ngx_connection_t *c;
c = u->peer.connection;
rev = c->read;
ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->log, 0,
"http upstream process body in memory");
if (rev->timedout) {
ngx_connection_error(c, NGX_ETIMEDOUT, "upstream timed out");
ngx_http_upstream_finalize_request(r, u, NGX_HTTP_GATEWAY_TIME_OUT);
return;
}
b = &u->buffer;
for ( ;; ) {
size = b->end - b->last;
if (size == 0) {
ngx_log_error(NGX_LOG_ALERT, c->log, 0,
"upstream buffer is too small to read response");
ngx_http_upstream_finalize_request(r, u, NGX_ERROR);
return;
}
n = c->recv(c, b->last, size);
if (n == NGX_AGAIN) {
break;
}
if (n == 0 || n == NGX_ERROR) {
ngx_http_upstream_finalize_request(r, u, n);
return;
}
u->state->bytes_received += n;
u->state->response_length += n;
if (u->input_filter(u->input_filter_ctx, n) == NGX_ERROR) {
ngx_http_upstream_finalize_request(r, u, NGX_ERROR);
return;
}
if (!rev->ready) {
break;
}
}
if (u->length == 0) {
ngx_http_upstream_finalize_request(r, u, 0);
return;
}
if (ngx_handle_read_event(rev, 0) != NGX_OK) {
ngx_http_upstream_finalize_request(r, u, NGX_ERROR);
return;
}
if (rev->active) {
ngx_add_timer(rev, u->conf->read_timeout);
} else if (rev->timer_set) {
ngx_del_timer(rev);
}
}
static void
ngx_http_upstream_send_response(ngx_http_request_t *r, ngx_http_upstream_t *u)
{
@ -4359,12 +4241,6 @@ ngx_http_upstream_finalize_request(ngx_http_request_t *r,
#endif
if (r->subrequest_in_memory
&& u->headers_in.status_n >= NGX_HTTP_SPECIAL_RESPONSE)
{
u->buffer.last = u->buffer.pos;
}
r->read_event_handler = ngx_http_block_reading;
if (rc == NGX_DECLINED) {