IntenseWebs/nginx
3
0
Fork 0
mirror of https://github.com/nginx/nginx.git synced 2024-12-22 23:23:32 -06:00
Code Issues Packages Projects Releases Wiki Activity

Slice filter.

Browse Source 
Splits a request into subrequests, each providing a specific range of response.
The variable "$slice_range" must be used to set subrequest range and proper
cache key.  The directive "slice" sets slice size.

The following example splits requests into 1-megabyte cacheable subrequests.

server {
    listen 8000;

    location / {
        slice 1m;

        proxy_cache cache;
        proxy_cache_key $uri$is_args$args$slice_range;
        proxy_set_header Range $slice_range;
        proxy_cache_valid 200 206 1h;
        proxy_pass http://127.0.0.1:9000;
    }
}
This commit is contained in:
Roman Arutyunyan 2015-12-07 16:30:48 +03:00
parent 1b478c50de
commit 8ba626ccd7
6 changed files with 567 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
auto/modules
View File

@ -73,6 +73,11 @@ if [ $HTTP_SSI = YES ]; then
fi
if [ $HTTP_SLICE = YES ]; then
HTTP_POSTPONE=YES
fi
if [ $HTTP_ADDITION = YES ]; then
HTTP_POSTPONE=YES
fi
@ -110,6 +115,7 @@ fi
# ngx_http_copy_filter
# ngx_http_range_body_filter
# ngx_http_not_modified_filter
# ngx_http_slice_filter
HTTP_FILTER_MODULES="$HTTP_WRITE_FILTER_MODULE \
$HTTP_HEADER_FILTER_MODULE \
@ -179,6 +185,12 @@ if [ $HTTP_USERID = YES ]; then
HTTP_SRCS="$HTTP_SRCS $HTTP_USERID_SRCS"
fi
if [ $HTTP_SLICE = YES ]; then
HTTP_SRCS="$HTTP_SRCS $HTTP_SLICE_SRCS"
else
HTTP_SLICE_FILTER_MODULE=""
fi
if [ $HTTP_V2 = YES ]; then
have=NGX_HTTP_V2 . auto/have
@ -461,7 +473,8 @@ if [ $HTTP = YES ]; then
$HTTP_AUX_FILTER_MODULES \
$HTTP_COPY_FILTER_MODULE \
$HTTP_RANGE_BODY_FILTER_MODULE \
$HTTP_NOT_MODIFIED_FILTER_MODULE"
$HTTP_NOT_MODIFIED_FILTER_MODULE \
$HTTP_SLICE_FILTER_MODULE"
NGX_ADDON_DEPS="$NGX_ADDON_DEPS \$(HTTP_DEPS)"
fi

3
auto/options
View File

@ -71,6 +71,7 @@ HTTP_ACCESS=YES
HTTP_AUTH_BASIC=YES
HTTP_AUTH_REQUEST=NO
HTTP_USERID=YES
HTTP_SLICE=NO
HTTP_AUTOINDEX=YES
HTTP_RANDOM_INDEX=NO
HTTP_STATUS=NO
@ -226,6 +227,7 @@ do
--with-http_random_index_module) HTTP_RANDOM_INDEX=YES ;;
--with-http_secure_link_module) HTTP_SECURE_LINK=YES ;;
--with-http_degradation_module) HTTP_DEGRADATION=YES ;;
--with-http_slice_module) HTTP_SLICE=YES ;;
--without-http_charset_module) HTTP_CHARSET=NO ;;
--without-http_gzip_module) HTTP_GZIP=NO ;;
@ -394,6 +396,7 @@ cat << END
--with-http_random_index_module enable ngx_http_random_index_module
--with-http_secure_link_module enable ngx_http_secure_link_module
--with-http_degradation_module enable ngx_http_degradation_module
--with-http_slice_module enable ngx_http_slice_module
--with-http_stub_status_module enable ngx_http_stub_status_module
--without-http_charset_module disable ngx_http_charset_module

4
auto/sources
View File

@ -360,6 +360,10 @@ HTTP_USERID_FILTER_MODULE=ngx_http_userid_filter_module
HTTP_USERID_SRCS=src/http/modules/ngx_http_userid_filter_module.c
HTTP_SLICE_FILTER_MODULE=ngx_http_slice_filter_module
HTTP_SLICE_SRCS=src/http/modules/ngx_http_slice_filter_module.c
HTTP_REALIP_MODULE=ngx_http_realip_module
HTTP_REALIP_SRCS=src/http/modules/ngx_http_realip_module.c

28
src/http/modules/ngx_http_range_filter_module.c
View File

@ -154,7 +154,7 @@ ngx_http_range_header_filter(ngx_http_request_t *r)
if (r->http_version < NGX_HTTP_VERSION_10
|| r->headers_out.status != NGX_HTTP_OK
|| r != r->main
|| (r != r->main && !r->subrequest_ranges)
|| r->headers_out.content_length_n == -1
|| !r->allow_ranges)
{
@ -222,6 +222,8 @@ parse:
return NGX_ERROR;
}
ctx->offset = r->headers_out.content_offset;
if (ngx_array_init(&ctx->ranges, r->pool, 1, sizeof(ngx_http_range_t))
!= NGX_OK)
{
@ -273,10 +275,21 @@ static ngx_int_t
ngx_http_range_parse(ngx_http_request_t *r, ngx_http_range_filter_ctx_t *ctx,
ngx_uint_t ranges)
{
u_char *p;
off_t start, end, size, content_length, cutoff, cutlim;
ngx_uint_t suffix;
ngx_http_range_t *range;
u_char *p;
off_t start, end, size, content_length, cutoff,
cutlim;
ngx_uint_t suffix;
ngx_http_range_t *range;
ngx_http_range_filter_ctx_t *mctx;
if (r != r->main) {
mctx = ngx_http_get_module_ctx(r->main,
ngx_http_range_body_filter_module);
if (mctx) {
ctx->ranges = mctx->ranges;
return NGX_OK;
}
}
p = r->headers_in.range->value.data + 6;
size = 0;
@ -395,6 +408,10 @@ ngx_http_range_singlepart_header(ngx_http_request_t *r,
ngx_table_elt_t *content_range;
ngx_http_range_t *range;
if (r != r->main) {
return ngx_http_next_header_filter(r);
}
content_range = ngx_list_push(&r->headers_out.headers);
if (content_range == NULL) {
return NGX_ERROR;
@ -422,6 +439,7 @@ ngx_http_range_singlepart_header(ngx_http_request_t *r,
- content_range->value.data;
r->headers_out.content_length_n = range->end - range->start;
r->headers_out.content_offset = range->start;
if (r->headers_out.content_length) {
r->headers_out.content_length->hash = 0;

521
src/http/modules/ngx_http_slice_filter_module.c Normal file
View File

@ -0,0 +1,521 @@
/*
* Copyright (C) Roman Arutyunyan
* Copyright (C) Nginx, Inc.
*/
#include <ngx_config.h>
#include <ngx_core.h>
#include <ngx_http.h>
typedef struct {
size_t size;
} ngx_http_slice_loc_conf_t;
typedef struct {
off_t start;
off_t end;
ngx_str_t range;
ngx_str_t etag;
ngx_uint_t last; /* unsigned last:1; */
} ngx_http_slice_ctx_t;
typedef struct {
off_t start;
off_t end;
off_t complete_length;
} ngx_http_slice_content_range_t;
static ngx_int_t ngx_http_slice_header_filter(ngx_http_request_t *r);
static ngx_int_t ngx_http_slice_body_filter(ngx_http_request_t *r,
ngx_chain_t *in);
static ngx_int_t ngx_http_slice_parse_content_range(ngx_http_request_t *r,
ngx_http_slice_content_range_t *cr);
static ngx_int_t ngx_http_slice_range_variable(ngx_http_request_t *r,
ngx_http_variable_value_t *v, uintptr_t data);
static off_t ngx_http_slice_get_start(ngx_http_request_t *r);
static void *ngx_http_slice_create_loc_conf(ngx_conf_t *cf);
static char *ngx_http_slice_merge_loc_conf(ngx_conf_t *cf, void *parent,
void *child);
static ngx_int_t ngx_http_slice_add_variables(ngx_conf_t *cf);
static ngx_int_t ngx_http_slice_init(ngx_conf_t *cf);
static ngx_command_t ngx_http_slice_filter_commands[] = {
{ ngx_string("slice"),
NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1,
ngx_conf_set_size_slot,
NGX_HTTP_LOC_CONF_OFFSET,
offsetof(ngx_http_slice_loc_conf_t, size),
NULL },
ngx_null_command
};
static ngx_http_module_t ngx_http_slice_filter_module_ctx = {
ngx_http_slice_add_variables, /* preconfiguration */
ngx_http_slice_init, /* postconfiguration */
NULL, /* create main configuration */
NULL, /* init main configuration */
NULL, /* create server configuration */
NULL, /* merge server configuration */
ngx_http_slice_create_loc_conf, /* create location configuration */
ngx_http_slice_merge_loc_conf /* merge location configuration */
};
ngx_module_t ngx_http_slice_filter_module = {
NGX_MODULE_V1,
&ngx_http_slice_filter_module_ctx, /* module context */
ngx_http_slice_filter_commands, /* module directives */
NGX_HTTP_MODULE, /* module type */
NULL, /* init master */
NULL, /* init module */
NULL, /* init process */
NULL, /* init thread */
NULL, /* exit thread */
NULL, /* exit process */
NULL, /* exit master */
NGX_MODULE_V1_PADDING
};
static ngx_str_t ngx_http_slice_range_name = ngx_string("slice_range");
static ngx_http_output_header_filter_pt ngx_http_next_header_filter;
static ngx_http_output_body_filter_pt ngx_http_next_body_filter;
static ngx_int_t
ngx_http_slice_header_filter(ngx_http_request_t *r)
{
off_t end;
ngx_int_t rc;
ngx_table_elt_t *h;
ngx_http_slice_ctx_t *ctx;
ngx_http_slice_loc_conf_t *slcf;
ngx_http_slice_content_range_t cr;
ctx = ngx_http_get_module_ctx(r, ngx_http_slice_filter_module);
if (ctx == NULL) {
return ngx_http_next_header_filter(r);
}
if (r->headers_out.status != NGX_HTTP_PARTIAL_CONTENT) {
if (r == r->main) {
ngx_http_set_ctx(r, NULL, ngx_http_slice_filter_module);
return ngx_http_next_header_filter(r);
}
ngx_log_error(NGX_LOG_ERR, r->connection->log, 0,
"unexpected status code %ui in slice response",
r->headers_out.status);
return NGX_ERROR;
}
h = r->headers_out.etag;
if (ctx->etag.len) {
if (h == NULL
|| h->value.len != ctx->etag.len
|| ngx_strncmp(h->value.data, ctx->etag.data, ctx->etag.len)
!= 0)
{
ngx_log_error(NGX_LOG_ERR, r->connection->log, 0,
"etag mismatch in slice response");
return NGX_ERROR;
}
}
if (h) {
ctx->etag = h->value;
}
if (ngx_http_slice_parse_content_range(r, &cr) != NGX_OK) {
ngx_log_error(NGX_LOG_ERR, r->connection->log, 0,
"invalid range in slice response");
return NGX_ERROR;
}
if (cr.complete_length == -1) {
ngx_log_error(NGX_LOG_ERR, r->connection->log, 0,
"no complete length in slice response");
return NGX_ERROR;
}
ngx_log_debug3(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
"http slice response range: %O-%O/%O",
cr.start, cr.end, cr.complete_length);
slcf = ngx_http_get_module_loc_conf(r, ngx_http_slice_filter_module);
end = ngx_min(cr.start + (off_t) slcf->size, cr.complete_length);
if (cr.start != ctx->start || cr.end != end) {
ngx_log_error(NGX_LOG_ERR, r->connection->log, 0,
"unexpected range in slice response: %O-%O",
cr.start, cr.end);
return NGX_ERROR;
}
ctx->start = end;
r->headers_out.status = NGX_HTTP_OK;
r->headers_out.status_line.len = 0;
r->headers_out.content_length_n = cr.complete_length;
r->headers_out.content_offset = cr.start;
r->headers_out.content_range->hash = 0;
r->headers_out.content_range = NULL;
r->allow_ranges = 1;
r->subrequest_ranges = 1;
r->single_range = 1;
rc = ngx_http_next_header_filter(r);
if (r != r->main) {
return rc;
}
if (r->headers_out.status == NGX_HTTP_PARTIAL_CONTENT) {
if (ctx->start + (off_t) slcf->size <= r->headers_out.content_offset) {
ctx->start = slcf->size
* (r->headers_out.content_offset / slcf->size);
}
ctx->end = r->headers_out.content_offset
+ r->headers_out.content_length_n;
} else {
ctx->end = cr.complete_length;
}
return rc;
}
static ngx_int_t
ngx_http_slice_body_filter(ngx_http_request_t *r, ngx_chain_t *in)
{
ngx_int_t rc;
ngx_chain_t *cl;
ngx_http_request_t *sr;
ngx_http_slice_ctx_t *ctx;
ngx_http_slice_loc_conf_t *slcf;
ctx = ngx_http_get_module_ctx(r, ngx_http_slice_filter_module);
if (ctx == NULL || r != r->main) {
return ngx_http_next_body_filter(r, in);
}
for (cl = in; cl; cl = cl->next) {
if (cl->buf->last_buf) {
cl->buf->last_buf = 0;
cl->buf->sync = 1;
ctx->last = 1;
}
}
rc = ngx_http_next_body_filter(r, in);
if (rc == NGX_ERROR || !ctx->last) {
return rc;
}
if (ctx->start >= ctx->end) {
ngx_http_set_ctx(r, NULL, ngx_http_slice_filter_module);
ngx_http_send_special(r, NGX_HTTP_LAST);
return rc;
}
if (ngx_http_subrequest(r, &r->uri, &r->args, &sr, NULL, 0) != NGX_OK) {
return NGX_ERROR;
}
ngx_http_set_ctx(sr, ctx, ngx_http_slice_filter_module);
slcf = ngx_http_get_module_loc_conf(r, ngx_http_slice_filter_module);
ctx->range.len = ngx_sprintf(ctx->range.data, "bytes=%O-%O", ctx->start,
ctx->start + (off_t) slcf->size - 1)
- ctx->range.data;
ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
"http slice subrequest: \"%V\"", &ctx->range);
return rc;
}
static ngx_int_t
ngx_http_slice_parse_content_range(ngx_http_request_t *r,
ngx_http_slice_content_range_t *cr)
{
off_t start, end, complete_length, cutoff, cutlim;
u_char *p;
ngx_table_elt_t *h;
h = r->headers_out.content_range;
if (h == NULL
|| h->value.len < 7
|| ngx_strncmp(h->value.data, "bytes ", 6) != 0)
{
return NGX_ERROR;
}
p = h->value.data + 6;
cutoff = NGX_MAX_OFF_T_VALUE / 10;
cutlim = NGX_MAX_OFF_T_VALUE % 10;
start = 0;
end = 0;
complete_length = 0;
while (*p == ' ') { p++; }
if (*p < '0' || *p > '9') {
return NGX_ERROR;
}
while (*p >= '0' && *p <= '9') {
if (start >= cutoff && (start > cutoff || *p - '0' > cutlim)) {
return NGX_ERROR;
}
start = start * 10 + *p++ - '0';
}
while (*p == ' ') { p++; }
if (*p++ != '-') {
return NGX_ERROR;
}
while (*p == ' ') { p++; }
if (*p < '0' || *p > '9') {
return NGX_ERROR;
}
while (*p >= '0' && *p <= '9') {
if (end >= cutoff && (end > cutoff || *p - '0' > cutlim)) {
return NGX_ERROR;
}
end = end * 10 + *p++ - '0';
}
end++;
while (*p == ' ') { p++; }
if (*p++ != '/') {
return NGX_ERROR;
}
while (*p == ' ') { p++; }
if (*p != '*') {
if (*p < '0' || *p > '9') {
return NGX_ERROR;
}
while (*p >= '0' && *p <= '9') {
if (complete_length >= cutoff
&& (complete_length > cutoff || *p - '0' > cutlim))
{
return NGX_ERROR;
}
complete_length = complete_length * 10 + *p++ - '0';
}
} else {
complete_length = -1;
p++;
}
while (*p == ' ') { p++; }
if (*p != '\0') {
return NGX_ERROR;
}
cr->start = start;
cr->end = end;
cr->complete_length = complete_length;
return NGX_OK;
}
static ngx_int_t
ngx_http_slice_range_variable(ngx_http_request_t *r,
ngx_http_variable_value_t *v, uintptr_t data)
{
u_char *p;
ngx_http_slice_ctx_t *ctx;
ngx_http_slice_loc_conf_t *slcf;
ctx = ngx_http_get_module_ctx(r, ngx_http_slice_filter_module);
if (ctx == NULL) {
if (r != r->main || r->headers_out.status) {
v->not_found = 1;
return NGX_OK;
}
slcf = ngx_http_get_module_loc_conf(r, ngx_http_slice_filter_module);
if (slcf->size == 0) {
v->not_found = 1;
return NGX_OK;
}
ctx = ngx_pcalloc(r->pool, sizeof(ngx_http_slice_ctx_t));
if (ctx == NULL) {
return NGX_ERROR;
}
ngx_http_set_ctx(r, ctx, ngx_http_slice_filter_module);
p = ngx_pnalloc(r->pool, sizeof("bytes=-") - 1 + 2 * NGX_OFF_T_LEN);
if (p == NULL) {
return NGX_ERROR;
}
ctx->start = slcf->size * (ngx_http_slice_get_start(r) / slcf->size);
ctx->range.data = p;
ctx->range.len = ngx_sprintf(p, "bytes=%O-%O", ctx->start,
ctx->start + (off_t) slcf->size - 1)
- p;
}
v->data = ctx->range.data;
v->valid = 1;
v->not_found = 0;
v->no_cacheable = 1;
v->len = ctx->range.len;
return NGX_OK;
}
static off_t
ngx_http_slice_get_start(ngx_http_request_t *r)
{
off_t start, cutoff, cutlim;
u_char *p;
ngx_table_elt_t *h;
if (r->headers_in.if_range) {
return 0;
}
h = r->headers_in.range;
if (h == NULL
|| h->value.len < 7
|| ngx_strncasecmp(h->value.data, (u_char *) "bytes=", 6) != 0)
{
return 0;
}
p = h->value.data + 6;
if (ngx_strchr(p, ',')) {
return 0;
}
while (*p == ' ') { p++; }
if (*p == '-') {
return 0;
}
cutoff = NGX_MAX_OFF_T_VALUE / 10;
cutlim = NGX_MAX_OFF_T_VALUE % 10;
start = 0;
while (*p >= '0' && *p <= '9') {
if (start >= cutoff && (start > cutoff || *p - '0' > cutlim)) {
return 0;
}
start = start * 10 + *p++ - '0';
}
return start;
}
static void *
ngx_http_slice_create_loc_conf(ngx_conf_t *cf)
{
ngx_http_slice_loc_conf_t *slcf;
slcf = ngx_palloc(cf->pool, sizeof(ngx_http_slice_loc_conf_t));
if (slcf == NULL) {
return NULL;
}
slcf->size = NGX_CONF_UNSET_SIZE;
return slcf;
}
static char *
ngx_http_slice_merge_loc_conf(ngx_conf_t *cf, void *parent, void *child)
{
ngx_http_slice_loc_conf_t *prev = parent;
ngx_http_slice_loc_conf_t *conf = child;
ngx_conf_merge_size_value(conf->size, prev->size, 0);
return NGX_CONF_OK;
}
static ngx_int_t
ngx_http_slice_add_variables(ngx_conf_t *cf)
{
ngx_http_variable_t *var;
var = ngx_http_add_variable(cf, &ngx_http_slice_range_name, 0);
if (var == NULL) {
return NGX_ERROR;
}
var->get_handler = ngx_http_slice_range_variable;
return NGX_OK;
}
static ngx_int_t
ngx_http_slice_init(ngx_conf_t *cf)
{
ngx_http_next_header_filter = ngx_http_top_header_filter;
ngx_http_top_header_filter = ngx_http_slice_header_filter;
ngx_http_next_body_filter = ngx_http_top_body_filter;
ngx_http_top_body_filter = ngx_http_slice_body_filter;
return NGX_OK;
}

2
src/http/ngx_http_request.h
View File

@ -271,6 +271,7 @@ typedef struct {
ngx_array_t cache_control;
off_t content_length_n;
off_t content_offset;
time_t date_time;
time_t last_modified_time;
} ngx_http_headers_out_t;
@ -530,6 +531,7 @@ struct ngx_http_request_s {
unsigned filter_need_in_memory:1;
unsigned filter_need_temporary:1;
unsigned allow_ranges:1;
unsigned subrequest_ranges:1;
unsigned single_range:1;
unsigned disable_not_modified:1;