Slice filter.

Splits a request into subrequests, each providing a specific range of response.
The variable "$slice_range" must be used to set subrequest range and proper
cache key.  The directive "slice" sets slice size.

The following example splits requests into 1-megabyte cacheable subrequests.

server {
    listen 8000;

    location / {
        slice 1m;

        proxy_cache cache;
        proxy_cache_key $uri$is_args$args$slice_range;
        proxy_set_header Range $slice_range;
        proxy_cache_valid 200 206 1h;
        proxy_pass http://127.0.0.1:9000;
    }
}

auto/modules

@@ -73,6 +73,11 @@ if [ $HTTP_SSI = YES ]; then
 fi
 
 
+if [ $HTTP_SLICE = YES ]; then
+    HTTP_POSTPONE=YES
+fi
+
+
 if [ $HTTP_ADDITION = YES ]; then
     HTTP_POSTPONE=YES
 fi
@@ -110,6 +115,7 @@ fi
 #     ngx_http_copy_filter
 #     ngx_http_range_body_filter
 #     ngx_http_not_modified_filter
+#     ngx_http_slice_filter
 
 HTTP_FILTER_MODULES="$HTTP_WRITE_FILTER_MODULE \
                      $HTTP_HEADER_FILTER_MODULE \
@@ -179,6 +185,12 @@ if [ $HTTP_USERID = YES ]; then
     HTTP_SRCS="$HTTP_SRCS $HTTP_USERID_SRCS"
 fi
 
+if [ $HTTP_SLICE = YES ]; then
+    HTTP_SRCS="$HTTP_SRCS $HTTP_SLICE_SRCS"
+else
+    HTTP_SLICE_FILTER_MODULE=""
+fi
+
 
 if [ $HTTP_V2 = YES ]; then
     have=NGX_HTTP_V2 . auto/have
@@ -461,7 +473,8 @@ if [ $HTTP = YES ]; then
              $HTTP_AUX_FILTER_MODULES \
              $HTTP_COPY_FILTER_MODULE \
              $HTTP_RANGE_BODY_FILTER_MODULE \
-             $HTTP_NOT_MODIFIED_FILTER_MODULE"
+             $HTTP_NOT_MODIFIED_FILTER_MODULE \
+             $HTTP_SLICE_FILTER_MODULE"
 
     NGX_ADDON_DEPS="$NGX_ADDON_DEPS \$(HTTP_DEPS)"
 fi

auto/options

@@ -71,6 +71,7 @@ HTTP_ACCESS=YES
 HTTP_AUTH_BASIC=YES
 HTTP_AUTH_REQUEST=NO
 HTTP_USERID=YES
+HTTP_SLICE=NO
 HTTP_AUTOINDEX=YES
 HTTP_RANDOM_INDEX=NO
 HTTP_STATUS=NO
@@ -226,6 +227,7 @@ do
         --with-http_random_index_module) HTTP_RANDOM_INDEX=YES      ;;
         --with-http_secure_link_module)  HTTP_SECURE_LINK=YES       ;;
         --with-http_degradation_module)  HTTP_DEGRADATION=YES       ;;
+        --with-http_slice_module)        HTTP_SLICE=YES             ;;
 
         --without-http_charset_module)   HTTP_CHARSET=NO            ;;
         --without-http_gzip_module)      HTTP_GZIP=NO               ;;
@@ -394,6 +396,7 @@ cat << END
   --with-http_random_index_module    enable ngx_http_random_index_module
   --with-http_secure_link_module     enable ngx_http_secure_link_module
   --with-http_degradation_module     enable ngx_http_degradation_module
+  --with-http_slice_module           enable ngx_http_slice_module
   --with-http_stub_status_module     enable ngx_http_stub_status_module
 
   --without-http_charset_module      disable ngx_http_charset_module

auto/sources

@@ -360,6 +360,10 @@ HTTP_USERID_FILTER_MODULE=ngx_http_userid_filter_module
 HTTP_USERID_SRCS=src/http/modules/ngx_http_userid_filter_module.c
 
 
+HTTP_SLICE_FILTER_MODULE=ngx_http_slice_filter_module
+HTTP_SLICE_SRCS=src/http/modules/ngx_http_slice_filter_module.c
+
+
 HTTP_REALIP_MODULE=ngx_http_realip_module
 HTTP_REALIP_SRCS=src/http/modules/ngx_http_realip_module.c
 

src/http/modules/ngx_http_range_filter_module.c

@@ -154,7 +154,7 @@ ngx_http_range_header_filter(ngx_http_request_t *r)
 
     if (r->http_version < NGX_HTTP_VERSION_10
         || r->headers_out.status != NGX_HTTP_OK
-        || r != r->main
+        || (r != r->main && !r->subrequest_ranges)
         || r->headers_out.content_length_n == -1
         || !r->allow_ranges)
     {
@@ -222,6 +222,8 @@ parse:
         return NGX_ERROR;
     }
 
+    ctx->offset = r->headers_out.content_offset;
+
     if (ngx_array_init(&ctx->ranges, r->pool, 1, sizeof(ngx_http_range_t))
         != NGX_OK)
     {
@@ -273,10 +275,21 @@ static ngx_int_t
 ngx_http_range_parse(ngx_http_request_t *r, ngx_http_range_filter_ctx_t *ctx,
     ngx_uint_t ranges)
 {
-    u_char            *p;
+    u_char                       *p;
-    off_t              start, end, size, content_length, cutoff, cutlim;
+    off_t                         start, end, size, content_length, cutoff,
-    ngx_uint_t         suffix;
+                                  cutlim;
-    ngx_http_range_t  *range;
+    ngx_uint_t                    suffix;
+    ngx_http_range_t             *range;
+    ngx_http_range_filter_ctx_t  *mctx;
+
+    if (r != r->main) {
+        mctx = ngx_http_get_module_ctx(r->main,
+                                       ngx_http_range_body_filter_module);
+        if (mctx) {
+            ctx->ranges = mctx->ranges;
+            return NGX_OK;
+        }
+    }
 
     p = r->headers_in.range->value.data + 6;
     size = 0;
@@ -395,6 +408,10 @@ ngx_http_range_singlepart_header(ngx_http_request_t *r,
     ngx_table_elt_t   *content_range;
     ngx_http_range_t  *range;
 
+    if (r != r->main) {
+        return ngx_http_next_header_filter(r);
+    }
+
     content_range = ngx_list_push(&r->headers_out.headers);
     if (content_range == NULL) {
         return NGX_ERROR;
@@ -422,6 +439,7 @@ ngx_http_range_singlepart_header(ngx_http_request_t *r,
                                - content_range->value.data;
 
     r->headers_out.content_length_n = range->end - range->start;
+    r->headers_out.content_offset = range->start;
 
     if (r->headers_out.content_length) {
         r->headers_out.content_length->hash = 0;

src/http/modules/ngx_http_slice_filter_module.c

@@ -0,0 +1,521 @@
+
+/*
+ * Copyright (C) Roman Arutyunyan
+ * Copyright (C) Nginx, Inc.
+ */
+
+
+#include <ngx_config.h>
+#include <ngx_core.h>
+#include <ngx_http.h>
+
+
+typedef struct {
+    size_t      size;
+} ngx_http_slice_loc_conf_t;
+
+
+typedef struct {
+    off_t       start;
+    off_t       end;
+    ngx_str_t   range;
+    ngx_str_t   etag;
+    ngx_uint_t  last;  /* unsigned  last:1; */
+} ngx_http_slice_ctx_t;
+
+
+typedef struct {
+    off_t       start;
+    off_t       end;
+    off_t       complete_length;
+} ngx_http_slice_content_range_t;
+
+
+static ngx_int_t ngx_http_slice_header_filter(ngx_http_request_t *r);
+static ngx_int_t ngx_http_slice_body_filter(ngx_http_request_t *r,
+    ngx_chain_t *in);
+static ngx_int_t ngx_http_slice_parse_content_range(ngx_http_request_t *r,
+    ngx_http_slice_content_range_t *cr);
+static ngx_int_t ngx_http_slice_range_variable(ngx_http_request_t *r,
+    ngx_http_variable_value_t *v, uintptr_t data);
+static off_t ngx_http_slice_get_start(ngx_http_request_t *r);
+static void *ngx_http_slice_create_loc_conf(ngx_conf_t *cf);
+static char *ngx_http_slice_merge_loc_conf(ngx_conf_t *cf, void *parent,
+    void *child);
+static ngx_int_t ngx_http_slice_add_variables(ngx_conf_t *cf);
+static ngx_int_t ngx_http_slice_init(ngx_conf_t *cf);
+
+
+static ngx_command_t  ngx_http_slice_filter_commands[] = {
+
+    { ngx_string("slice"),
+      NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1,
+      ngx_conf_set_size_slot,
+      NGX_HTTP_LOC_CONF_OFFSET,
+      offsetof(ngx_http_slice_loc_conf_t, size),
+      NULL },
+
+      ngx_null_command
+};
+
+
+static ngx_http_module_t  ngx_http_slice_filter_module_ctx = {
+    ngx_http_slice_add_variables,          /* preconfiguration */
+    ngx_http_slice_init,                   /* postconfiguration */
+
+    NULL,                                  /* create main configuration */
+    NULL,                                  /* init main configuration */
+
+    NULL,                                  /* create server configuration */
+    NULL,                                  /* merge server configuration */
+
+    ngx_http_slice_create_loc_conf,        /* create location configuration */
+    ngx_http_slice_merge_loc_conf          /* merge location configuration */
+};
+
+
+ngx_module_t  ngx_http_slice_filter_module = {
+    NGX_MODULE_V1,
+    &ngx_http_slice_filter_module_ctx,     /* module context */
+    ngx_http_slice_filter_commands,        /* module directives */
+    NGX_HTTP_MODULE,                       /* module type */
+    NULL,                                  /* init master */
+    NULL,                                  /* init module */
+    NULL,                                  /* init process */
+    NULL,                                  /* init thread */
+    NULL,                                  /* exit thread */
+    NULL,                                  /* exit process */
+    NULL,                                  /* exit master */
+    NGX_MODULE_V1_PADDING
+};
+
+
+static ngx_str_t  ngx_http_slice_range_name = ngx_string("slice_range");
+
+static ngx_http_output_header_filter_pt  ngx_http_next_header_filter;
+static ngx_http_output_body_filter_pt    ngx_http_next_body_filter;
+
+
+static ngx_int_t
+ngx_http_slice_header_filter(ngx_http_request_t *r)
+{
+    off_t                            end;
+    ngx_int_t                        rc;
+    ngx_table_elt_t                 *h;
+    ngx_http_slice_ctx_t            *ctx;
+    ngx_http_slice_loc_conf_t       *slcf;
+    ngx_http_slice_content_range_t   cr;
+
+    ctx = ngx_http_get_module_ctx(r, ngx_http_slice_filter_module);
+    if (ctx == NULL) {
+        return ngx_http_next_header_filter(r);
+    }
+
+    if (r->headers_out.status != NGX_HTTP_PARTIAL_CONTENT) {
+        if (r == r->main) {
+            ngx_http_set_ctx(r, NULL, ngx_http_slice_filter_module);
+            return ngx_http_next_header_filter(r);
+        }
+
+        ngx_log_error(NGX_LOG_ERR, r->connection->log, 0,
+                      "unexpected status code %ui in slice response",
+                      r->headers_out.status);
+        return NGX_ERROR;
+    }
+
+    h = r->headers_out.etag;
+
+    if (ctx->etag.len) {
+        if (h == NULL
+            || h->value.len != ctx->etag.len
+            || ngx_strncmp(h->value.data, ctx->etag.data, ctx->etag.len)
+               != 0)
+        {
+            ngx_log_error(NGX_LOG_ERR, r->connection->log, 0,
+                          "etag mismatch in slice response");
+            return NGX_ERROR;
+        }
+    }
+
+    if (h) {
+        ctx->etag = h->value;
+    }
+
+    if (ngx_http_slice_parse_content_range(r, &cr) != NGX_OK) {
+        ngx_log_error(NGX_LOG_ERR, r->connection->log, 0,
+                      "invalid range in slice response");
+        return NGX_ERROR;
+    }
+
+    if (cr.complete_length == -1) {
+        ngx_log_error(NGX_LOG_ERR, r->connection->log, 0,
+                      "no complete length in slice response");
+        return NGX_ERROR;
+    }
+
+    ngx_log_debug3(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
+                   "http slice response range: %O-%O/%O",
+                   cr.start, cr.end, cr.complete_length);
+
+    slcf = ngx_http_get_module_loc_conf(r, ngx_http_slice_filter_module);
+
+    end = ngx_min(cr.start + (off_t) slcf->size, cr.complete_length);
+
+    if (cr.start != ctx->start || cr.end != end) {
+        ngx_log_error(NGX_LOG_ERR, r->connection->log, 0,
+                      "unexpected range in slice response: %O-%O",
+                      cr.start, cr.end);
+        return NGX_ERROR;
+    }
+
+    ctx->start = end;
+
+    r->headers_out.status = NGX_HTTP_OK;
+    r->headers_out.status_line.len = 0;
+    r->headers_out.content_length_n = cr.complete_length;
+    r->headers_out.content_offset = cr.start;
+    r->headers_out.content_range->hash = 0;
+    r->headers_out.content_range = NULL;
+
+    r->allow_ranges = 1;
+    r->subrequest_ranges = 1;
+    r->single_range = 1;
+
+    rc = ngx_http_next_header_filter(r);
+
+    if (r != r->main) {
+        return rc;
+    }
+
+    if (r->headers_out.status == NGX_HTTP_PARTIAL_CONTENT) {
+        if (ctx->start + (off_t) slcf->size <= r->headers_out.content_offset) {
+            ctx->start = slcf->size
+                         * (r->headers_out.content_offset / slcf->size);
+        }
+
+        ctx->end = r->headers_out.content_offset
+                   + r->headers_out.content_length_n;
+
+    } else {
+        ctx->end = cr.complete_length;
+    }
+
+    return rc;
+}
+
+
+static ngx_int_t
+ngx_http_slice_body_filter(ngx_http_request_t *r, ngx_chain_t *in)
+{
+    ngx_int_t                   rc;
+    ngx_chain_t                *cl;
+    ngx_http_request_t         *sr;
+    ngx_http_slice_ctx_t       *ctx;
+    ngx_http_slice_loc_conf_t  *slcf;
+
+    ctx = ngx_http_get_module_ctx(r, ngx_http_slice_filter_module);
+
+    if (ctx == NULL || r != r->main) {
+        return ngx_http_next_body_filter(r, in);
+    }
+
+    for (cl = in; cl; cl = cl->next) {
+        if (cl->buf->last_buf) {
+            cl->buf->last_buf = 0;
+            cl->buf->sync = 1;
+            ctx->last = 1;
+        }
+    }
+
+    rc = ngx_http_next_body_filter(r, in);
+
+    if (rc == NGX_ERROR || !ctx->last) {
+        return rc;
+    }
+
+    if (ctx->start >= ctx->end) {
+        ngx_http_set_ctx(r, NULL, ngx_http_slice_filter_module);
+        ngx_http_send_special(r, NGX_HTTP_LAST);
+        return rc;
+    }
+
+    if (ngx_http_subrequest(r, &r->uri, &r->args, &sr, NULL, 0) != NGX_OK) {
+        return NGX_ERROR;
+    }
+
+    ngx_http_set_ctx(sr, ctx, ngx_http_slice_filter_module);
+
+    slcf = ngx_http_get_module_loc_conf(r, ngx_http_slice_filter_module);
+
+    ctx->range.len = ngx_sprintf(ctx->range.data, "bytes=%O-%O", ctx->start,
+                                 ctx->start + (off_t) slcf->size - 1)
+                     - ctx->range.data;
+
+    ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
+                   "http slice subrequest: \"%V\"", &ctx->range);
+
+    return rc;
+}
+
+
+static ngx_int_t
+ngx_http_slice_parse_content_range(ngx_http_request_t *r,
+    ngx_http_slice_content_range_t *cr)
+{
+    off_t             start, end, complete_length, cutoff, cutlim;
+    u_char           *p;
+    ngx_table_elt_t  *h;
+
+    h = r->headers_out.content_range;
+
+    if (h == NULL
+        || h->value.len < 7
+        || ngx_strncmp(h->value.data, "bytes ", 6) != 0)
+    {
+        return NGX_ERROR;
+    }
+
+    p = h->value.data + 6;
+
+    cutoff = NGX_MAX_OFF_T_VALUE / 10;
+    cutlim = NGX_MAX_OFF_T_VALUE % 10;
+
+    start = 0;
+    end = 0;
+    complete_length = 0;
+
+    while (*p == ' ') { p++; }
+
+    if (*p < '0' || *p > '9') {
+        return NGX_ERROR;
+    }
+
+    while (*p >= '0' && *p <= '9') {
+        if (start >= cutoff && (start > cutoff || *p - '0' > cutlim)) {
+            return NGX_ERROR;
+        }
+
+        start = start * 10 + *p++ - '0';
+    }
+
+    while (*p == ' ') { p++; }
+
+    if (*p++ != '-') {
+        return NGX_ERROR;
+    }
+
+    while (*p == ' ') { p++; }
+
+    if (*p < '0' || *p > '9') {
+        return NGX_ERROR;
+    }
+
+    while (*p >= '0' && *p <= '9') {
+        if (end >= cutoff && (end > cutoff || *p - '0' > cutlim)) {
+            return NGX_ERROR;
+        }
+
+        end = end * 10 + *p++ - '0';
+    }
+
+    end++;
+
+    while (*p == ' ') { p++; }
+
+    if (*p++ != '/') {
+        return NGX_ERROR;
+    }
+
+    while (*p == ' ') { p++; }
+
+    if (*p != '*') {
+        if (*p < '0' || *p > '9') {
+            return NGX_ERROR;
+        }
+
+        while (*p >= '0' && *p <= '9') {
+            if (complete_length >= cutoff
+                && (complete_length > cutoff || *p - '0' > cutlim))
+            {
+                return NGX_ERROR;
+            }
+
+            complete_length = complete_length * 10 + *p++ - '0';
+        }
+
+    } else {
+        complete_length = -1;
+        p++;
+    }
+
+    while (*p == ' ') { p++; }
+
+    if (*p != '\0') {
+        return NGX_ERROR;
+    }
+
+    cr->start = start;
+    cr->end = end;
+    cr->complete_length = complete_length;
+
+    return NGX_OK;
+}
+
+
+static ngx_int_t
+ngx_http_slice_range_variable(ngx_http_request_t *r,
+    ngx_http_variable_value_t *v, uintptr_t data)
+{
+    u_char                     *p;
+    ngx_http_slice_ctx_t       *ctx;
+    ngx_http_slice_loc_conf_t  *slcf;
+
+    ctx = ngx_http_get_module_ctx(r, ngx_http_slice_filter_module);
+
+    if (ctx == NULL) {
+        if (r != r->main || r->headers_out.status) {
+            v->not_found = 1;
+            return NGX_OK;
+        }
+
+        slcf = ngx_http_get_module_loc_conf(r, ngx_http_slice_filter_module);
+
+        if (slcf->size == 0) {
+            v->not_found = 1;
+            return NGX_OK;
+        }
+
+        ctx = ngx_pcalloc(r->pool, sizeof(ngx_http_slice_ctx_t));
+        if (ctx == NULL) {
+            return NGX_ERROR;
+        }
+
+        ngx_http_set_ctx(r, ctx, ngx_http_slice_filter_module);
+
+        p = ngx_pnalloc(r->pool, sizeof("bytes=-") - 1 + 2 * NGX_OFF_T_LEN);
+        if (p == NULL) {
+            return NGX_ERROR;
+        }
+
+        ctx->start = slcf->size * (ngx_http_slice_get_start(r) / slcf->size);
+
+        ctx->range.data = p;
+        ctx->range.len = ngx_sprintf(p, "bytes=%O-%O", ctx->start,
+                                     ctx->start + (off_t) slcf->size - 1)
+                         - p;
+    }
+
+    v->data = ctx->range.data;
+    v->valid = 1;
+    v->not_found = 0;
+    v->no_cacheable = 1;
+    v->len = ctx->range.len;
+
+    return NGX_OK;
+}
+
+
+static off_t
+ngx_http_slice_get_start(ngx_http_request_t *r)
+{
+    off_t             start, cutoff, cutlim;
+    u_char           *p;
+    ngx_table_elt_t  *h;
+
+    if (r->headers_in.if_range) {
+        return 0;
+    }
+
+    h = r->headers_in.range;
+
+    if (h == NULL
+        || h->value.len < 7
+        || ngx_strncasecmp(h->value.data, (u_char *) "bytes=", 6) != 0)
+    {
+        return 0;
+    }
+
+    p = h->value.data + 6;
+
+    if (ngx_strchr(p, ',')) {
+        return 0;
+    }
+
+    while (*p == ' ') { p++; }
+
+    if (*p == '-') {
+        return 0;
+    }
+
+    cutoff = NGX_MAX_OFF_T_VALUE / 10;
+    cutlim = NGX_MAX_OFF_T_VALUE % 10;
+
+    start = 0;
+
+    while (*p >= '0' && *p <= '9') {
+        if (start >= cutoff && (start > cutoff || *p - '0' > cutlim)) {
+            return 0;
+        }
+
+        start = start * 10 + *p++ - '0';
+    }
+
+    return start;
+}
+
+
+static void *
+ngx_http_slice_create_loc_conf(ngx_conf_t *cf)
+{
+    ngx_http_slice_loc_conf_t  *slcf;
+
+    slcf = ngx_palloc(cf->pool, sizeof(ngx_http_slice_loc_conf_t));
+    if (slcf == NULL) {
+        return NULL;
+    }
+
+    slcf->size = NGX_CONF_UNSET_SIZE;
+
+    return slcf;
+}
+
+
+static char *
+ngx_http_slice_merge_loc_conf(ngx_conf_t *cf, void *parent, void *child)
+{
+    ngx_http_slice_loc_conf_t *prev = parent;
+    ngx_http_slice_loc_conf_t *conf = child;
+
+    ngx_conf_merge_size_value(conf->size, prev->size, 0);
+
+    return NGX_CONF_OK;
+}
+
+
+static ngx_int_t
+ngx_http_slice_add_variables(ngx_conf_t *cf)
+{
+    ngx_http_variable_t  *var;
+
+    var = ngx_http_add_variable(cf, &ngx_http_slice_range_name, 0);
+    if (var == NULL) {
+        return NGX_ERROR;
+    }
+
+    var->get_handler = ngx_http_slice_range_variable;
+
+    return NGX_OK;
+}
+
+
+static ngx_int_t
+ngx_http_slice_init(ngx_conf_t *cf)
+{
+    ngx_http_next_header_filter = ngx_http_top_header_filter;
+    ngx_http_top_header_filter = ngx_http_slice_header_filter;
+
+    ngx_http_next_body_filter = ngx_http_top_body_filter;
+    ngx_http_top_body_filter = ngx_http_slice_body_filter;
+
+    return NGX_OK;
+}

src/http/ngx_http_request.h

@@ -271,6 +271,7 @@ typedef struct {
     ngx_array_t                       cache_control;
 
     off_t                             content_length_n;
+    off_t                             content_offset;
     time_t                            date_time;
     time_t                            last_modified_time;
 } ngx_http_headers_out_t;
@@ -530,6 +531,7 @@ struct ngx_http_request_s {
     unsigned                          filter_need_in_memory:1;
     unsigned                          filter_need_temporary:1;
     unsigned                          allow_ranges:1;
+    unsigned                          subrequest_ranges:1;
     unsigned                          single_range:1;
     unsigned                          disable_not_modified:1;