Cache locks initial implementation.

New directives: proxy_cache_lock on/off, proxy_cache_lock_timeout.  With
proxy_cache_lock set to on, only one request will be allowed to go to
upstream for a particular cache item.  Others will wait for a response
to appear in cache (or cache lock released) up to proxy_cache_lock_timeout.

Waiting requests will recheck if they have cached response ready (or are
allowed to run) every 500ms.

Note: we intentionally don't intercept NGX_DECLINED possibly returned by
ngx_http_file_cache_read().  This needs more work (possibly safe, but needs
further investigation).  Anyway, it's exceptional situation.

Note: probably there should be a way to disable caching of responses
if there is already one request fetching resource to cache (without waiting
at all).  Two possible ways include another cache lock option ("no_cache")
or using proxy_no_cache with some supplied variable.

Note: probably there should be a way to lock updating requests as well.  For
now "proxy_cache_use_stale updating" is available.

src/http/modules/ngx_http_proxy_module.c

@@ -402,6 +402,20 @@ static ngx_command_t  ngx_http_proxy_commands[] = {
       offsetof(ngx_http_proxy_loc_conf_t, upstream.cache_methods),
       &ngx_http_upstream_cache_method_mask },
 
+    { ngx_string("proxy_cache_lock"),
+      NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_FLAG,
+      ngx_conf_set_flag_slot,
+      NGX_HTTP_LOC_CONF_OFFSET,
+      offsetof(ngx_http_proxy_loc_conf_t, upstream.cache_lock),
+      NULL },
+
+    { ngx_string("proxy_cache_lock_timeout"),
+      NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1,
+      ngx_conf_set_msec_slot,
+      NGX_HTTP_LOC_CONF_OFFSET,
+      offsetof(ngx_http_proxy_loc_conf_t, upstream.cache_lock_timeout),
+      NULL },
+
 #endif
 
     { ngx_string("proxy_temp_path"),
@@ -2435,6 +2449,8 @@ ngx_http_proxy_create_loc_conf(ngx_conf_t *cf)
     conf->upstream.cache_bypass = NGX_CONF_UNSET_PTR;
     conf->upstream.no_cache = NGX_CONF_UNSET_PTR;
     conf->upstream.cache_valid = NGX_CONF_UNSET_PTR;
+    conf->upstream.cache_lock = NGX_CONF_UNSET;
+    conf->upstream.cache_lock_timeout = NGX_CONF_UNSET_MSEC;
 #endif
 
     conf->upstream.hide_headers = NGX_CONF_UNSET_PTR;
@@ -2681,6 +2697,12 @@ ngx_http_proxy_merge_loc_conf(ngx_conf_t *cf, void *parent, void *child)
         conf->cache_key = prev->cache_key;
     }
 
+    ngx_conf_merge_value(conf->upstream.cache_lock,
+                              prev->upstream.cache_lock, 0);
+
+    ngx_conf_merge_msec_value(conf->upstream.cache_lock_timeout,
+                              prev->upstream.cache_lock_timeout, 5000);
+
 #endif
 
     if (conf->method.len == 0) {

src/http/ngx_http_cache.h

@@ -79,6 +79,14 @@ struct ngx_http_cache_s {
     ngx_http_file_cache_t           *file_cache;
     ngx_http_file_cache_node_t      *node;
 
+    ngx_msec_t                       lock_timeout;
+    ngx_msec_t                       wait_time;
+
+    ngx_event_t                      wait_event;
+
+    unsigned                         lock:1;
+    unsigned                         waiting:1;
+
     unsigned                         updated:1;
     unsigned                         updating:1;
     unsigned                         exists:1;

src/http/ngx_http_file_cache.c

@@ -10,6 +10,9 @@
 #include <ngx_md5.h>
 
 
+static ngx_int_t ngx_http_file_cache_lock(ngx_http_request_t *r,
+    ngx_http_cache_t *c);
+static void ngx_http_file_cache_lock_wait_handler(ngx_event_t *ev);
 static ngx_int_t ngx_http_file_cache_read(ngx_http_request_t *r,
     ngx_http_cache_t *c);
 static ssize_t ngx_http_file_cache_aio_read(ngx_http_request_t *r,
@@ -181,13 +184,13 @@ ngx_http_file_cache_create(ngx_http_request_t *r)
         return NGX_ERROR;
     }
 
+    cln->handler = ngx_http_file_cache_cleanup;
+    cln->data = c;
+
     if (ngx_http_file_cache_exists(cache, c) == NGX_ERROR) {
         return NGX_ERROR;
     }
 
-    cln->handler = ngx_http_file_cache_cleanup;
-    cln->data = c;
-
     if (ngx_http_file_cache_name(r, cache->path) != NGX_OK) {
         return NGX_ERROR;
     }
@@ -244,15 +247,24 @@ ngx_http_file_cache_open(ngx_http_request_t *r)
 
     c = r->cache;
 
+    if (c->waiting) {
+        return NGX_AGAIN;
+    }
+
     if (c->buf) {
         return ngx_http_file_cache_read(r, c);
     }
 
     cache = c->file_cache;
 
-    cln = ngx_pool_cleanup_add(r->pool, 0);
-    if (cln == NULL) {
-        return NGX_ERROR;
+    if (c->node == NULL) {
+        cln = ngx_pool_cleanup_add(r->pool, 0);
+        if (cln == NULL) {
+            return NGX_ERROR;
+        }
+
+        cln->handler = ngx_http_file_cache_cleanup;
+        cln->data = c;
     }
 
     rc = ngx_http_file_cache_exists(cache, c);
@@ -264,9 +276,6 @@ ngx_http_file_cache_open(ngx_http_request_t *r)
         return rc;
     }
 
-    cln->handler = ngx_http_file_cache_cleanup;
-    cln->data = c;
-
     if (rc == NGX_AGAIN) {
         return NGX_HTTP_CACHE_SCARCE;
     }
@@ -306,7 +315,7 @@ ngx_http_file_cache_open(ngx_http_request_t *r)
     }
 
     if (!test) {
-        return NGX_DECLINED;
+        goto done;
     }
 
     clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module);
@@ -330,7 +339,7 @@ ngx_http_file_cache_open(ngx_http_request_t *r)
 
         case NGX_ENOENT:
         case NGX_ENOTDIR:
-            return rv;
+            goto done;
 
         default:
             ngx_log_error(NGX_LOG_CRIT, r->connection->log, of.err,
@@ -354,6 +363,114 @@ ngx_http_file_cache_open(ngx_http_request_t *r)
     }
 
     return ngx_http_file_cache_read(r, c);
+
+done:
+
+    if (rv == NGX_DECLINED) {
+        return ngx_http_file_cache_lock(r, c);
+    }
+
+    return rv;
+}
+
+
+static ngx_int_t
+ngx_http_file_cache_lock(ngx_http_request_t *r, ngx_http_cache_t *c)
+{
+    ngx_msec_t                 now, timer;
+    ngx_http_file_cache_t     *cache;
+
+    if (!c->lock) {
+        return NGX_DECLINED;
+    }
+
+    cache = c->file_cache;
+
+    ngx_shmtx_lock(&cache->shpool->mutex);
+
+    if (!c->node->updating) {
+        c->node->updating = 1;
+        c->updating = 1;
+    }
+
+    ngx_shmtx_unlock(&cache->shpool->mutex);
+
+    ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
+                   "http file cache lock u:%d wt:%M",
+                   c->updating, c->wait_time);
+
+    if (c->updating) {
+        return NGX_DECLINED;
+    }
+
+    c->waiting = 1;
+
+    now = ngx_current_msec;
+
+    if (c->wait_time == 0) {
+        c->wait_time = now + c->lock_timeout;
+
+        c->wait_event.handler = ngx_http_file_cache_lock_wait_handler;
+        c->wait_event.data = r;
+        c->wait_event.log = r->connection->log;
+    }
+
+    timer = c->wait_time - now;
+
+    ngx_add_timer(&c->wait_event, (timer > 500) ? 500 : timer);
+
+    r->main->blocked++;
+
+    return NGX_AGAIN;
+}
+
+
+static void
+ngx_http_file_cache_lock_wait_handler(ngx_event_t *ev)
+{
+    ngx_uint_t                 wait;
+    ngx_msec_t                 timer;
+    ngx_http_cache_t          *c;
+    ngx_http_request_t        *r;
+    ngx_http_file_cache_t     *cache;
+
+    r = ev->data;
+    c = r->cache;
+
+    ngx_log_debug0(NGX_LOG_DEBUG_HTTP, ev->log, 0,
+                   "http file cache wait handler wt:%M cur:%M",
+                   c->wait_time, ngx_current_msec);
+
+    timer = c->wait_time - ngx_current_msec;
+
+    if ((ngx_msec_int_t) timer <= 0) {
+        ngx_log_debug0(NGX_LOG_DEBUG_HTTP, ev->log, 0,
+                       "http file cache lock timeout");
+        c->lock = 0;
+        goto wakeup;
+    }
+
+    cache = c->file_cache;
+    wait = 0;
+
+    ngx_shmtx_lock(&cache->shpool->mutex);
+
+    if (c->node->updating) {
+        wait = 1;
+    }
+
+    ngx_shmtx_unlock(&cache->shpool->mutex);
+
+    if (wait) {
+        ngx_add_timer(ev, (timer > 500) ? 500 : timer);
+        return;
+    }
+
+wakeup:
+
+    c->waiting = 0;
+    r->main->blocked--;
+    r->connection->write->handler(r->connection->write);
 }
 
 
@@ -518,13 +635,19 @@ ngx_http_file_cache_exists(ngx_http_file_cache_t *cache, ngx_http_cache_t *c)
 
     ngx_shmtx_lock(&cache->shpool->mutex);
 
-    fcn = ngx_http_file_cache_lookup(cache, c->key);
+    fcn = c->node;
+
+    if (fcn == NULL) {
+        fcn = ngx_http_file_cache_lookup(cache, c->key);
+    }
 
     if (fcn) {
         ngx_queue_remove(&fcn->queue);
 
-        fcn->uses++;
-        fcn->count++;
+        if (c->node == NULL) {
+            fcn->uses++;
+            fcn->count++;
+        }
 
         if (fcn->error) {
 
@@ -621,6 +744,10 @@ ngx_http_file_cache_name(ngx_http_request_t *r, ngx_path_t *path)
 
     c = r->cache;
 
+    if (c->file.name.len) {
+        return NGX_OK;
+    }
+
     c->file.name.len = path->name.len + 1 + path->len
                        + 2 * NGX_HTTP_CACHE_KEY_LEN;
 
@@ -957,6 +1084,10 @@ ngx_http_file_cache_free(ngx_http_cache_t *c, ngx_temp_file_t *tf)
             }
         }
     }
+
+    if (c->wait_event.timer_set) {
+        ngx_del_timer(&c->wait_event);
+    }
 }
 
 

src/http/ngx_http_upstream.c

@@ -707,6 +707,9 @@ ngx_http_upstream_cache(ngx_http_request_t *r, ngx_http_upstream_t *u)
         c->body_start = u->conf->buffer_size;
         c->file_cache = u->conf->cache->data;
 
+        c->lock = u->conf->cache_lock;
+        c->lock_timeout = u->conf->cache_lock_timeout;
+
         u->cache_status = NGX_HTTP_CACHE_MISS;
     }
 

src/http/ngx_http_upstream.h

@@ -165,6 +165,9 @@ typedef struct {
     ngx_uint_t                       cache_use_stale;
     ngx_uint_t                       cache_methods;
 
+    ngx_flag_t                       cache_lock;
+    ngx_msec_t                       cache_lock_timeout;
+
     ngx_array_t                     *cache_valid;
     ngx_array_t                     *cache_bypass;
     ngx_array_t                     *no_cache;