diff --git a/README.md b/README.md index a3b575b..0fba8ee 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,7 @@ pooling, proxies, retries, [and more](#features)! * [`make-fetch-happen` options](#extra-options) * [`opts.cachePath`](#opts-cache-path) * [`opts.cache`](#opts-cache) + * [`opts.cacheAdditionalHeaders`](#opts-cache-additional-headers) * [`opts.proxy`](#opts-proxy) * [`opts.noProxy`](#opts-no-proxy) * [`opts.ca, opts.cert, opts.key`](#https-opts) @@ -139,6 +140,7 @@ make-fetch-happen augments the `minipass-fetch` API with additional features ava * [`opts.cachePath`](#opts-cache-path) - Cache target to read/write * [`opts.cache`](#opts-cache) - `fetch` cache mode. Controls cache *behavior*. +* [`opts.cacheAdditionalHeaders`](#opts-cache-additional-headers) - Store additional headers in the cache * [`opts.proxy`](#opts-proxy) - Proxy agent * [`opts.noProxy`](#opts-no-proxy) - Domain segments to disable proxying for. * [`opts.ca, opts.cert, opts.key, opts.strictSSL`](#https-opts) @@ -219,6 +221,34 @@ fetch('https://registry.npmjs.org/make-fetch-happen', { }) ``` +#### `> opts.cacheAdditionalHeaders` + +The following headers are always stored in the cache when present: + +- `cache-control` +- `content-encoding` +- `content-language` +- `content-type` +- `date` +- `etag` +- `expires` +- `last-modified` +- `link` +- `location` +- `pragma` +- `vary` + +This option allows a user to store additional custom headers in the cache. + + +##### Example + +```javascript +fetch('https://registry.npmjs.org/make-fetch-happen', { + cacheAdditionalHeaders: ['my-custom-header'], +}) +``` + #### `> opts.proxy` A string or `new url.URL()`-d URI to proxy through. Different Proxy handlers will be diff --git a/lib/cache/entry.js b/lib/cache/entry.js index dba89d7..12e2fb2 100644 --- a/lib/cache/entry.js +++ b/lib/cache/entry.js @@ -99,6 +99,12 @@ const getMetadata = (request, response, options) => { } } + for (const name of options.cacheAdditionalHeaders) { + if (response.headers.has(name)) { + metadata.resHeaders[name] = response.headers.get(name) + } + } + return metadata } @@ -331,6 +337,7 @@ class CacheEntry { // that reads from cacache and attach it to a new Response const body = new Minipass() const headers = { ...this.policy.responseHeaders() } + const onResume = () => { const cacheStream = cacache.get.stream.byDigest( this.options.cachePath, this.entry.integrity, { memoize: this.options.memoize } @@ -417,6 +424,24 @@ class CacheEntry { } } + for (const name of options.cacheAdditionalHeaders) { + const inMeta = hasOwnProperty(metadata.resHeaders, name) + const inEntry = hasOwnProperty(this.entry.metadata.resHeaders, name) + const inPolicy = hasOwnProperty(this.policy.response.headers, name) + + // if the header is in the existing entry, but it is not in the metadata + // then we need to write it to the metadata as this will refresh the on-disk cache + if (!inMeta && inEntry) { + metadata.resHeaders[name] = this.entry.metadata.resHeaders[name] + } + // if the header is in the metadata, but not in the policy, then we need to set + // it in the policy so that it's included in the immediate response. future + // responses will load a new cache entry, so we don't need to change that + if (!inPolicy && inMeta) { + this.policy.response.headers[name] = metadata.resHeaders[name] + } + } + try { await cacache.index.insert(options.cachePath, this.key, this.entry.integrity, { size: this.entry.size, diff --git a/lib/options.js b/lib/options.js index daa9ecd..f775112 100644 --- a/lib/options.js +++ b/lib/options.js @@ -40,6 +40,8 @@ const configureOptions = (opts) => { } } + options.cacheAdditionalHeaders = options.cacheAdditionalHeaders || [] + // cacheManager is deprecated, but if it's set and // cachePath is not we should copy it to the new field if (options.cacheManager && !options.cachePath) { diff --git a/test/cache.js b/test/cache.js index 565dc76..f4023da 100644 --- a/test/cache.js +++ b/test/cache.js @@ -166,6 +166,59 @@ t.test('no match, fetches and replies even when no content-length', async (t) => }, 'resHeaders has only the relevant headers for caching') }) +t.test('no matches, can store additional headers', async (t) => { + const srv = nock(HOST) + .get('/test') + .reply(200, CONTENT, { + ...getHeaders(CONTENT), + 'x-foo': 'something', + }) + + const reqKey = cacheKey(new Request(`${HOST}/test`)) + const dir = t.testdir() + const res = await fetch(`${HOST}/test`, { cachePath: dir, cacheAdditionalHeaders: ['x-foo'] }) + t.ok(srv.isDone(), 'req is fulfilled') + t.equal(res.status, 200) + t.equal(res.url, `${HOST}/test`, 'has a url property matching the request') + t.equal(res.headers.get('cache-control'), 'max-age=300', 'kept cache-control') + t.equal(res.headers.get('content-type'), 'application/octet-stream', 'kept content-stream') + t.equal(res.headers.get('content-length'), `${CONTENT.length}`, 'kept content-length') + t.equal(res.headers.get('x-local-cache'), encodeURIComponent(dir), 'has cache dir') + t.equal(res.headers.get('x-local-cache-key'), encodeURIComponent(reqKey), 'has cache key') + t.equal(res.headers.get('x-local-cache-mode'), 'stream', 'should stream store') + t.equal(res.headers.get('x-local-cache-status'), 'miss', 'identifies as cache miss') + t.ok(res.headers.has('x-local-cache-time'), 'has cache time') + t.equal(res.headers.get('x-foo'), 'something', 'original response has all headers') + t.notOk(res.headers.has('x-local-cache-hash'), 'hash header is only set when served from cache') + + const dirBeforeRead = await readdir(dir) + t.same(dirBeforeRead, [], 'should not write to the cache yet') + + const buf = await res.buffer() + t.same(buf, CONTENT, 'got the correct content') + const dirAfterRead = await readdir(dir) + // note, this does not make any assumptions about what directories + // are in the cache, only that there is something there. this is so + // our tests do not have to change if cacache version bumps its content + // and/or index directories + t.ok(dirAfterRead.length > 0, 'cache has data after consuming the body') + + // compact with a function that always returns false + // results in a list of all entries in the index + const entries = await cacache.index.compact(dir, reqKey, () => false) + t.equal(entries.length, 1, 'should only have one entry') + const entry = entries[0] + t.equal(entry.integrity, INTEGRITY, 'integrity matches') + t.equal(entry.metadata.url, `${HOST}/test`, 'url matches') + t.same(entry.metadata.reqHeaders, {}, 'metadata has no request headers as none are relevant') + t.same(entry.metadata.resHeaders, { + 'content-type': res.headers.get('content-type'), + 'cache-control': res.headers.get('cache-control'), + date: res.headers.get('date'), + 'x-foo': 'something', + }, 'resHeaders has the relevant headers for caching and our additional header') +}) + t.test('no matches, cache mode only-if-cached rejects', async (t) => { const dir = t.testdir() @@ -204,6 +257,48 @@ t.test('cache hit, no revalidation', async (t) => { t.ok(res.headers.has('x-local-cache-time')) }) +t.test('cache hit, no revalidation, responds with additional headers', async (t) => { + const srv = nock(HOST) + .get('/test') + .reply(200, CONTENT, { + ...getHeaders(CONTENT), + 'x-foo': 'something', + }) + + const dir = t.testdir() + const reqKey = cacheKey(new Request(`${HOST}/test`)) + const cacheRes = await fetch(`${HOST}/test`, { + cachePath: dir, + retry: false, + cacheAdditionalHeaders: ['x-foo'], + }) + await cacheRes.buffer() // drain it immediately so it stores to the cache + t.ok(srv.isDone(), 'req has fulfilled') + + const res = await fetch(`${HOST}/test`, { + cachePath: dir, + retry: false, + cacheAdditionalHeaders: ['x-foo'], + }) + const buf = await res.buffer() + t.same(buf, CONTENT, 'got the right content') + t.equal(res.status, 200, 'got a 200') + t.equal(res.url, `${HOST}/test`, 'has the right url') + t.equal(res.headers.get('cache-control'), 'max-age=300', 'kept cache-control') + t.equal(res.headers.get('content-type'), 'application/octet-stream', 'kept content-type') + t.equal(res.headers.get('content-length'), `${CONTENT.length}`, 'kept content-length') + t.equal(res.headers.get('x-foo'), 'something', 'kept the additional x-foo header') + t.equal(res.headers.get('x-local-cache'), encodeURIComponent(dir), 'encoded the path') + t.equal(res.headers.get('x-local-cache-status'), 'hit', 'got a cache hit') + t.equal(res.headers.get('x-local-cache-key'), encodeURIComponent(reqKey), + 'got the right cache key') + t.equal(res.headers.get('x-local-cache-mode'), 'stream', 'should stream read') + t.equal(res.headers.get('x-local-cache-hash'), encodeURIComponent(INTEGRITY), + 'has the right hash') + // just make sure x-local-cache-time is set, no need to assert its value + t.ok(res.headers.has('x-local-cache-time')) +}) + t.test('cache hit, cache mode no-cache 304', async (t) => { const srv = nock(HOST) .get('/test') @@ -1293,6 +1388,7 @@ t.test('revalidate updates headers in the metadata with new values', async (t) = etag: '"beef"', date: new Date().toISOString(), 'content-type': 'text/plain', + 'x-foo': 'something', }, } @@ -1309,6 +1405,8 @@ t.test('revalidate updates headers in the metadata with new values', async (t) = 'initial entry does not have cache-control') t.equal(beforeEntries[0].metadata.resHeaders['content-type'], 'text/plain', 'initial entry has a content-type') + t.equal(beforeEntries[0].metadata.resHeaders['x-foo'], 'something', + 'initial entry has x-foo') // NOTE: the body must be undefined, not null, otherwise nock // will add an implicit content-type of application/json @@ -1320,14 +1418,22 @@ t.test('revalidate updates headers in the metadata with new values', async (t) = date: new Date().toISOString(), etag: '"beef"', 'cache-control': 'max-age=300', + 'x-bar': 'anything', }) - const revalidateRes = await fetch(`${HOST}/test`, { cachePath: dir }) + const revalidateRes = await fetch(`${HOST}/test`, { + cachePath: dir, + cacheAdditionalHeaders: ['x-foo', 'x-bar'], + }) t.equal(revalidateRes.status, 200, 'got a success status') t.equal(revalidateRes.headers.get('x-local-cache-status'), 'revalidated', 'identifies as revalidated') t.equal(revalidateRes.headers.get('content-type'), 'text/plain', 'got the content-type in the response') + t.equal(revalidateRes.headers.get('x-foo'), 'something', + 'got the cached x-foo in the response') + t.equal(revalidateRes.headers.get('x-bar'), 'anything', + 'got the new x-bar header') await revalidateRes.buffer() t.ok(srv.isDone()) @@ -1338,6 +1444,10 @@ t.test('revalidate updates headers in the metadata with new values', async (t) = 'now has cache-control header') t.equal(afterEntries[0].metadata.resHeaders['content-type'], 'text/plain', 'new index entry kept the content-type') + t.equal(afterEntries[0].metadata.resHeaders['x-foo'], 'something', + 'kept the x-foo header') + t.equal(afterEntries[0].metadata.resHeaders['x-bar'], 'anything', + 'stored the new x-bar header') t.notOk(afterEntries[0].metadata.reqHeaders['user-agent'], 'no longer has a user-agent in reqHeaders') }) diff --git a/test/options.js b/test/options.js index 5fc32ec..0434729 100644 --- a/test/options.js +++ b/test/options.js @@ -14,6 +14,7 @@ test('configure options', async (t) => { cache: 'default', rejectUnauthorized: true, dns: defaultDns, + cacheAdditionalHeaders: [], } t.same(opts, expectedObject, 'should return default opts') }) @@ -26,6 +27,7 @@ test('configure options', async (t) => { cache: 'default', rejectUnauthorized: true, dns: defaultDns, + cacheAdditionalHeaders: [], } t.same(opts, expectedObject, 'should return default opts') }) @@ -39,6 +41,7 @@ test('configure options', async (t) => { cache: 'default', rejectUnauthorized: true, dns: defaultDns, + cacheAdditionalHeaders: [], } t.same(opts, expectedObject, 'should return upper cased method') }) @@ -51,6 +54,7 @@ test('configure options', async (t) => { cache: 'default', rejectUnauthorized: true, dns: defaultDns, + cacheAdditionalHeaders: [], } t.same(trueOpts, trueExpectedObject, 'should return default opts and copy strictSSL') @@ -61,6 +65,7 @@ test('configure options', async (t) => { cache: 'default', rejectUnauthorized: false, dns: defaultDns, + cacheAdditionalHeaders: [], } t.same(falseOpts, falseExpectedObject, 'should return default opts and copy strictSSL') @@ -87,6 +92,7 @@ test('configure options', async (t) => { cache: 'default', rejectUnauthorized: true, dns: defaultDns, + cacheAdditionalHeaders: [], } t.same(opts, expectedObject, 'should return default retry property') }) @@ -100,6 +106,7 @@ test('configure options', async (t) => { cache: 'default', rejectUnauthorized: true, dns: { ...defaultDns, ttl: 100 }, + cacheAdditionalHeaders: [], } t.same(opts, expectedObject, 'should extend default dns with custom ttl') }) @@ -114,6 +121,7 @@ test('configure options', async (t) => { cache: 'default', rejectUnauthorized: true, dns: { ...defaultDns, lookup }, + cacheAdditionalHeaders: [], } t.same(opts, expectedObject, 'should extend default dns with custom lookup') }) @@ -129,6 +137,7 @@ test('configure options', async (t) => { cache: 'default', rejectUnauthorized: true, dns: defaultDns, + cacheAdditionalHeaders: [], } t.same(opts, expectedObject, 'should return default retry property') }) @@ -142,6 +151,7 @@ test('configure options', async (t) => { cache: 'default', rejectUnauthorized: true, dns: defaultDns, + cacheAdditionalHeaders: [], } t.same(opts, expectedObject, 'should return default retry property') }) @@ -155,6 +165,7 @@ test('configure options', async (t) => { cache: 'default', rejectUnauthorized: true, dns: defaultDns, + cacheAdditionalHeaders: [], } t.same(opts, expectedObject, 'should set retry value, if number') }) @@ -168,6 +179,7 @@ test('configure options', async (t) => { cache: 'default', rejectUnauthorized: true, dns: defaultDns, + cacheAdditionalHeaders: [], } t.same(opts, expectedObject, 'should set retry value') }) @@ -181,6 +193,7 @@ test('configure options', async (t) => { cache: 'default', rejectUnauthorized: true, dns: defaultDns, + cacheAdditionalHeaders: [], } t.same(opts, expectedObject, 'should return default retry property') }) @@ -196,6 +209,7 @@ test('configure options', async (t) => { retry: { retries: 0 }, cache: 'default', dns: defaultDns, + cacheAdditionalHeaders: [], } t.same(opts, expectedObject, 'should set the default cache') }) @@ -209,6 +223,7 @@ test('configure options', async (t) => { retry: { retries: 0 }, cache: 'something', dns: defaultDns, + cacheAdditionalHeaders: [], } t.same(opts, expectedObject, 'should keep the provided cache') }) @@ -222,6 +237,7 @@ test('configure options', async (t) => { retry: { retries: 0 }, cache: 'default', cachePath: './foo', + cacheAdditionalHeaders: [], } t.match(opts, expectedObject) }) @@ -236,6 +252,7 @@ test('configure options', async (t) => { cache: 'default', cachePath: './foo', cacheManager: './foo', + cacheAdditionalHeaders: [], } t.match(opts, expectedObject) }) @@ -248,6 +265,7 @@ test('configure options', async (t) => { rejectUnauthorized: true, retry: { retries: 0 }, cache: 'no-store', + cacheAdditionalHeaders: [], } t.match(opts, expectedObject) })