Skip to content
Snippets Groups Projects
Unverified Commit 5d624ed4 authored by Rhys Arkins's avatar Rhys Arkins Committed by GitHub
Browse files

feat(internal): request caching in http module (#6497)

parent 21028a70
No related branches found
No related tags found
No related merge requests found
import safeStringify from 'fast-safe-stringify'; import safeStringify from 'fast-safe-stringify';
export function clone<T>(input: T): T { export function clone<T>(input: T = null): T {
return JSON.parse(safeStringify(input)); return JSON.parse(safeStringify(input));
} }
// Jest Snapshot v1, https://goo.gl/fbAQLP // Jest Snapshot v1, https://goo.gl/fbAQLP
exports[`util/got/index uses basic auth 1`] = ` exports[`util/got/index gets 1`] = `
Object { Object {
"body": Object {}, "body": Object {},
"options": Object { "options": Object {
"auth": ":test",
"baseUrl": "https://api.github.com/", "baseUrl": "https://api.github.com/",
"cache": false, "cache": false,
"decompress": true, "decompress": true,
"followRedirect": true, "followRedirect": true,
"form": false, "form": false,
"gotTimeout": Object {
"request": 60000,
},
"hash": "", "hash": "",
"headers": Object { "headers": Object {
"accept": "application/json", "accept": "application/json",
...@@ -27,6 +23,7 @@ Object { ...@@ -27,6 +23,7 @@ Object {
"beforeRetry": Array [], "beforeRetry": Array [],
"init": Array [], "init": Array [],
}, },
"hostType": "github",
"hostname": "api.github.com", "hostname": "api.github.com",
"href": "https://api.github.com/some", "href": "https://api.github.com/some",
"json": true, "json": true,
...@@ -43,24 +40,21 @@ Object { ...@@ -43,24 +40,21 @@ Object {
"search": "", "search": "",
"stream": false, "stream": false,
"throwHttpErrors": true, "throwHttpErrors": true,
"useCache": false,
"useElectronNet": false, "useElectronNet": false,
}, },
} }
`; `;
exports[`util/got/index uses basic auth 2`] = ` exports[`util/got/index gets 2`] = `
Object { Object {
"body": Object {}, "body": Object {},
"options": Object { "options": Object {
"auth": ":test",
"baseUrl": "https://api.github.com/", "baseUrl": "https://api.github.com/",
"cache": false, "cache": false,
"decompress": true, "decompress": true,
"followRedirect": true, "followRedirect": true,
"form": false, "form": false,
"gotTimeout": Object {
"request": 60000,
},
"hash": "", "hash": "",
"headers": Object { "headers": Object {
"accept": "application/json", "accept": "application/json",
...@@ -75,10 +69,11 @@ Object { ...@@ -75,10 +69,11 @@ Object {
"beforeRetry": Array [], "beforeRetry": Array [],
"init": Array [], "init": Array [],
}, },
"hostType": "github",
"hostname": "api.github.com", "hostname": "api.github.com",
"href": "https://api.github.com/some", "href": "https://api.github.com/some",
"json": true, "json": true,
"method": "GET", "method": "HEAD",
"path": "/some", "path": "/some",
"pathname": "/some", "pathname": "/some",
"protocol": "https:", "protocol": "https:",
...@@ -96,20 +91,23 @@ Object { ...@@ -96,20 +91,23 @@ Object {
} }
`; `;
exports[`util/got/index uses bearer auth 1`] = ` exports[`util/got/index uses basic auth 1`] = `
Object { Object {
"body": Object {}, "body": Object {},
"options": Object { "options": Object {
"auth": ":test",
"baseUrl": "https://api.github.com/", "baseUrl": "https://api.github.com/",
"cache": false, "cache": false,
"decompress": true, "decompress": true,
"followRedirect": true, "followRedirect": true,
"form": false, "form": false,
"gotTimeout": Object {
"request": 60000,
},
"hash": "", "hash": "",
"headers": Object { "headers": Object {
"accept": "application/json", "accept": "application/json",
"accept-encoding": "gzip, deflate", "accept-encoding": "gzip, deflate",
"authorization": "Bearer XXX",
"user-agent": "got/9.6.0 (https://github.com/sindresorhus/got)", "user-agent": "got/9.6.0 (https://github.com/sindresorhus/got)",
}, },
"hooks": Object { "hooks": Object {
...@@ -141,20 +139,23 @@ Object { ...@@ -141,20 +139,23 @@ Object {
} }
`; `;
exports[`util/got/index uses bearer auth 2`] = ` exports[`util/got/index uses basic auth 2`] = `
Object { Object {
"body": Object {}, "body": Object {},
"options": Object { "options": Object {
"auth": ":test",
"baseUrl": "https://api.github.com/", "baseUrl": "https://api.github.com/",
"cache": false, "cache": false,
"decompress": true, "decompress": true,
"followRedirect": true, "followRedirect": true,
"form": false, "form": false,
"gotTimeout": Object {
"request": 60000,
},
"hash": "", "hash": "",
"headers": Object { "headers": Object {
"accept": "application/json", "accept": "application/json",
"accept-encoding": "gzip, deflate", "accept-encoding": "gzip, deflate",
"authorization": "Bearer XXX",
"user-agent": "got/9.6.0 (https://github.com/sindresorhus/got)", "user-agent": "got/9.6.0 (https://github.com/sindresorhus/got)",
}, },
"hooks": Object { "hooks": Object {
...@@ -186,7 +187,7 @@ Object { ...@@ -186,7 +187,7 @@ Object {
} }
`; `;
exports[`util/got/index uses no cache 1`] = ` exports[`util/got/index uses bearer auth 1`] = `
Object { Object {
"body": Object {}, "body": Object {},
"options": Object { "options": Object {
...@@ -199,6 +200,7 @@ Object { ...@@ -199,6 +200,7 @@ Object {
"headers": Object { "headers": Object {
"accept": "application/json", "accept": "application/json",
"accept-encoding": "gzip, deflate", "accept-encoding": "gzip, deflate",
"authorization": "Bearer XXX",
"user-agent": "got/9.6.0 (https://github.com/sindresorhus/got)", "user-agent": "got/9.6.0 (https://github.com/sindresorhus/got)",
}, },
"hooks": Object { "hooks": Object {
...@@ -209,7 +211,6 @@ Object { ...@@ -209,7 +211,6 @@ Object {
"beforeRetry": Array [], "beforeRetry": Array [],
"init": Array [], "init": Array [],
}, },
"hostType": "github",
"hostname": "api.github.com", "hostname": "api.github.com",
"href": "https://api.github.com/some", "href": "https://api.github.com/some",
"json": true, "json": true,
...@@ -226,13 +227,12 @@ Object { ...@@ -226,13 +227,12 @@ Object {
"search": "", "search": "",
"stream": false, "stream": false,
"throwHttpErrors": true, "throwHttpErrors": true,
"useCache": false,
"useElectronNet": false, "useElectronNet": false,
}, },
} }
`; `;
exports[`util/got/index uses no cache 2`] = ` exports[`util/got/index uses bearer auth 2`] = `
Object { Object {
"body": Object {}, "body": Object {},
"options": Object { "options": Object {
...@@ -245,6 +245,7 @@ Object { ...@@ -245,6 +245,7 @@ Object {
"headers": Object { "headers": Object {
"accept": "application/json", "accept": "application/json",
"accept-encoding": "gzip, deflate", "accept-encoding": "gzip, deflate",
"authorization": "Bearer XXX",
"user-agent": "got/9.6.0 (https://github.com/sindresorhus/got)", "user-agent": "got/9.6.0 (https://github.com/sindresorhus/got)",
}, },
"hooks": Object { "hooks": Object {
...@@ -255,11 +256,10 @@ Object { ...@@ -255,11 +256,10 @@ Object {
"beforeRetry": Array [], "beforeRetry": Array [],
"init": Array [], "init": Array [],
}, },
"hostType": "github",
"hostname": "api.github.com", "hostname": "api.github.com",
"href": "https://api.github.com/some", "href": "https://api.github.com/some",
"json": true, "json": true,
"method": "HEAD", "method": "GET",
"path": "/some", "path": "/some",
"pathname": "/some", "pathname": "/some",
"protocol": "https:", "protocol": "https:",
......
import crypto from 'crypto';
import { logger } from '../../logger';
import * as runCache from '../cache/run';
import { clone } from '../clone';
import { create } from './util';
// With this caching, it means every GET request is cached during each repository run
function cloneBody(response: any): any {
return {
...response,
body: clone(response.body),
};
}
export default create({
options: {},
handler: (options, next) => {
if (options.stream) {
return next(options);
}
if (!['github', 'npm'].includes(options.hostType)) {
return next(options).then(cloneBody);
}
if (options.method === 'GET') {
const cacheKey = crypto
.createHash('md5')
.update(
'got-' +
JSON.stringify({ href: options.href, headers: options.headers })
)
.digest('hex');
if (options.useCache === false) {
logger.trace('GET cache skipped: ' + options.href);
} else {
const cachedGot = runCache.get(cacheKey);
// istanbul ignore if
if (cachedGot) {
logger.trace('GET cache hit: ' + options.href);
return cachedGot;
}
logger.trace('GET cache miss: ' + options.href);
}
const promisedRes = next(options).catch((err) => {
runCache.set(cacheKey, null);
throw err;
});
runCache.set(cacheKey, promisedRes);
return promisedRes.then(cloneBody);
}
return next(options);
},
});
...@@ -69,7 +69,7 @@ describe(getName(__filename), () => { ...@@ -69,7 +69,7 @@ describe(getName(__filename), () => {
expect(req.isDone()).toBe(true); expect(req.isDone()).toBe(true);
}); });
it('uses no cache', async () => { it('gets', async () => {
const req = mock({}) const req = mock({})
.head('/some') .head('/some')
.reply(200, {}) .reply(200, {})
...@@ -93,29 +93,4 @@ describe(getName(__filename), () => { ...@@ -93,29 +93,4 @@ describe(getName(__filename), () => {
expect(req.isDone()).toBe(true); expect(req.isDone()).toBe(true);
}); });
it('streams no cache', async () => {
const req = mock();
const stream = api.stream('/some', {
baseUrl,
});
expect(stream).toBeDefined();
let data = '';
stream.on('data', (c) => {
data += c;
});
const done = new Promise((resolve, reject) => {
stream.on('end', resolve);
stream.on('error', reject);
});
await done;
expect(data).toBe('{}');
expect(req.isDone()).toBe(true);
});
}); });
import got from 'got'; import got from 'got';
import auth from './auth'; import auth from './auth';
import cacheGet from './cache-get';
import hostRules from './host-rules'; import hostRules from './host-rules';
import { mergeInstances } from './util'; import { mergeInstances } from './util';
...@@ -11,6 +10,6 @@ export * from './common'; ...@@ -11,6 +10,6 @@ export * from './common';
* - Cache all GET requests for the lifetime of the repo * - Cache all GET requests for the lifetime of the repo
* *
*/ */
export const api = mergeInstances(got, cacheGet, hostRules, auth); export const api = mergeInstances(got, hostRules, auth);
export default api; export default api;
import crypto from 'crypto';
import URL from 'url'; import URL from 'url';
import { GotPromise } from 'got';
import * as runCache from '../cache/run';
import { clone } from '../clone';
import got from '../got'; import got from '../got';
interface OutgoingHttpHeaders { interface OutgoingHttpHeaders {
...@@ -28,10 +32,21 @@ export interface HttpResponse<T = string> { ...@@ -28,10 +32,21 @@ export interface HttpResponse<T = string> {
headers: any; headers: any;
} }
async function cloneResponse<T>(
promisedResponse: GotPromise<any>
): Promise<HttpResponse<T>> {
const response = await promisedResponse;
// clone body and headers so that the cached result doesn't get accidentally mutated
return {
body: clone<T>(response.body),
headers: clone(response.headers),
};
}
export class Http<GetOptions = HttpOptions, PostOptions = HttpPostOptions> { export class Http<GetOptions = HttpOptions, PostOptions = HttpPostOptions> {
constructor(private hostType: string, private options?: HttpOptions) {} constructor(private hostType: string, private options?: HttpOptions) {}
protected async request<T>( protected request<T>(
requestUrl: string | URL, requestUrl: string | URL,
httpOptions?: InternalHttpOptions httpOptions?: InternalHttpOptions
): Promise<HttpResponse<T> | null> { ): Promise<HttpResponse<T> | null> {
...@@ -72,8 +87,26 @@ export class Http<GetOptions = HttpOptions, PostOptions = HttpPostOptions> { ...@@ -72,8 +87,26 @@ export class Http<GetOptions = HttpOptions, PostOptions = HttpPostOptions> {
process.env.RENOVATE_USER_AGENT || process.env.RENOVATE_USER_AGENT ||
'https://github.com/renovatebot/renovate', 'https://github.com/renovatebot/renovate',
}; };
const res = await got(url, options);
return { body: res.body, headers: res.headers }; // Cache GET requests unless useCache=false
let promisedRes: GotPromise<any>;
if (options.method === 'get') {
const cacheKey = crypto
.createHash('md5')
.update('got-' + JSON.stringify({ url, headers: options.headers }))
.digest('hex');
if (options.useCache !== false) {
// check cache unless bypassing it
promisedRes = runCache.get(cacheKey);
}
if (promisedRes === undefined) {
// cache miss OR cache bypass
promisedRes = got(url, options);
}
runCache.set(cacheKey, promisedRes); // always set
return cloneResponse<T>(promisedRes);
}
return cloneResponse<T>(got(url, options));
} }
get(url: string, options: HttpOptions = {}): Promise<HttpResponse> { get(url: string, options: HttpOptions = {}): Promise<HttpResponse> {
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment