Skip to content
Snippets Groups Projects
Unverified Commit 1e1cdf8b authored by Rhys Arkins's avatar Rhys Arkins Committed by GitHub
Browse files

feat(github): cache pagination results (#2458)

Renovate will cache the results body of any paginated results from GitHub, such as list of Pull Requests or release tags. Then, after receiving the first page of results in future, Renovate will compare the received eTag with the cached eTag and if they match then it will return the cached results instead of querying them again via the GitHub API.

The benefits should be two-fold:
- Reduce the number of API calls required to GitHub
- Reduce the time taken to retrieve such results
parent 0d2e3bd4
Branches
No related tags found
No related merge requests found
...@@ -42,6 +42,20 @@ async function get(path, options, retries = 5) { ...@@ -42,6 +42,20 @@ async function get(path, options, retries = 5) {
const pageLimit = opts.pageLimit || 10; const pageLimit = opts.pageLimit || 10;
const linkHeader = parseLinkHeader(res.headers.link); const linkHeader = parseLinkHeader(res.headers.link);
if (linkHeader && linkHeader.next && linkHeader.last) { if (linkHeader && linkHeader.next && linkHeader.last) {
const { etag } = res.headers;
const cacheNamespace = 'github-pagination';
// istanbul ignore next
try {
const cacheResult = await renovateCache.get(cacheNamespace, path);
if (cacheResult.etag === etag) {
logger.trace({ path }, 'Returning cached paginated result');
res.body = cacheResult.body;
return res;
}
logger.trace({ path }, 'Outdated pagination cache');
} catch (err) {
logger.trace({ path }, 'Paginated cache miss');
}
let lastPage = +linkHeader.last.page; let lastPage = +linkHeader.last.page;
if (!process.env.RENOVATE_PAGINATE_ALL) { if (!process.env.RENOVATE_PAGINATE_ALL) {
lastPage = Math.min(pageLimit, lastPage); lastPage = Math.min(pageLimit, lastPage);
...@@ -61,6 +75,13 @@ async function get(path, options, retries = 5) { ...@@ -61,6 +75,13 @@ async function get(path, options, retries = 5) {
res.body = res.body.concat( res.body = res.body.concat(
...pages.filter(Boolean).map(page => page.body) ...pages.filter(Boolean).map(page => page.body)
); );
const cacheMinutes = 60 * 24;
await renovateCache.set(
cacheNamespace,
path,
{ etag, body: res.body },
cacheMinutes
);
} }
} }
if ( if (
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment