From 1e1cdf8b6d0c3a9eb0271a15e5662f137e7ae4cc Mon Sep 17 00:00:00 2001
From: Rhys Arkins <rhys@arkins.net>
Date: Mon, 3 Sep 2018 17:35:48 +0200
Subject: [PATCH] feat(github): cache pagination results (#2458)
Renovate will cache the results body of any paginated results from GitHub, such as list of Pull Requests or release tags. Then, after receiving the first page of results in future, Renovate will compare the received eTag with the cached eTag and if they match then it will return the cached results instead of querying them again via the GitHub API.
The benefits should be two-fold:
- Reduce the number of API calls required to GitHub
- Reduce the time taken to retrieve such results
---
lib/platform/github/gh-got-wrapper.js | 21 +++++++++++++++++++++
1 file changed, 21 insertions(+)
diff --git a/lib/platform/github/gh-got-wrapper.js b/lib/platform/github/gh-got-wrapper.js
index c5789abbb3..3a2e61827b 100644
--- a/lib/platform/github/gh-got-wrapper.js
+++ b/lib/platform/github/gh-got-wrapper.js
@@ -42,6 +42,20 @@ async function get(path, options, retries = 5) {
const pageLimit = opts.pageLimit || 10;
const linkHeader = parseLinkHeader(res.headers.link);
if (linkHeader && linkHeader.next && linkHeader.last) {
+ const { etag } = res.headers;
+ const cacheNamespace = 'github-pagination';
+ // istanbul ignore next
+ try {
+ const cacheResult = await renovateCache.get(cacheNamespace, path);
+ if (cacheResult.etag === etag) {
+ logger.trace({ path }, 'Returning cached paginated result');
+ res.body = cacheResult.body;
+ return res;
+ }
+ logger.trace({ path }, 'Outdated pagination cache');
+ } catch (err) {
+ logger.trace({ path }, 'Paginated cache miss');
+ }
let lastPage = +linkHeader.last.page;
if (!process.env.RENOVATE_PAGINATE_ALL) {
lastPage = Math.min(pageLimit, lastPage);
@@ -61,6 +75,13 @@ async function get(path, options, retries = 5) {
res.body = res.body.concat(
...pages.filter(Boolean).map(page => page.body)
);
+ const cacheMinutes = 60 * 24;
+ await renovateCache.set(
+ cacheNamespace,
+ path,
+ { etag, body: res.body },
+ cacheMinutes
+ );
}
}
if (
--
GitLab