diff --git a/lib/api/gh-got-wrapper.js b/lib/api/gh-got-wrapper.js
index d6ad1537fe78487bf65e997b9efa91b6ab952907..89018d6d754b1352f5d6c33eddd638425470e86a 100644
--- a/lib/api/gh-got-wrapper.js
+++ b/lib/api/gh-got-wrapper.js
@@ -1,5 +1,6 @@
const logger = require('../logger');
const ghGot = require('gh-got');
+const parseLinkHeader = require('parse-link-header');
// istanbul ignore next
function sleep(ms) {
@@ -28,13 +29,11 @@ async function get(path, opts, retries = 5) {
const res = await ghGot(path, opts);
if (opts && opts.paginate) {
// Check if result is paginated
- const linkHeader =
- res && res.headers && res.headers.link ? res.headers.link : '';
- const matches = linkHeader.match(
- /<https:\/\/api.github\.com\/(.*?)>; rel="next".*/
- );
- if (matches) {
- res.body = res.body.concat((await get(matches[1], opts, retries)).body);
+ const linkHeader = parseLinkHeader(res.headers.link);
+ if (linkHeader && linkHeader.next) {
+ res.body = res.body.concat(
+ (await get(linkHeader.next.url, opts, retries)).body
+ );
}
}
return res;
diff --git a/package.json b/package.json
index 52ec17a4730aa9cef9e7478bb7b045018ed8ff6a..735c25ea7faf52bc3fc348f21316e843c6cb3378 100644
--- a/package.json
+++ b/package.json
@@ -66,6 +66,7 @@
"moment": "2.19.1",
"moment-timezone": "0.5.13",
"npm": "5.5.1",
+ "parse-link-header": "1.0.1",
"registry-auth-token": "3.3.1",
"root-require": "0.3.1",
"semver": "5.4.1",
diff --git a/test/api/gh-got-wrapper.spec.js b/test/api/gh-got-wrapper.spec.js
index c3bab6214fe89e3b55defb91a8c985658a20007f..ef738176f8c908cf91b33c565f53dfb77f986b4b 100644
--- a/test/api/gh-got-wrapper.spec.js
+++ b/test/api/gh-got-wrapper.spec.js
@@ -19,26 +19,41 @@ describe('api/gh-got-wrapper', () => {
it('paginates', async () => {
ghGot.mockReturnValueOnce({
headers: {
- link: '<https://api.github.com/something>; rel="next">',
+ link:
+ '<https://api.github.com/search/code?q=addClass+user%3Amozilla&page=2>; rel="next", <https://api.github.com/search/code?q=addClass+user%3Amozilla&page=34>; rel="last"',
},
body: ['a'],
});
+ ghGot.mockReturnValueOnce({
+ headers: {
+ link:
+ '<https://api.github.com/search/code?q=addClass+user%3Amozilla&page=3>; rel="next", <https://api.github.com/search/code?q=addClass+user%3Amozilla&page=34>; rel="last"',
+ },
+ body: ['b', 'c'],
+ });
ghGot.mockReturnValueOnce({
headers: {},
- body: ['b'],
+ body: ['d'],
});
const res = await get('some-url', { paginate: true });
- expect(res.body).toHaveLength(2);
+ expect(res.body).toHaveLength(4);
+ expect(ghGot.mock.calls).toHaveLength(3);
});
- it('warns if body cannot be paginated', async () => {
+ it('attempts to paginate', async () => {
ghGot.mockReturnValueOnce({
headers: {
- link: '<https://api.github.com/something>; rel="next">',
+ link:
+ '<https://api.github.com/search/code?q=addClass+user%3Amozilla&page=34>; rel="last"',
},
- body: {},
+ body: ['a'],
});
- const res = await get('some-url');
- expect(res.body).toEqual({});
+ ghGot.mockReturnValueOnce({
+ headers: {},
+ body: ['b'],
+ });
+ const res = await get('some-url', { paginate: true });
+ expect(res.body).toHaveLength(1);
+ expect(ghGot.mock.calls).toHaveLength(1);
});
it('should retry 502s', async () => {
ghGot.mockImplementationOnce(() =>
diff --git a/yarn.lock b/yarn.lock
index 76eb3c78714f4c59dec85f0d7a346d73704081f0..f26ac9227f54d95c0dc4dc29c60f1529a6c29224 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -1160,7 +1160,7 @@ debug@^3.0.1:
dependencies:
ms "2.0.0"
-debuglog@*, debuglog@^1.0.1:
+debuglog@^1.0.1:
version "1.0.1"
resolved "https://registry.yarnpkg.com/debuglog/-/debuglog-1.0.1.tgz#aa24ffb9ac3df9a2351837cfb2d279360cd78492"
@@ -2252,7 +2252,7 @@ import-lazy@^2.1.0:
version "2.1.0"
resolved "https://registry.yarnpkg.com/import-lazy/-/import-lazy-2.1.0.tgz#05698e3d45c88e8d7e9d92cb0584e77f096f3e43"
-imurmurhash@*, imurmurhash@^0.1.4:
+imurmurhash@^0.1.4:
version "0.1.4"
resolved "https://registry.yarnpkg.com/imurmurhash/-/imurmurhash-0.1.4.tgz#9218b9b2b928a238b13dc4fb6b6d576f231453ea"
@@ -3060,10 +3060,6 @@ lodash._basecopy@^3.0.0:
version "3.0.1"
resolved "https://registry.yarnpkg.com/lodash._basecopy/-/lodash._basecopy-3.0.1.tgz#8da0e6a876cf344c0ad8a54882111dd3c5c7ca36"
-lodash._baseindexof@*:
- version "3.1.0"
- resolved "https://registry.yarnpkg.com/lodash._baseindexof/-/lodash._baseindexof-3.1.0.tgz#fe52b53a1c6761e42618d654e4a25789ed61822c"
-
lodash._basetostring@^3.0.0:
version "3.0.1"
resolved "https://registry.yarnpkg.com/lodash._basetostring/-/lodash._basetostring-3.0.1.tgz#d1861d877f824a52f669832dcaf3ee15566a07d5"
@@ -3079,14 +3075,10 @@ lodash._basevalues@^3.0.0:
version "3.0.0"
resolved "https://registry.yarnpkg.com/lodash._basevalues/-/lodash._basevalues-3.0.0.tgz#5b775762802bde3d3297503e26300820fdf661b7"
-lodash._bindcallback@*, lodash._bindcallback@^3.0.0:
+lodash._bindcallback@^3.0.0:
version "3.0.1"
resolved "https://registry.yarnpkg.com/lodash._bindcallback/-/lodash._bindcallback-3.0.1.tgz#e531c27644cf8b57a99e17ed95b35c748789392e"
-lodash._cacheindexof@*:
- version "3.0.2"
- resolved "https://registry.yarnpkg.com/lodash._cacheindexof/-/lodash._cacheindexof-3.0.2.tgz#3dc69ac82498d2ee5e3ce56091bafd2adc7bde92"
-
lodash._createassigner@^3.0.0:
version "3.1.1"
resolved "https://registry.yarnpkg.com/lodash._createassigner/-/lodash._createassigner-3.1.1.tgz#838a5bae2fdaca63ac22dee8e19fa4e6d6970b11"
@@ -3095,17 +3087,11 @@ lodash._createassigner@^3.0.0:
lodash._isiterateecall "^3.0.0"
lodash.restparam "^3.0.0"
-lodash._createcache@*:
- version "3.1.2"
- resolved "https://registry.yarnpkg.com/lodash._createcache/-/lodash._createcache-3.1.2.tgz#56d6a064017625e79ebca6b8018e17440bdcf093"
- dependencies:
- lodash._getnative "^3.0.0"
-
lodash._createset@~4.0.0:
version "4.0.3"
resolved "https://registry.yarnpkg.com/lodash._createset/-/lodash._createset-4.0.3.tgz#0f4659fbb09d75194fa9e2b88a6644d363c9fe26"
-lodash._getnative@*, lodash._getnative@^3.0.0:
+lodash._getnative@^3.0.0:
version "3.9.1"
resolved "https://registry.yarnpkg.com/lodash._getnative/-/lodash._getnative-3.9.1.tgz#570bc7dede46d61cdcde687d65d3eecbaa3aaff5"
@@ -3187,7 +3173,7 @@ lodash.once@^4.0.0:
version "4.1.1"
resolved "https://registry.yarnpkg.com/lodash.once/-/lodash.once-4.1.1.tgz#0dd3971213c7c56df880977d504c88fb471a97ac"
-lodash.restparam@*, lodash.restparam@^3.0.0:
+lodash.restparam@^3.0.0:
version "3.6.1"
resolved "https://registry.yarnpkg.com/lodash.restparam/-/lodash.restparam-3.6.1.tgz#936a4e309ef330a7645ed4145986c85ae5b20805"
@@ -3999,6 +3985,12 @@ parse-json@^2.2.0:
dependencies:
error-ex "^1.2.0"
+parse-link-header@1.0.1:
+ version "1.0.1"
+ resolved "https://registry.yarnpkg.com/parse-link-header/-/parse-link-header-1.0.1.tgz#bedfe0d2118aeb84be75e7b025419ec8a61140a7"
+ dependencies:
+ xtend "~4.0.1"
+
parse-passwd@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/parse-passwd/-/parse-passwd-1.0.0.tgz#6d5b934a456993b23d37f40a382d6f1666a8e5c6"
@@ -4355,7 +4347,7 @@ readable-stream@~2.0.5:
string_decoder "~0.10.x"
util-deprecate "~1.0.1"
-readdir-scoped-modules@*, readdir-scoped-modules@^1.0.0:
+readdir-scoped-modules@^1.0.0:
version "1.0.2"
resolved "https://registry.yarnpkg.com/readdir-scoped-modules/-/readdir-scoped-modules-1.0.2.tgz#9fafa37d286be5d92cbaebdee030dc9b5f406747"
dependencies:
@@ -5294,7 +5286,7 @@ v8flags@^2.1.1:
dependencies:
user-home "^1.1.1"
-validate-npm-package-license@*, validate-npm-package-license@^3.0.1:
+validate-npm-package-license@^3.0.1:
version "3.0.1"
resolved "https://registry.yarnpkg.com/validate-npm-package-license/-/validate-npm-package-license-3.0.1.tgz#2804babe712ad3379459acfbe24746ab2c303fbc"
dependencies: