Skip to content
Snippets Groups Projects
Commit ffef63f4 authored by Rhys Arkins's avatar Rhys Arkins Committed by GitHub
Browse files

feat: suppor gitlab api pagination (#971)

Adds paginated results capability for gitlab and enables it for getFileList. This should enable all files when using APIv4.

Hopeful this c-l-o-s-e-s #962 & #968 
parent 9a97a590
Branches
No related tags found
No related merge requests found
let logger = require('../logger'); let logger = require('../logger');
const get = require('gl-got'); const get = require('./gl-got-wrapper');
const config = {}; const config = {};
...@@ -129,7 +129,8 @@ async function getFileList(branchName) { ...@@ -129,7 +129,8 @@ async function getFileList(branchName) {
return config.fileList; return config.fileList;
} }
const res = await get( const res = await get(
`projects/${config.repoName}/repository/tree?ref=${branchName}&recursive=true` `projects/${config.repoName}/repository/tree?ref=${branchName}&recursive=true`,
{ paginate: true }
); );
config.fileList = res.body config.fileList = res.body
.filter(item => item.type === 'blob') .filter(item => item.type === 'blob')
......
const glGot = require('gl-got');
const parseLinkHeader = require('parse-link-header');
async function get(path, opts, retries = 5) {
const res = await glGot(path, opts);
if (opts && opts.paginate) {
// Check if result is paginated
const linkHeader = parseLinkHeader(res.headers.link);
if (linkHeader && linkHeader.next) {
res.body = res.body.concat(
(await get(linkHeader.next.url, opts, retries)).body
);
}
}
return res;
}
const helpers = ['get', 'post', 'put', 'patch', 'head', 'delete'];
for (const x of helpers) {
get[x] = (url, opts) =>
get(url, Object.assign({}, opts, { method: x.toUpperCase() }));
}
module.exports = get;
...@@ -10,9 +10,9 @@ describe('api/gitlab', () => { ...@@ -10,9 +10,9 @@ describe('api/gitlab', () => {
// reset module // reset module
jest.resetModules(); jest.resetModules();
jest.mock('gl-got'); jest.mock('../../lib/api/gl-got-wrapper');
gitlab = require('../../lib/api/gitlab'); gitlab = require('../../lib/api/gitlab');
get = require('gl-got'); get = require('../../lib/api/gl-got-wrapper');
}); });
describe('getRepos', () => { describe('getRepos', () => {
......
const get = require('../../lib/api/gl-got-wrapper');
const glGot = require('gl-got');
jest.mock('gl-got');
describe('api/gl-got-wrapper', () => {
const body = ['a', 'b'];
beforeEach(() => {
jest.resetAllMocks();
});
it('paginates', async () => {
glGot.mockReturnValueOnce({
headers: {
link:
'<https://api.gitlab.com/search/code?q=addClass+user%3Amozilla&page=2>; rel="next", <https://api.gitlab.com/search/code?q=addClass+user%3Amozilla&page=34>; rel="last"',
},
body: ['a'],
});
glGot.mockReturnValueOnce({
headers: {
link:
'<https://api.gitlab.com/search/code?q=addClass+user%3Amozilla&page=3>; rel="next", <https://api.gitlab.com/search/code?q=addClass+user%3Amozilla&page=34>; rel="last"',
},
body: ['b', 'c'],
});
glGot.mockReturnValueOnce({
headers: {},
body: ['d'],
});
const res = await get('some-url', { paginate: true });
expect(res.body).toHaveLength(4);
expect(glGot.mock.calls).toHaveLength(3);
});
it('attempts to paginate', async () => {
glGot.mockReturnValueOnce({
headers: {
link:
'<https://api.gitlab.com/search/code?q=addClass+user%3Amozilla&page=34>; rel="last"',
},
body: ['a'],
});
glGot.mockReturnValueOnce({
headers: {},
body: ['b'],
});
const res = await get('some-url', { paginate: true });
expect(res.body).toHaveLength(1);
expect(glGot.mock.calls).toHaveLength(1);
});
it('posts', async () => {
glGot.mockImplementationOnce(() => ({
body,
}));
const res = await get.post('some-url');
expect(res.body).toEqual(body);
});
});
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment