diff --git a/lib/api/github.js b/lib/api/github.js
index 13e727057fd8909c70ebc0d1d22a0474b709bc60..3a7d4a3eb267d8d9e5209f9fc77315d0de491cbf 100644
--- a/lib/api/github.js
+++ b/lib/api/github.js
@@ -147,6 +147,7 @@ async function initRepo(repoName, token, endpoint, repoLogger) {
process.env.GITHUB_ENDPOINT = endpoint;
}
config.repoName = repoName;
+ config.fileList = null;
const platformConfig = {};
try {
const res = await ghGotRetry(`repos/${repoName}`, {
@@ -226,29 +227,32 @@ async function setBaseBranch(branchName) {
// Search
-// Returns an array of file paths in current repo matching the fileName
-async function findFilePaths(fileName, content) {
- let results = [];
- let url = `search/code?q=`;
- if (content) {
- url += `${content}+`;
+// Get full file list
+async function getFileList(branchName) {
+ if (config.fileList) {
+ return config.fileList;
}
- url += `repo:${config.repoName}+filename:${fileName}&per_page=100`;
- do {
- const res = await ghGotRetry(url);
- const exactMatches = res.body.items.filter(item => item.name === fileName);
- // GitHub seems to return files in the root with a leading `/`
- // which then breaks things later on down the line
- results = results.concat(
- exactMatches.map(item => item.path.replace(/^\//, ''))
- );
- const linkHeader = res.headers.link || '';
- const matches = linkHeader.match(
- /<https:\/\/api.github\.com\/(.*?)>; rel="next".*/
+ const res = await ghGotRetry(
+ `repos/${config.repoName}/git/trees/${branchName}?recursive=true`
+ );
+ if (res.body.truncated) {
+ logger.warn(
+ { repository: config.repoName },
+ 'repository tree is truncated'
);
- url = matches ? matches[1] : null;
- } while (url);
- return results;
+ }
+ config.fileList = res.body.tree
+ .filter(item => item.type === 'blob')
+ .map(item => item.path)
+ .sort();
+ return config.fileList;
+}
+
+// Return all files in the repository matching the filename
+async function findFilePaths(fileName, branchName = config.baseBranch) {
+ return (await getFileList(branchName)).filter(fullFilePath =>
+ fullFilePath.endsWith(fileName)
+ );
}
// Branch
diff --git a/lib/api/gitlab.js b/lib/api/gitlab.js
index 443aa3c64c7fe2f5607b79e933e9bec9bef33669..48d3f6a0eab46af0f15755aa2bc309fdf5520417 100644
--- a/lib/api/gitlab.js
+++ b/lib/api/gitlab.js
@@ -101,6 +101,7 @@ async function initRepo(repoName, token, endpoint, repoLogger) {
}
logger.debug(`Detected Gitlab API ${config.apiVersion}`);
config.repoName = repoName.replace('/', '%2F');
+ config.fileList = null;
try {
const res = await glGot(`projects/${config.repoName}`);
config.defaultBranch = res.body.default_branch;
@@ -123,10 +124,26 @@ async function setBaseBranch(branchName) {
// Search
-// Returns an array of file paths in current repo matching the fileName
-async function findFilePaths() {
- logger.debug("Can't find multiple package.json files in GitLab");
- return [];
+// Get full file list
+async function getFileList(branchName) {
+ if (config.fileList) {
+ return config.fileList;
+ }
+ const res = await glGot(
+ `projects/${config.repoName}/repository/tree?ref=${branchName}&recursive=1`
+ );
+ config.fileList = res.body
+ .filter(item => item.type === 'blob')
+ .map(item => item.path)
+ .sort();
+ return config.fileList;
+}
+
+// Return all files in the repository matching the filename
+async function findFilePaths(fileName, branchName = config.baseBranch) {
+ return (await getFileList(branchName)).filter(fullFilePath =>
+ fullFilePath.endsWith(fileName)
+ );
}
// Branch
diff --git a/lib/workers/repository/apis.js b/lib/workers/repository/apis.js
index d035572fe750ad259aaa4946a19c0c720636cfc1..a95d5e7f119bcb656d6f314fb44cb731029b2ef4 100644
--- a/lib/workers/repository/apis.js
+++ b/lib/workers/repository/apis.js
@@ -280,22 +280,13 @@ async function detectPackageFiles(input, detectAllLanguages = false) {
);
}
}
- if (config.packageFiles.length === 0) {
- logger.debug('Checking manually if repository has a package.json');
- if (await config.api.getFileJson('package.json')) {
- config.packageFiles = ['package.json'];
- }
- }
if (config.packageFiles.length) {
config.types.npm = true;
}
}
if (detectAllLanguages || config.meteor.enabled) {
logger.debug('Detecting meteor package.js files');
- const meteorPackageFiles = await config.api.findFilePaths(
- 'package.js',
- 'Npm.depends'
- );
+ const meteorPackageFiles = await config.api.findFilePaths('package.js');
if (meteorPackageFiles.length) {
logger.info(
{ count: meteorPackageFiles.length },
diff --git a/test/api/__snapshots__/github.spec.js.snap b/test/api/__snapshots__/github.spec.js.snap
index 0cc62aac05d7ecceb4ec875a02262a1481bed5b2..d3bc2955f24525d86feff34a3894e6da302e8455 100644
--- a/test/api/__snapshots__/github.spec.js.snap
+++ b/test/api/__snapshots__/github.spec.js.snap
@@ -452,106 +452,7 @@ Array [
]
`;
-exports[`api/github findFilePaths(fileName) paginates 1`] = `
-Array [
- Array [
- "repos/some/repo",
- Object {
- "headers": Object {
- "accept": "application/vnd.github.loki-preview+json",
- },
- },
- ],
- Array [
- "repos/some/repo/git/refs/heads/master",
- undefined,
- ],
- Array [
- "repos/some/repo/branches/master/protection/required_status_checks",
- Object {
- "headers": Object {
- "accept": "application/vnd.github.loki-preview+json",
- },
- },
- ],
- Array [
- "search/code?q=repo:some/repo+filename:package.json&per_page=100",
- undefined,
- ],
- Array [
- "search/code?q=repo%3Arenovate-tests%2Fonboarding-1+filename%3Apackage.json&per_page=2&page=2",
- undefined,
- ],
-]
-`;
-
-exports[`api/github findFilePaths(fileName) paginates 2`] = `
-Array [
- "package.json",
- "src/app/package.json",
- "src/otherapp/package.json",
-]
-`;
-
-exports[`api/github findFilePaths(fileName) should return empty array if none found 1`] = `
-Array [
- Array [
- "repos/some/repo",
- Object {
- "headers": Object {
- "accept": "application/vnd.github.loki-preview+json",
- },
- },
- ],
- Array [
- "repos/some/repo/git/refs/heads/master",
- undefined,
- ],
- Array [
- "repos/some/repo/branches/master/protection/required_status_checks",
- Object {
- "headers": Object {
- "accept": "application/vnd.github.loki-preview+json",
- },
- },
- ],
- Array [
- "search/code?q=repo:some/repo+filename:package.json&per_page=100",
- undefined,
- ],
-]
-`;
-
exports[`api/github findFilePaths(fileName) should return the files matching the fileName 1`] = `
-Array [
- Array [
- "repos/some/repo",
- Object {
- "headers": Object {
- "accept": "application/vnd.github.loki-preview+json",
- },
- },
- ],
- Array [
- "repos/some/repo/git/refs/heads/master",
- undefined,
- ],
- Array [
- "repos/some/repo/branches/master/protection/required_status_checks",
- Object {
- "headers": Object {
- "accept": "application/vnd.github.loki-preview+json",
- },
- },
- ],
- Array [
- "search/code?q=some-content+repo:some/repo+filename:package.json&per_page=100",
- undefined,
- ],
-]
-`;
-
-exports[`api/github findFilePaths(fileName) should return the files matching the fileName 2`] = `
Array [
"package.json",
"src/app/package.json",
diff --git a/test/api/__snapshots__/gitlab.spec.js.snap b/test/api/__snapshots__/gitlab.spec.js.snap
index a3f769b1fcb0794e51688ce15ad93e8ad0f1ba9b..4b1036017a31553ee16ec58f873b6f5e82b1d7be 100644
--- a/test/api/__snapshots__/gitlab.spec.js.snap
+++ b/test/api/__snapshots__/gitlab.spec.js.snap
@@ -193,6 +193,14 @@ Array [
]
`;
+exports[`api/gitlab findFilePaths(fileName) should return the files matching the fileName 1`] = `
+Array [
+ "package.json",
+ "src/app/package.json",
+ "src/otherapp/package.json",
+]
+`;
+
exports[`api/gitlab getBranch returns a branch 1`] = `"foo"`;
exports[`api/gitlab getBranchLastCommitTime should return a Date 1`] = `2012-09-20T08:50:22.000Z`;
diff --git a/test/api/github.spec.js b/test/api/github.spec.js
index bc110a81a5036e05e95a6cc34f95b1dc5f3d163d..9d6e245dc6aa31e072af95a8a928c1b418b76aa3 100644
--- a/test/api/github.spec.js
+++ b/test/api/github.spec.js
@@ -594,66 +594,46 @@ describe('api/github', () => {
});
});
describe('findFilePaths(fileName)', () => {
- it('should return empty array if none found', async () => {
+ it('warns if truncated result', async () => {
await initRepo('some/repo', 'token');
ghGot.mockImplementationOnce(() => ({
- headers: { link: '' },
body: {
- items: [],
+ truncated: true,
+ tree: [],
},
}));
const files = await github.findFilePaths('package.json');
- expect(ghGot.mock.calls).toMatchSnapshot();
expect(files.length).toBe(0);
});
- it('should return the files matching the fileName', async () => {
+ it('caches the result', async () => {
await initRepo('some/repo', 'token');
ghGot.mockImplementationOnce(() => ({
- headers: { link: '' },
body: {
- items: [
- { name: 'package.json', path: '/package.json' },
- {
- name: 'package.json.something-else',
- path: 'some-dir/package.json.some-thing-else',
- },
- { name: 'package.json', path: 'src/app/package.json' },
- { name: 'package.json', path: 'src/otherapp/package.json' },
- ],
+ truncated: true,
+ tree: [],
},
}));
- const files = await github.findFilePaths('package.json', 'some-content');
- expect(ghGot.mock.calls).toMatchSnapshot();
- expect(files).toMatchSnapshot();
+ let files = await github.findFilePaths('package.json');
+ expect(files.length).toBe(0);
+ files = await github.findFilePaths('package.js');
+ expect(files.length).toBe(0);
});
- it('paginates', async () => {
+ it('should return the files matching the fileName', async () => {
await initRepo('some/repo', 'token');
ghGot.mockImplementationOnce(() => ({
- headers: {
- link:
- '<https://api.github.com/search/code?q=repo%3Arenovate-tests%2Fonboarding-1+filename%3Apackage.json&per_page=2&page=2>; rel="next", <https://api.github.com/search/code?q=repo%3Arenovate-tests%2Fonboarding-1+filename%3Apackage.json&per_page=2&page=2>; rel="last" <https://api.github.com/search/code?q=repo%3Arenovate-tests%2Fonboarding-1+filename%3Apackage.json&per_page=2&page=1>; rel="first", <https://api.github.com/search/code?q=repo%3Arenovate-tests%2Fonboarding-1+filename%3Apackage.json&per_page=2&page=1>; rel="prev"',
- },
body: {
- items: [
- { name: 'package.json', path: '/package.json' },
+ tree: [
+ { type: 'blob', path: 'package.json' },
{
- name: 'package.json.something-else',
+ type: 'blob',
path: 'some-dir/package.json.some-thing-else',
},
- ],
- },
- }));
- ghGot.mockImplementationOnce(() => ({
- headers: { link: '' },
- body: {
- items: [
- { name: 'package.json', path: 'src/app/package.json' },
- { name: 'package.json', path: 'src/otherapp/package.json' },
+ { type: 'blob', path: 'src/app/package.json' },
+ { type: 'blob', path: 'src/otherapp/package.json' },
],
},
}));
const files = await github.findFilePaths('package.json');
- expect(ghGot.mock.calls).toMatchSnapshot();
expect(files).toMatchSnapshot();
});
});
diff --git a/test/api/gitlab.spec.js b/test/api/gitlab.spec.js
index 1780f09412a951e8f5d26ac6bd1be59f5ef00416..aaa65a26dff7fcbf2a2503d75bcc7d846e475d84 100644
--- a/test/api/gitlab.spec.js
+++ b/test/api/gitlab.spec.js
@@ -191,11 +191,40 @@ describe('api/gitlab', () => {
});
});
describe('findFilePaths(fileName)', () => {
- it('should return empty array', async () => {
+ it('warns if truncated result', async () => {
await initRepo('some/repo', 'token');
+ glGot.mockImplementationOnce(() => ({
+ body: [],
+ }));
const files = await gitlab.findFilePaths('package.json');
expect(files.length).toBe(0);
});
+ it('caches the result', async () => {
+ await initRepo('some/repo', 'token');
+ glGot.mockImplementationOnce(() => ({
+ body: [],
+ }));
+ let files = await gitlab.findFilePaths('package.json');
+ expect(files.length).toBe(0);
+ files = await gitlab.findFilePaths('package.js');
+ expect(files.length).toBe(0);
+ });
+ it('should return the files matching the fileName', async () => {
+ await initRepo('some/repo', 'token');
+ glGot.mockImplementationOnce(() => ({
+ body: [
+ { type: 'blob', path: 'package.json' },
+ {
+ type: 'blob',
+ path: 'some-dir/package.json.some-thing-else',
+ },
+ { type: 'blob', path: 'src/app/package.json' },
+ { type: 'blob', path: 'src/otherapp/package.json' },
+ ],
+ }));
+ const files = await gitlab.findFilePaths('package.json');
+ expect(files).toMatchSnapshot();
+ });
});
describe('branchExists(branchName)', () => {
it('should return true if 200 OK', async () => {
diff --git a/test/workers/repository/__snapshots__/apis.spec.js.snap b/test/workers/repository/__snapshots__/apis.spec.js.snap
index 3e22cd325917f38026889b1247d209937a387058..a927fdfe7a1cd8a6fb775e118ea3cb5ddb09a309 100644
--- a/test/workers/repository/__snapshots__/apis.spec.js.snap
+++ b/test/workers/repository/__snapshots__/apis.spec.js.snap
@@ -37,12 +37,6 @@ Array [
]
`;
-exports[`workers/repository/apis detectPackageFiles(config) defaults to package.json if found 1`] = `
-Array [
- "package.json",
-]
-`;
-
exports[`workers/repository/apis detectPackageFiles(config) finds Dockerfiles 1`] = `
Array [
"package.json",
diff --git a/test/workers/repository/apis.spec.js b/test/workers/repository/apis.spec.js
index b6b2173b9016d58af1ce1b2dcf44413f4a393c34..3db5589fdef2ec93f3746d7fe51402baccd99f34 100644
--- a/test/workers/repository/apis.spec.js
+++ b/test/workers/repository/apis.spec.js
@@ -320,31 +320,6 @@ describe('workers/repository/apis', () => {
expect(res.foundIgnoredPaths).toMatchSnapshot();
expect(res.warnings).toMatchSnapshot();
});
- it('defaults to package.json if found', async () => {
- const config = {
- ...defaultConfig,
- api: {
- findFilePaths: jest.fn(() => []),
- getFileJson: jest.fn(() => ({})),
- },
- logger,
- };
- const res = await apis.detectPackageFiles(config);
- expect(res.packageFiles).toHaveLength(1);
- expect(res.packageFiles).toMatchSnapshot();
- });
- it('returns empty if package.json not found', async () => {
- const config = {
- ...defaultConfig,
- api: {
- findFilePaths: jest.fn(() => []),
- getFileJson: jest.fn(() => null),
- },
- logger,
- };
- const res = await apis.detectPackageFiles(config);
- expect(res.packageFiles).toEqual([]);
- });
});
describe('resolvePackageFiles', () => {
let config;
diff --git a/test/workers/repository/onboarding.spec.js b/test/workers/repository/onboarding.spec.js
index 8132c0151061c316ea795f90053eea7282e60d36..2c5baebccaaa1497d1618954e26db46e96b0e3ba 100644
--- a/test/workers/repository/onboarding.spec.js
+++ b/test/workers/repository/onboarding.spec.js
@@ -267,6 +267,8 @@ describe('lib/workers/repository/onboarding', () => {
expect(config.api.commitFilesToBranch.mock.calls.length).toBe(0);
});
it('commits files and returns false if no pr', async () => {
+ config.api.findFilePaths.mockReturnValueOnce(['package.json']);
+ config.api.findFilePaths.mockReturnValue([]);
const res = await onboarding.getOnboardingStatus(config);
expect(res.repoIsOnboarded).toEqual(false);
expect(config.api.findPr.mock.calls.length).toBe(1);
@@ -274,6 +276,8 @@ describe('lib/workers/repository/onboarding', () => {
expect(config.api.commitFilesToBranch.mock.calls[0]).toMatchSnapshot();
});
it('pins private repos', async () => {
+ config.api.findFilePaths.mockReturnValueOnce(['package.json']);
+ config.api.findFilePaths.mockReturnValue([]);
onboarding.isRepoPrivate.mockReturnValueOnce(true);
const res = await onboarding.getOnboardingStatus(config);
expect(res.repoIsOnboarded).toEqual(false);