From 33851a5aea1f6414ce4e2f59185e39db0228a160 Mon Sep 17 00:00:00 2001
From: Rhys Arkins <rhys@keylocation.sg>
Date: Sun, 15 Oct 2017 17:38:45 +0200
Subject: [PATCH] refactor: replace github code search with getTree/manual
search (#944)
Previously Renovate was using the GitHub code search API once for each filename we are looking for. Instead, we now retrieve a list of files in the repository and cache it so it can be reused for filename matches.
Closes #939
---
lib/api/github.js | 47 +++++-----
lib/workers/repository/apis.js | 5 +-
test/api/__snapshots__/github.spec.js.snap | 99 ----------------------
test/api/github.spec.js | 52 ++++--------
4 files changed, 42 insertions(+), 161 deletions(-)
diff --git a/lib/api/github.js b/lib/api/github.js
index 13e727057f..da169d0b3d 100644
--- a/lib/api/github.js
+++ b/lib/api/github.js
@@ -226,29 +226,32 @@ async function setBaseBranch(branchName) {
// Search
-// Returns an array of file paths in current repo matching the fileName
-async function findFilePaths(fileName, content) {
- let results = [];
- let url = `search/code?q=`;
- if (content) {
- url += `${content}+`;
- }
- url += `repo:${config.repoName}+filename:${fileName}&per_page=100`;
- do {
- const res = await ghGotRetry(url);
- const exactMatches = res.body.items.filter(item => item.name === fileName);
- // GitHub seems to return files in the root with a leading `/`
- // which then breaks things later on down the line
- results = results.concat(
- exactMatches.map(item => item.path.replace(/^\//, ''))
- );
- const linkHeader = res.headers.link || '';
- const matches = linkHeader.match(
- /<https:\/\/api.github\.com\/(.*?)>; rel="next".*/
+// Get full file list
+async function getFileList(branchName) {
+ if (config.fileList) {
+ return config.fileList;
+ }
+ const res = await ghGotRetry(
+ `repos/${config.repoName}/git/trees/${branchName}?recursive=1`
+ );
+ if (res.body.truncated) {
+ logger.warn(
+ { repository: config.repoName },
+ 'repository tree is truncated'
);
- url = matches ? matches[1] : null;
- } while (url);
- return results;
+ }
+ config.fileList = res.body.tree
+ .filter(item => item.type === 'blob')
+ .map(item => item.path)
+ .sort();
+ return config.fileList;
+}
+
+// Return all files in the repository matching the filename
+async function findFilePaths(fileName, branchName = config.baseBranch) {
+ return (await getFileList(branchName)).filter(fullFilePath =>
+ fullFilePath.endsWith(fileName)
+ );
}
// Branch
diff --git a/lib/workers/repository/apis.js b/lib/workers/repository/apis.js
index d035572fe7..281b64ce8b 100644
--- a/lib/workers/repository/apis.js
+++ b/lib/workers/repository/apis.js
@@ -292,10 +292,7 @@ async function detectPackageFiles(input, detectAllLanguages = false) {
}
if (detectAllLanguages || config.meteor.enabled) {
logger.debug('Detecting meteor package.js files');
- const meteorPackageFiles = await config.api.findFilePaths(
- 'package.js',
- 'Npm.depends'
- );
+ const meteorPackageFiles = await config.api.findFilePaths('package.js');
if (meteorPackageFiles.length) {
logger.info(
{ count: meteorPackageFiles.length },
diff --git a/test/api/__snapshots__/github.spec.js.snap b/test/api/__snapshots__/github.spec.js.snap
index d291793c82..4b3c768951 100644
--- a/test/api/__snapshots__/github.spec.js.snap
+++ b/test/api/__snapshots__/github.spec.js.snap
@@ -452,106 +452,7 @@ Array [
]
`;
-exports[`api/github findFilePaths(fileName) paginates 1`] = `
-Array [
- Array [
- "repos/some/repo",
- Object {
- "headers": Object {
- "accept": "application/vnd.github.loki-preview+json",
- },
- },
- ],
- Array [
- "repos/some/repo/git/refs/heads/master",
- undefined,
- ],
- Array [
- "repos/some/repo/branches/master/protection/required_status_checks",
- Object {
- "headers": Object {
- "accept": "application/vnd.github.loki-preview+json",
- },
- },
- ],
- Array [
- "search/code?q=repo:some/repo+filename:package.json&per_page=100",
- undefined,
- ],
- Array [
- "search/code?q=repo%3Arenovate-tests%2Fonboarding-1+filename%3Apackage.json&per_page=2&page=2",
- undefined,
- ],
-]
-`;
-
-exports[`api/github findFilePaths(fileName) paginates 2`] = `
-Array [
- "package.json",
- "src/app/package.json",
- "src/otherapp/package.json",
-]
-`;
-
-exports[`api/github findFilePaths(fileName) should return empty array if none found 1`] = `
-Array [
- Array [
- "repos/some/repo",
- Object {
- "headers": Object {
- "accept": "application/vnd.github.loki-preview+json",
- },
- },
- ],
- Array [
- "repos/some/repo/git/refs/heads/master",
- undefined,
- ],
- Array [
- "repos/some/repo/branches/master/protection/required_status_checks",
- Object {
- "headers": Object {
- "accept": "application/vnd.github.loki-preview+json",
- },
- },
- ],
- Array [
- "search/code?q=repo:some/repo+filename:package.json&per_page=100",
- undefined,
- ],
-]
-`;
-
exports[`api/github findFilePaths(fileName) should return the files matching the fileName 1`] = `
-Array [
- Array [
- "repos/some/repo",
- Object {
- "headers": Object {
- "accept": "application/vnd.github.loki-preview+json",
- },
- },
- ],
- Array [
- "repos/some/repo/git/refs/heads/master",
- undefined,
- ],
- Array [
- "repos/some/repo/branches/master/protection/required_status_checks",
- Object {
- "headers": Object {
- "accept": "application/vnd.github.loki-preview+json",
- },
- },
- ],
- Array [
- "search/code?q=some-content+repo:some/repo+filename:package.json&per_page=100",
- undefined,
- ],
-]
-`;
-
-exports[`api/github findFilePaths(fileName) should return the files matching the fileName 2`] = `
Array [
"package.json",
"src/app/package.json",
diff --git a/test/api/github.spec.js b/test/api/github.spec.js
index 8bbd55995e..a6ea923cce 100644
--- a/test/api/github.spec.js
+++ b/test/api/github.spec.js
@@ -591,66 +591,46 @@ describe('api/github', () => {
});
});
describe('findFilePaths(fileName)', () => {
- it('should return empty array if none found', async () => {
+ it('warns if truncated result', async () => {
await initRepo('some/repo', 'token');
ghGot.mockImplementationOnce(() => ({
- headers: { link: '' },
body: {
- items: [],
+ truncated: true,
+ tree: [],
},
}));
const files = await github.findFilePaths('package.json');
- expect(ghGot.mock.calls).toMatchSnapshot();
expect(files.length).toBe(0);
});
- it('should return the files matching the fileName', async () => {
+ it('caches the result', async () => {
await initRepo('some/repo', 'token');
ghGot.mockImplementationOnce(() => ({
- headers: { link: '' },
body: {
- items: [
- { name: 'package.json', path: '/package.json' },
- {
- name: 'package.json.something-else',
- path: 'some-dir/package.json.some-thing-else',
- },
- { name: 'package.json', path: 'src/app/package.json' },
- { name: 'package.json', path: 'src/otherapp/package.json' },
- ],
+ truncated: true,
+ tree: [],
},
}));
- const files = await github.findFilePaths('package.json', 'some-content');
- expect(ghGot.mock.calls).toMatchSnapshot();
- expect(files).toMatchSnapshot();
+ let files = await github.findFilePaths('package.json');
+ expect(files.length).toBe(0);
+ files = await github.findFilePaths('package.js');
+ expect(files.length).toBe(0);
});
- it('paginates', async () => {
+ it('should return the files matching the fileName', async () => {
await initRepo('some/repo', 'token');
ghGot.mockImplementationOnce(() => ({
- headers: {
- link:
- '<https://api.github.com/search/code?q=repo%3Arenovate-tests%2Fonboarding-1+filename%3Apackage.json&per_page=2&page=2>; rel="next", <https://api.github.com/search/code?q=repo%3Arenovate-tests%2Fonboarding-1+filename%3Apackage.json&per_page=2&page=2>; rel="last" <https://api.github.com/search/code?q=repo%3Arenovate-tests%2Fonboarding-1+filename%3Apackage.json&per_page=2&page=1>; rel="first", <https://api.github.com/search/code?q=repo%3Arenovate-tests%2Fonboarding-1+filename%3Apackage.json&per_page=2&page=1>; rel="prev"',
- },
body: {
- items: [
- { name: 'package.json', path: '/package.json' },
+ tree: [
+ { type: 'blob', path: 'package.json' },
{
- name: 'package.json.something-else',
+ type: 'blob',
path: 'some-dir/package.json.some-thing-else',
},
- ],
- },
- }));
- ghGot.mockImplementationOnce(() => ({
- headers: { link: '' },
- body: {
- items: [
- { name: 'package.json', path: 'src/app/package.json' },
- { name: 'package.json', path: 'src/otherapp/package.json' },
+ { type: 'blob', path: 'src/app/package.json' },
+ { type: 'blob', path: 'src/otherapp/package.json' },
],
},
}));
const files = await github.findFilePaths('package.json');
- expect(ghGot.mock.calls).toMatchSnapshot();
expect(files).toMatchSnapshot();
});
});
--
GitLab