Skip to content
Snippets Groups Projects
Commit fe40f1ef authored by rtaum's avatar rtaum Committed by Rhys Arkins
Browse files

feat(python): add simple endpoint support (#3125)

Closes #2970 
parent 4c26d593
Branches
No related tags found
No related merge requests found
const url = require('url'); const url = require('url');
const is = require('@sindresorhus/is'); const is = require('@sindresorhus/is');
const { parse } = require('node-html-parser');
const { matches } = require('../../versioning/pep440'); const { matches } = require('../../versioning/pep440');
const got = require('../../util/got'); const got = require('../../util/got');
...@@ -35,8 +35,14 @@ async function getPkgReleases({ compatibility, lookupName, registryUrls }) { ...@@ -35,8 +35,14 @@ async function getPkgReleases({ compatibility, lookupName, registryUrls }) {
if (process.env.PIP_INDEX_URL) { if (process.env.PIP_INDEX_URL) {
hostUrls = [process.env.PIP_INDEX_URL]; hostUrls = [process.env.PIP_INDEX_URL];
} }
for (const hostUrl of hostUrls) { for (let hostUrl of hostUrls) {
const dep = await getDependency(lookupName, hostUrl, compatibility); hostUrl += hostUrl.endsWith('/') ? '' : '/';
let dep;
if (hostUrl.endsWith('/simple/')) {
dep = getSimpleDependency(lookupName, hostUrl);
} else {
dep = await getDependency(lookupName, hostUrl, compatibility);
}
if (dep !== null) { if (dep !== null) {
return dep; return dep;
} }
...@@ -91,3 +97,48 @@ async function getDependency(depName, hostUrl, compatibility) { ...@@ -91,3 +97,48 @@ async function getDependency(depName, hostUrl, compatibility) {
return null; return null;
} }
} }
async function getSimpleDependency(depName, hostUrl) {
const lookupUrl = url.resolve(hostUrl, `${depName}`);
try {
const dependency = {};
const response = await got(url.parse(lookupUrl), {
json: false,
});
const dep = response && response.body;
if (!dep) {
logger.debug({ dependency: depName }, 'pip package not found');
return null;
}
const root = parse(dep);
const links = root.querySelectorAll('a');
const versions = new Set();
for (const link of links) {
const result = extractVersionFromLinkText(link.text);
if (result) {
versions.add(result);
}
}
dependency.releases = [];
if (versions && versions.size > 0) {
dependency.releases = [...versions].map(version => ({
version,
}));
}
return dependency;
} catch (err) {
logger.info(
'pypi dependency not found: ' + depName + '(searching in ' + hostUrl + ')'
);
return null;
}
}
function extractVersionFromLinkText(text) {
const versionRegexp = /\d+(\.\d+)+/;
const result = text.match(versionRegexp);
if (result && result.length > 0) {
return result[0];
}
return null;
}
...@@ -25,9 +25,7 @@ function extractPackageFile(content) { ...@@ -25,9 +25,7 @@ function extractPackageFile(content) {
} }
let registryUrls; let registryUrls;
if (pipfile.source) { if (pipfile.source) {
registryUrls = pipfile.source.map(source => registryUrls = pipfile.source.map(source => source.url);
source.url.replace(/simple(\/)?$/, 'pypi/')
);
} }
const deps = [ const deps = [
......
<!DOCTYPE html>
<html>
<head>
<title>Links for dj-database-url</title>
</head>
<body>
<h1>Links for dj-database-url</h1>
<a href="https://files.pythonhosted.org/packages/04/89/29cdbc86a0890a4f1e46b6f4bb9b7959e461e0202f6a305bd8b586cc1404/dj-database-url-0.1.2.tar.gz#sha256=6169f2c272326e3cced6999effb19013365ea73f6ed6c731efa4e346711d8969">dj-database-url.tar.gz</a><br/>
<a href="https://files.pythonhosted.org/packages/bd/80/f8430a065c09367cd766cdea08f80d11b625944a653f96c2bd02d183355b/dj-database-url-0.1.3.tar.gz#sha256=222744896dcbe939aa940217c940a8a95981be13beb9af639a1da024be4f9411">dj-database-url.tar.gz</a><br/>
</body>
</html>
<!DOCTYPE html>
<html>
<head>
<title>Links for dj-database-url</title>
</head>
<body>
<h1>Links for dj-database-url</h1>
<a href="https://files.pythonhosted.org/packages/04/89/29cdbc86a0890a4f1e46b6f4bb9b7959e461e0202f6a305bd8b586cc1404/dj-database-url-0.1.2.tar.gz#sha256=6169f2c272326e3cced6999effb19013365ea73f6ed6c731efa4e346711d8969">dj-database-url-0.1.2.tar.gz</a><br/>
<a href="https://files.pythonhosted.org/packages/bd/80/f8430a065c09367cd766cdea08f80d11b625944a653f96c2bd02d183355b/dj-database-url-0.1.3.tar.gz#sha256=222744896dcbe939aa940217c940a8a95981be13beb9af639a1da024be4f9411">dj-database-url-0.1.3.tar.gz</a><br/>
<a href="https://files.pythonhosted.org/packages/3e/78/c18103be8b9f06a3cb9ee93adb63f91c16f8c8781c8abb0e5b06acef7106/dj-database-url-0.1.4.tar.gz#sha256=14faa143247e267aefd807490e1e89e3ad9fac0a06c4aee3f9fe328849bd15cf">dj-database-url-0.1.4.tar.gz</a><br/>
<a href="https://files.pythonhosted.org/packages/d6/cb/ab7fc10ea1e3571c34bcd74c3b3090e2d3c378d01ad79550fc967cd84114/dj-database-url-0.2.0.tar.gz#sha256=5edd253ccb407a0bd19e91c4c9bbe164632639767086b4a38f2d20e00010488b">dj-database-url-0.2.0.tar.gz</a><br/>
<a href="https://files.pythonhosted.org/packages/a6/94/72572715f45dd132ddc1e26aa4e8c2b13395759386a541e7f00124ceda11/dj-database-url-0.2.1.tar.gz#sha256=f95c0b2e9e70cc246bd101720e1be492524ecf0dd5ea39241b51ef142faefecc">dj-database-url-0.2.1.tar.gz</a><br/>
<a href="https://files.pythonhosted.org/packages/19/11/2867ccdaa0203ed14d9f725b26b6dd3264c4209a16d5bf0096597ecf9a7a/dj-database-url-0.2.2.tar.gz#sha256=492a7294b85ad8ac1b13be0b7337f381d2d44c4da185f289ab7c26dd765ef6cb">dj-database-url-0.2.2.tar.gz</a><br/>
<a href="https://files.pythonhosted.org/packages/e1/0e/2cceb7afb13cf784e385928530af59b49dd1524a76323e293f18ea28a6de/dj-database-url-0.3.0.tar.gz#sha256=f2e273ed34acbb560962d5cf12917936d8df02297df09bd3089b8546d4584138">dj-database-url-0.3.0.tar.gz</a><br/>
<a href="https://files.pythonhosted.org/packages/ef/b6/9283fcf61ced22bf90e7b4a84ba5b53d126b2c9b0dc9b667347698097026/dj_database_url-0.3.0-py2.py3-none-any.whl#sha256=ca01768fdecde134301f3170743226f60edff5c3935f12437378ebd911506353">dj_database_url-0.3.0-py2.py3-none-any.whl</a><br/>
<a href="https://files.pythonhosted.org/packages/64/d9/99774e3f66683ded1d3aa3f66045f671cefc0b550aca4ccfeaeeed4e074a/dj-database-url-0.4.0.tar.gz#sha256=858312abb7b330ea875733a65806a36ad04d7b8451c6ce8835118a2fa10d6870">dj-database-url-0.4.0.tar.gz</a><br/>
<a href="https://files.pythonhosted.org/packages/39/9f/30f937db9f9e7a4e4e3205682af4c34c65d647ff9850897ddfbbf5dc6178/dj-database-url-0.4.1.tar.gz#sha256=7f4c78d2a090df8dfaf56d5d3ff7bbee17360436e4879558317e2314424864cd">dj-database-url-0.4.1.tar.gz</a><br/>
<a href="https://files.pythonhosted.org/packages/c8/4b/b23dbcf4c5711f26e2222bb2e300915c9c8d35e643b0af00c2d8f36c9490/dj-database-url-0.4.2.tar.gz#sha256=a6832d8445ee9d788c5baa48aef8130bf61fdc442f7d9a548424d25cd85c9f08">dj-database-url-0.4.2.tar.gz</a><br/>
<a href="https://files.pythonhosted.org/packages/91/84/50cbfabb91593cff18a37046986f7c2eb69224a694a52ae614711dfa11c6/dj_database_url-0.4.2-py2.py3-none-any.whl#sha256=e16d94c382ea0564c48038fa7fe8d9c890ef1ab1a8ec4cb48e732c124b9482fd">dj_database_url-0.4.2-py2.py3-none-any.whl</a><br/>
<a href="https://files.pythonhosted.org/packages/01/c4/98fbf678e810029be8078419f7bba626aafa2e81bc38748757db954c477c/dj-database-url-0.5.0.tar.gz#sha256=4aeaeb1f573c74835b0686a2b46b85990571159ffc21aa57ecd4d1e1cb334163">dj-database-url-0.5.0.tar.gz</a><br/>
<a href="https://files.pythonhosted.org/packages/d4/a6/4b8578c1848690d0c307c7c0596af2077536c9ef2a04d42b00fabaa7e49d/dj_database_url-0.5.0-py2.py3-none-any.whl#sha256=851785365761ebe4994a921b433062309eb882fedd318e1b0fcecc607ed02da9">dj_database_url-0.5.0-py2.py3-none-any.whl</a><br/>
</body>
</html>
// Jest Snapshot v1, https://goo.gl/fbAQLP // Jest Snapshot v1, https://goo.gl/fbAQLP
exports[`datasource/pypi getPkgReleases process data from simple endpoint 1`] = `
Object {
"releases": Array [
Object {
"version": "0.1.2",
},
Object {
"version": "0.1.3",
},
Object {
"version": "0.1.4",
},
Object {
"version": "0.2.0",
},
Object {
"version": "0.2.1",
},
Object {
"version": "0.2.2",
},
Object {
"version": "0.3.0",
},
Object {
"version": "0.4.0",
},
Object {
"version": "0.4.1",
},
Object {
"version": "0.4.2",
},
Object {
"version": "0.5.0",
},
],
}
`;
exports[`datasource/pypi getPkgReleases processes real data 1`] = ` exports[`datasource/pypi getPkgReleases processes real data 1`] = `
Object { Object {
"releases": Array [ "releases": Array [
...@@ -130,9 +170,9 @@ Array [ ...@@ -130,9 +170,9 @@ Array [
"hash": null, "hash": null,
"host": "custom.pypi.net", "host": "custom.pypi.net",
"hostname": "custom.pypi.net", "hostname": "custom.pypi.net",
"href": "https://custom.pypi.net/azure-cli-monitor/json", "href": "https://custom.pypi.net/foo/azure-cli-monitor/json",
"path": "/azure-cli-monitor/json", "path": "/foo/azure-cli-monitor/json",
"pathname": "/azure-cli-monitor/json", "pathname": "/foo/azure-cli-monitor/json",
"port": null, "port": null,
"protocol": "https:", "protocol": "https:",
"query": null, "query": null,
...@@ -178,9 +218,9 @@ Array [ ...@@ -178,9 +218,9 @@ Array [
"hash": null, "hash": null,
"host": "custom.pypi.net", "host": "custom.pypi.net",
"hostname": "custom.pypi.net", "hostname": "custom.pypi.net",
"href": "https://custom.pypi.net/azure-cli-monitor/json", "href": "https://custom.pypi.net/foo/azure-cli-monitor/json",
"path": "/azure-cli-monitor/json", "path": "/foo/azure-cli-monitor/json",
"pathname": "/azure-cli-monitor/json", "pathname": "/foo/azure-cli-monitor/json",
"port": null, "port": null,
"protocol": "https:", "protocol": "https:",
"query": null, "query": null,
...@@ -197,9 +237,9 @@ Array [ ...@@ -197,9 +237,9 @@ Array [
"hash": null, "hash": null,
"host": "second-index", "host": "second-index",
"hostname": "second-index", "hostname": "second-index",
"href": "https://second-index/azure-cli-monitor/json", "href": "https://second-index/foo/azure-cli-monitor/json",
"path": "/azure-cli-monitor/json", "path": "/foo/azure-cli-monitor/json",
"pathname": "/azure-cli-monitor/json", "pathname": "/foo/azure-cli-monitor/json",
"port": null, "port": null,
"protocol": "https:", "protocol": "https:",
"query": null, "query": null,
......
...@@ -5,6 +5,10 @@ const datasource = require('../../lib/datasource'); ...@@ -5,6 +5,10 @@ const datasource = require('../../lib/datasource');
jest.mock('../../lib/util/got'); jest.mock('../../lib/util/got');
const res1 = fs.readFileSync('test/_fixtures/pypi/azure-cli-monitor.json'); const res1 = fs.readFileSync('test/_fixtures/pypi/azure-cli-monitor.json');
const htmlResponse = fs.readFileSync('test/_fixtures/pypi/versions-html.html');
const badResponse = fs.readFileSync(
'test/_fixtures/pypi/versions-html-badfile.html'
);
describe('datasource/pypi', () => { describe('datasource/pypi', () => {
describe('getPkgReleases', () => { describe('getPkgReleases', () => {
...@@ -160,5 +164,67 @@ describe('datasource/pypi', () => { ...@@ -160,5 +164,67 @@ describe('datasource/pypi', () => {
}) })
).toMatchSnapshot(); ).toMatchSnapshot();
}); });
it('process data from simple endpoint', async () => {
got.mockReturnValueOnce({
body: htmlResponse + '',
});
const config = {
registryUrls: ['https://pypi.org/simple/'],
};
expect(
await datasource.getPkgReleases({
...config,
compatibility: { python: '2.7' },
datasource: 'pypi',
depName: 'dj-database-url',
})
).toMatchSnapshot();
});
it('returns null for empty resonse', async () => {
got.mockReturnValueOnce({});
const config = {
registryUrls: ['https://pypi.org/simple/'],
};
expect(
await datasource.getPkgReleases({
...config,
compatibility: { python: '2.7' },
datasource: 'pypi',
depName: 'dj-database-url',
})
).toBeNull();
});
it('returns null for 404 response from simple endpoint', async () => {
got.mockImplementationOnce(() => {
throw new Error();
});
const config = {
registryUrls: ['https://pypi.org/simple/'],
};
expect(
await datasource.getPkgReleases({
...config,
compatibility: { python: '2.7' },
datasource: 'pypi',
depName: 'dj-database-url',
})
).toBeNull();
});
it('returns null for response with no versions', async () => {
got.mockReturnValueOnce({
body: badResponse + '',
});
const config = {
registryUrls: ['https://pypi.org/simple/'],
};
expect(
await datasource.getPkgReleases({
...config,
compatibility: { python: '2.7' },
datasource: 'pypi',
depName: 'dj-database-url',
})
).toEqual({ releases: [] });
});
}); });
}); });
...@@ -9,7 +9,7 @@ Array [ ...@@ -9,7 +9,7 @@ Array [
"depType": "packages", "depType": "packages",
"pipenvNestedVersion": false, "pipenvNestedVersion": false,
"registryUrls": Array [ "registryUrls": Array [
"https://pypi.org/pypi/", "https://pypi.org/simple",
"http://example.com/private-pypi/", "http://example.com/private-pypi/",
], ],
}, },
...@@ -20,7 +20,7 @@ Array [ ...@@ -20,7 +20,7 @@ Array [
"depType": "packages", "depType": "packages",
"pipenvNestedVersion": false, "pipenvNestedVersion": false,
"registryUrls": Array [ "registryUrls": Array [
"https://pypi.org/pypi/", "https://pypi.org/simple",
"http://example.com/private-pypi/", "http://example.com/private-pypi/",
], ],
}, },
...@@ -31,7 +31,7 @@ Array [ ...@@ -31,7 +31,7 @@ Array [
"depType": "packages", "depType": "packages",
"pipenvNestedVersion": true, "pipenvNestedVersion": true,
"registryUrls": Array [ "registryUrls": Array [
"https://pypi.org/pypi/", "https://pypi.org/simple",
"http://example.com/private-pypi/", "http://example.com/private-pypi/",
], ],
}, },
...@@ -42,7 +42,7 @@ Array [ ...@@ -42,7 +42,7 @@ Array [
"depType": "dev-packages", "depType": "dev-packages",
"pipenvNestedVersion": false, "pipenvNestedVersion": false,
"registryUrls": Array [ "registryUrls": Array [
"https://pypi.org/pypi/", "https://pypi.org/simple",
"http://example.com/private-pypi/", "http://example.com/private-pypi/",
], ],
}, },
...@@ -58,7 +58,7 @@ Array [ ...@@ -58,7 +58,7 @@ Array [
"depType": "packages", "depType": "packages",
"pipenvNestedVersion": false, "pipenvNestedVersion": false,
"registryUrls": Array [ "registryUrls": Array [
"https://pypi.org/pypi/", "https://pypi.org/simple",
], ],
}, },
Object { Object {
...@@ -68,7 +68,7 @@ Array [ ...@@ -68,7 +68,7 @@ Array [
"depType": "packages", "depType": "packages",
"pipenvNestedVersion": false, "pipenvNestedVersion": false,
"registryUrls": Array [ "registryUrls": Array [
"https://pypi.org/pypi/", "https://pypi.org/simple",
], ],
}, },
Object { Object {
...@@ -78,7 +78,7 @@ Array [ ...@@ -78,7 +78,7 @@ Array [
"depType": "packages", "depType": "packages",
"pipenvNestedVersion": false, "pipenvNestedVersion": false,
"registryUrls": Array [ "registryUrls": Array [
"https://pypi.org/pypi/", "https://pypi.org/simple",
], ],
}, },
Object { Object {
...@@ -88,7 +88,7 @@ Array [ ...@@ -88,7 +88,7 @@ Array [
"depType": "packages", "depType": "packages",
"pipenvNestedVersion": false, "pipenvNestedVersion": false,
"registryUrls": Array [ "registryUrls": Array [
"https://pypi.org/pypi/", "https://pypi.org/simple",
], ],
}, },
Object { Object {
...@@ -98,7 +98,7 @@ Array [ ...@@ -98,7 +98,7 @@ Array [
"depType": "packages", "depType": "packages",
"pipenvNestedVersion": false, "pipenvNestedVersion": false,
"registryUrls": Array [ "registryUrls": Array [
"https://pypi.org/pypi/", "https://pypi.org/simple",
], ],
}, },
] ]
......
...@@ -44,16 +44,5 @@ describe('lib/manager/pipenv/extract', () => { ...@@ -44,16 +44,5 @@ describe('lib/manager/pipenv/extract', () => {
const res = extractPackageFile(content, config).deps; const res = extractPackageFile(content, config).deps;
expect(res[0].registryUrls).toEqual(['source-url', 'other-source-url']); expect(res[0].registryUrls).toEqual(['source-url', 'other-source-url']);
}); });
it('converts simple-API URLs to JSON-API URLs', () => {
const content =
'[[source]]\r\nurl = "https://my-pypi/foo/simple/"\r\n' +
'[[source]]\r\nurl = "https://other-pypi/foo/simple"\r\n' +
'[packages]\r\nfoo = "==1.0.0"\r\n';
const res = extractPackageFile(content, config).deps;
expect(res[0].registryUrls).toEqual([
'https://my-pypi/foo/pypi/',
'https://other-pypi/foo/pypi/',
]);
});
}); });
}); });
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment