Skip to content
Snippets Groups Projects
Unverified Commit e3ccc35d authored by Daniel Tschinder's avatar Daniel Tschinder Committed by GitHub
Browse files

feat(repology): Support all repositories of repology (#7833)

parent c63ffa5d
Branches
No related tags found
No related merge requests found
......@@ -20,7 +20,7 @@ Array [
"user-agent": "https://github.com/renovatebot/renovate",
},
"method": "GET",
"url": "https://repology.org/tools/project-by?repo=debian_stable&name_type=binname&target_page=api_v1_project&noautoresolve=on&name=nginx",
"url": "https://repology.org/api/v1/project/nginx",
},
]
`;
......@@ -45,7 +45,7 @@ Array [
"user-agent": "https://github.com/renovatebot/renovate",
},
"method": "GET",
"url": "https://repology.org/tools/project-by?repo=debian_stable&name_type=binname&target_page=api_v1_project&noautoresolve=on&name=pulseaudio-utils",
"url": "https://repology.org/api/v1/project/pulseaudio-utils",
},
]
`;
......@@ -70,7 +70,7 @@ Array [
"user-agent": "https://github.com/renovatebot/renovate",
},
"method": "GET",
"url": "https://repology.org/tools/project-by?repo=alpine_3_12&name_type=binname&target_page=api_v1_project&noautoresolve=on&name=gcc",
"url": "https://repology.org/api/v1/project/gcc",
},
]
`;
......@@ -95,17 +95,7 @@ Array [
"user-agent": "https://github.com/renovatebot/renovate",
},
"method": "GET",
"url": "https://repology.org/tools/project-by?repo=debian_stable&name_type=binname&target_page=api_v1_project&noautoresolve=on&name=gcc-defaults",
},
Object {
"headers": Object {
"accept": "application/json",
"accept-encoding": "gzip, deflate",
"host": "repology.org",
"user-agent": "https://github.com/renovatebot/renovate",
},
"method": "GET",
"url": "https://repology.org/tools/project-by?repo=debian_stable&name_type=srcname&target_page=api_v1_project&noautoresolve=on&name=gcc-defaults",
"url": "https://repology.org/api/v1/project/gcc-defaults",
},
]
`;
......@@ -120,17 +110,7 @@ Array [
"user-agent": "https://github.com/renovatebot/renovate",
},
"method": "GET",
"url": "https://repology.org/tools/project-by?repo=debian_stable&name_type=binname&target_page=api_v1_project&noautoresolve=on&name=nginx",
},
Object {
"headers": Object {
"accept": "application/json",
"accept-encoding": "gzip, deflate",
"host": "repology.org",
"user-agent": "https://github.com/renovatebot/renovate",
},
"method": "GET",
"url": "https://repology.org/tools/project-by?repo=debian_stable&name_type=srcname&target_page=api_v1_project&noautoresolve=on&name=nginx",
"url": "https://repology.org/api/v1/project/nginx",
},
]
`;
......@@ -145,42 +125,7 @@ Array [
"user-agent": "https://github.com/renovatebot/renovate",
},
"method": "GET",
"url": "https://repology.org/tools/project-by?repo=this_should&name_type=binname&target_page=api_v1_project&noautoresolve=on&name=never-exist",
},
Object {
"headers": Object {
"accept": "application/json",
"accept-encoding": "gzip, deflate",
"host": "repology.org",
"user-agent": "https://github.com/renovatebot/renovate",
},
"method": "GET",
"url": "https://repology.org/tools/project-by?repo=this_should&name_type=srcname&target_page=api_v1_project&noautoresolve=on&name=never-exist",
},
]
`;
exports[`datasource/repology/index getReleases returns null for unsupported repository 1`] = `
Array [
Object {
"headers": Object {
"accept": "application/json",
"accept-encoding": "gzip, deflate",
"host": "repology.org",
"user-agent": "https://github.com/renovatebot/renovate",
},
"method": "GET",
"url": "https://repology.org/tools/project-by?repo=unsupported_repo&name_type=binname&target_page=api_v1_project&noautoresolve=on&name=nginx",
},
Object {
"headers": Object {
"accept": "application/json",
"accept-encoding": "gzip, deflate",
"host": "repology.org",
"user-agent": "https://github.com/renovatebot/renovate",
},
"method": "GET",
"url": "https://repology.org/tools/project-by?repo=unsupported_repo&name_type=srcname&target_page=api_v1_project&noautoresolve=on&name=nginx",
"url": "https://repology.org/api/v1/project/never-exist",
},
]
`;
......@@ -195,32 +140,7 @@ Array [
"user-agent": "https://github.com/renovatebot/renovate",
},
"method": "GET",
"url": "https://repology.org/tools/project-by?repo=debian_stable&name_type=binname&target_page=api_v1_project&noautoresolve=on&name=nginx",
},
]
`;
exports[`datasource/repology/index getReleases throws error on unexpected response during source package lookup 1`] = `
Array [
Object {
"headers": Object {
"accept": "application/json",
"accept-encoding": "gzip, deflate",
"host": "repology.org",
"user-agent": "https://github.com/renovatebot/renovate",
},
"method": "GET",
"url": "https://repology.org/tools/project-by?repo=debian_stable&name_type=binname&target_page=api_v1_project&noautoresolve=on&name=nginx",
},
Object {
"headers": Object {
"accept": "application/json",
"accept-encoding": "gzip, deflate",
"host": "repology.org",
"user-agent": "https://github.com/renovatebot/renovate",
},
"method": "GET",
"url": "https://repology.org/tools/project-by?repo=debian_stable&name_type=srcname&target_page=api_v1_project&noautoresolve=on&name=nginx",
"url": "https://repology.org/api/v1/project/nginx",
},
]
`;
......
......@@ -6,37 +6,15 @@ import { EXTERNAL_HOST_ERROR } from '../../constants/error-messages';
import { id as versioning } from '../../versioning/loose';
import { RepologyPackage, id as datasource } from '.';
const repologyApiHost = 'https://repology.org/';
const repologyApiHost = 'https://repology.org/api/v1/';
type ResponseMock = { status: number; body?: string };
const mockProjectBy = (
repo: string,
name: string,
binary: ResponseMock,
source: ResponseMock
) => {
const endpoint = '/tools/project-by';
const defaultParams = {
target_page: 'api_v1_project',
noautoresolve: 'on',
};
if (binary) {
httpMock
.scope(repologyApiHost)
.get(endpoint)
.query({ ...defaultParams, repo, name, name_type: 'binname' })
.reply(binary.status, binary.body);
}
if (source) {
const mockProjectBy = (name: string, response: ResponseMock) => {
httpMock
.scope(repologyApiHost)
.get(endpoint)
.query({ ...defaultParams, repo, name, name_type: 'srcname' })
.reply(source.status, source.body);
}
.get(`/project/${name}`)
.reply(response.status, response.body);
};
const fixtureNginx = fs.readFileSync(
......@@ -65,12 +43,7 @@ describe(getName(__filename), () => {
afterEach(() => httpMock.reset());
it('returns null for empty result', async () => {
mockProjectBy(
'debian_stable',
'nginx',
{ status: 200, body: '[]' },
{ status: 200, body: '[]' }
);
mockProjectBy('nginx', { status: 200, body: '[]' });
expect(
await getPkgReleases({
......@@ -83,12 +56,7 @@ describe(getName(__filename), () => {
});
it('returns null for missing repository or package', async () => {
mockProjectBy(
'this_should',
'never-exist',
{ status: 404 },
{ status: 404 }
);
mockProjectBy('never-exist', { status: 404 });
expect(
await getPkgReleases({
......@@ -100,39 +68,8 @@ describe(getName(__filename), () => {
expect(httpMock.getTrace()).toMatchSnapshot();
});
it('returns null for unsupported repository', async () => {
mockProjectBy(
'unsupported_repo',
'nginx',
{ status: 403 },
{ status: 403 }
);
expect(
await getPkgReleases({
datasource,
versioning,
depName: 'unsupported_repo/nginx',
})
).toBeNull();
expect(httpMock.getTrace()).toMatchSnapshot();
});
it('throws error on unexpected response during binary package lookup', async () => {
mockProjectBy('debian_stable', 'nginx', { status: 500 }, null);
await expect(
getPkgReleases({
datasource,
versioning,
depName: 'debian_stable/nginx',
})
).rejects.toThrow(EXTERNAL_HOST_ERROR);
expect(httpMock.getTrace()).toMatchSnapshot();
});
it('throws error on unexpected response during source package lookup', async () => {
mockProjectBy('debian_stable', 'nginx', { status: 404 }, { status: 500 });
mockProjectBy('nginx', { status: 500 });
await expect(
getPkgReleases({
......@@ -156,12 +93,7 @@ describe(getName(__filename), () => {
});
it('returns correct version for binary package', async () => {
mockProjectBy(
'debian_stable',
'nginx',
{ status: 200, body: fixtureNginx },
null
);
mockProjectBy('nginx', { status: 200, body: fixtureNginx });
const res = await getPkgReleases({
datasource,
......@@ -175,12 +107,7 @@ describe(getName(__filename), () => {
});
it('returns correct version for source package', async () => {
mockProjectBy(
'debian_stable',
'gcc-defaults',
{ status: 404 },
{ status: 200, body: fixtureGccDefaults }
);
mockProjectBy('gcc-defaults', { status: 200, body: fixtureGccDefaults });
const res = await getPkgReleases({
datasource,
......@@ -194,12 +121,7 @@ describe(getName(__filename), () => {
});
it('returns correct version for multi-package project with same name', async () => {
mockProjectBy(
'alpine_3_12',
'gcc',
{ status: 200, body: fixtureGcc },
null
);
mockProjectBy('gcc', { status: 200, body: fixtureGcc });
const res = await getPkgReleases({
datasource,
......@@ -213,12 +135,10 @@ describe(getName(__filename), () => {
});
it('returns correct version for multi-package project with different name', async () => {
mockProjectBy(
'debian_stable',
'pulseaudio-utils',
{ status: 200, body: fixturePulseaudio },
null
);
mockProjectBy('pulseaudio-utils', {
status: 200,
body: fixturePulseaudio,
});
const res = await getPkgReleases({
datasource,
......@@ -238,12 +158,7 @@ describe(getName(__filename), () => {
];
const pkgsJSON = JSON.stringify(pkgs);
mockProjectBy(
'dummy',
'example',
{ status: 200, body: pkgsJSON },
{ status: 200, body: pkgsJSON }
);
mockProjectBy('example', { status: 200, body: pkgsJSON });
expect(
await getPkgReleases({
......
import { URLSearchParams } from 'url';
import { HOST_DISABLED } from '../../constants/error-messages';
import { logger } from '../../logger';
import { ExternalHostError } from '../../types/errors/external-host-error';
......@@ -25,36 +24,30 @@ export interface RepologyPackage {
async function queryPackage(
repoName: string,
pkgName: string,
pkgType: RepologyPackageType
pkgName: string
): Promise<RepologyPackage> {
try {
const query = new URLSearchParams({
repo: repoName,
name_type: pkgType,
target_page: 'api_v1_project',
noautoresolve: 'on',
name: pkgName,
}).toString();
// Retrieve list of packages by looking up Repology project
const url = `https://repology.org/tools/project-by?${query}`;
const url = `https://repology.org/api/v1/project/${pkgName}`;
const res = await http.getJson<RepologyPackage[]>(url);
let pkgs = res.body.filter((pkg) => pkg.repo === repoName);
// In some cases Repology bundles multiple packages into a single project,
// which would result in ambiguous results. If we have more than one result
// left, we should try to determine the correct package by comparing either
// binname or srcname (depending on pkgType) to the given dependency name.
// binname or srcname to the given dependency name.
if (pkgs.length > 1) {
pkgs = pkgs.filter((pkg) => !pkg.binname || pkg.binname === pkgName);
}
if (pkgs.length > 1) {
pkgs = pkgs.filter((pkg) => !pkg[pkgType] || pkg[pkgType] === pkgName);
pkgs = pkgs.filter((pkg) => !pkg.srcname || pkg.srcname === pkgName);
}
// Abort if there is still more than one package left, as the result would
// be ambiguous and unreliable. This should usually not happen...
if (pkgs.length > 1) {
logger.warn(
{ repoName, pkgName, pkgType, pkgs },
{ repoName, pkgName, pkgs },
'Repology lookup returned ambiguous results, ignoring...'
);
return null;
......@@ -64,14 +57,9 @@ async function queryPackage(
} catch (err) {
if (err.statusCode === 404) {
logger.debug(
{ repoName, pkgName, pkgType },
{ repoName, pkgName },
'Repository or package not found on Repology'
);
} else if (err.statusCode === 403) {
logger.debug(
{ repoName },
'Repology does not support tools/project-by lookups for repository'
);
} else {
throw err;
}
......@@ -95,21 +83,14 @@ async function getCachedPackage(
return cachedResult;
}
// Attempt a binary package lookup and return if successfully
const binPkg = await queryPackage(repoName, pkgName, 'binname');
if (binPkg) {
await packageCache.set(cacheNamespace, cacheKey, binPkg, cacheMinutes);
return binPkg;
}
// Otherwise, attempt a source package lookup and return if successfully
const srcPkg = await queryPackage(repoName, pkgName, 'srcname');
if (srcPkg) {
await packageCache.set(cacheNamespace, cacheKey, srcPkg, cacheMinutes);
return srcPkg;
// Attempt a package lookup and return if successfully
const pkg = await queryPackage(repoName, pkgName);
if (pkg) {
await packageCache.set(cacheNamespace, cacheKey, pkg, cacheMinutes);
return pkg;
}
// No binary or source package was found on Repology
// No package was found on Repology
return null;
}
......
......@@ -6,8 +6,6 @@ A [list of all supported repositories](https://repology.org/repositories/statist
As an example, the `Alpine Linux 3.12` repository points to `https://repology.org/repository/alpine_3_12` and therefor has the repository identifier `alpine_3_12`.
Please note that as of today, the Repology API endpoint `/tools/project-by` does not support a small subset of repositories. You can manually double-check [on their website](https://repology.org/tools/project-by) if your desired repository is supported and otherwise raise a request on their side. This datasource will also print a debug message `Repology does not support tools/project-by lookups for repository` if the given repository is unsupported.
**Usage Example**
A real world example for this specific datasource would be maintaining system packages within a Dockerfile, as this allows to specifically pin each dependency without having to manually keep the versions up-to-date. This can be achieved by configuring a generic regex manager in `renovate.json` for files named `Dockerfile`:
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment