Skip to content
Snippets Groups Projects
Unverified Commit 9fea985b authored by jjcaballero's avatar jjcaballero Committed by GitHub
Browse files

feat: create datasource for artifactory registry (#11602)

parent a7b34cf7
No related branches found
No related tags found
No related merge requests found
import { AdoptiumJavaDatasource } from './adoptium-java'; import { AdoptiumJavaDatasource } from './adoptium-java';
import { ArtifactoryDatasource } from './artifactory';
import { BitBucketTagsDatasource } from './bitbucket-tags'; import { BitBucketTagsDatasource } from './bitbucket-tags';
import { CdnJsDatasource } from './cdnjs'; import { CdnJsDatasource } from './cdnjs';
import { ClojureDatasource } from './clojure'; import { ClojureDatasource } from './clojure';
...@@ -40,6 +41,7 @@ const api = new Map<string, DatasourceApi>(); ...@@ -40,6 +41,7 @@ const api = new Map<string, DatasourceApi>();
export default api; export default api;
api.set(AdoptiumJavaDatasource.id, new AdoptiumJavaDatasource()); api.set(AdoptiumJavaDatasource.id, new AdoptiumJavaDatasource());
api.set(ArtifactoryDatasource.id, new ArtifactoryDatasource());
api.set('bitbucket-tags', new BitBucketTagsDatasource()); api.set('bitbucket-tags', new BitBucketTagsDatasource());
api.set('cdnjs', new CdnJsDatasource()); api.set('cdnjs', new CdnJsDatasource());
api.set('clojure', new ClojureDatasource()); api.set('clojure', new ClojureDatasource());
......
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">
<html>
<head>
<meta name="robots" content="noindex" />
<title>Repository Title</title>
</head>
<body>
<h1>Index</h1>
<pre>Name Last modified Size</pre><hr/>
<pre>
<a href="..">..</a>
<a href="1.0.0">1.0.0</a> 21-Jul-2021 20:08 -
<a href="1.0.1">1.0.1</a> 23-Aug-2021 20:03 -
<a href="1.0.2">1.0.2</a> 21-Jul-2021 20:09 -
<a href="1.0.3">1.0.3</a> 06-Feb-2021 09:54 -
</pre>
<hr/>
<address style="font-size:small;">Artifactory Port 8080</address>
</body>
</html>
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">
<html>
<head>
<meta name="robots" content="noindex" />
<title>Repository Title</title>
</head>
<body>
<h1>Index</h1>
<pre>Name Last modified Size</pre><hr/>
<pre>
<a href="../">../</a>
<a href="1.0.0/">1.0.0/</a> 21-Jul-2021 20:08 -
<a href="1.0.1/">1.0.1/</a> 23-Aug-2021 20:03 -
<a href="1.0.2/">1.0.2/</a> 21-Jul-2021 20:09 -
<a href="1.0.3/">1.0.3/</a> 06-Feb-2021 09:54 -
</pre>
<hr/>
<address style="font-size:small;">Artifactory Port 8080</address>
</body>
</html>
// Jest Snapshot v1, https://goo.gl/fbAQLP
exports[`datasource/artifactory/index getReleases parses real data (files): without slash at the end 1`] = `
Object {
"registryUrl": "https://jfrog.company.com/artifactory",
"releases": Array [
Object {
"releaseTimestamp": "2021-07-21T20:08:00.000Z",
"version": "1.0.0",
},
Object {
"releaseTimestamp": "2021-08-23T20:03:00.000Z",
"version": "1.0.1",
},
Object {
"releaseTimestamp": "2021-07-21T20:09:00.000Z",
"version": "1.0.2",
},
Object {
"releaseTimestamp": "2021-02-06T09:54:00.000Z",
"version": "1.0.3",
},
],
}
`;
exports[`datasource/artifactory/index getReleases parses real data (folders): with slash at the end 1`] = `
Object {
"registryUrl": "https://jfrog.company.com/artifactory",
"releases": Array [
Object {
"releaseTimestamp": "2021-07-21T20:08:00.000Z",
"version": "1.0.0",
},
Object {
"releaseTimestamp": "2021-08-23T20:03:00.000Z",
"version": "1.0.1",
},
Object {
"releaseTimestamp": "2021-07-21T20:09:00.000Z",
"version": "1.0.2",
},
Object {
"releaseTimestamp": "2021-02-06T09:54:00.000Z",
"version": "1.0.3",
},
],
}
`;
exports[`datasource/artifactory/index getReleases parses real data (merge strategy with 2 registries) 1`] = `
Object {
"releases": Array [
Object {
"registryUrl": "https://jfrog.company.com/artifactory",
"releaseTimestamp": "2021-07-21T20:08:00.000Z",
"version": "1.0.0",
},
Object {
"registryUrl": "https://jfrog.company.com/artifactory",
"releaseTimestamp": "2021-08-23T20:03:00.000Z",
"version": "1.0.1",
},
Object {
"registryUrl": "https://jfrog.company.com/artifactory",
"releaseTimestamp": "2021-07-21T20:09:00.000Z",
"version": "1.0.2",
},
Object {
"registryUrl": "https://jfrog.company.com/artifactory",
"releaseTimestamp": "2021-02-06T09:54:00.000Z",
"version": "1.0.3",
},
Object {
"registryUrl": "https://jfrog.company.com/artifactory/production",
"version": "1.3.0",
},
],
}
`;
export const datasource = 'artifactory';
import { getPkgReleases } from '..';
import * as httpMock from '../../../test/http-mock';
import { loadFixture } from '../../../test/util';
import { EXTERNAL_HOST_ERROR } from '../../constants/error-messages';
import { logger } from '../../logger';
import { joinUrlParts } from '../../util/url';
import { ArtifactoryDatasource } from '.';
const datasource = ArtifactoryDatasource.id;
const testRegistryUrl = 'https://jfrog.company.com/artifactory';
const testLookupName = 'project';
const testConfig = {
registryUrls: [testRegistryUrl],
depName: testLookupName,
};
const fixtureReleasesAsFolders = loadFixture('releases-as-folders.html');
const fixtureReleasesAsFiles = loadFixture('releases-as-files.html');
function getPath(folder: string): string {
return `/${folder}`;
}
describe('datasource/artifactory/index', () => {
beforeEach(() => {
jest.resetAllMocks();
});
describe('getReleases', () => {
it('parses real data (folders): with slash at the end', async () => {
httpMock
.scope(testRegistryUrl)
.get(getPath(testLookupName))
.reply(200, fixtureReleasesAsFolders);
const res = await getPkgReleases({
...testConfig,
datasource,
lookupName: testLookupName,
});
expect(res.releases).toHaveLength(4);
expect(res).toMatchSnapshot({
registryUrl: 'https://jfrog.company.com/artifactory',
});
});
it('parses real data (files): without slash at the end', async () => {
httpMock
.scope(testRegistryUrl)
.get(getPath(testLookupName))
.reply(200, fixtureReleasesAsFiles);
const res = await getPkgReleases({
...testConfig,
datasource,
lookupName: testLookupName,
});
expect(res.releases).toHaveLength(4);
expect(res).toMatchSnapshot({
registryUrl: 'https://jfrog.company.com/artifactory',
});
});
it('parses real data (merge strategy with 2 registries)', async () => {
const secondRegistryUrl: string = joinUrlParts(
testRegistryUrl,
'production'
);
httpMock
.scope(testRegistryUrl)
.get(getPath(testLookupName))
.reply(200, fixtureReleasesAsFiles);
httpMock
.scope(secondRegistryUrl)
.get(getPath(testLookupName))
.reply(200, '<html>\n<h1>Header</h1>\n<a>1.3.0</a>\n<hmtl/>');
const res = await getPkgReleases({
registryUrls: [testRegistryUrl, secondRegistryUrl],
depName: testLookupName,
datasource,
lookupName: testLookupName,
});
expect(res.releases).toHaveLength(5);
expect(res).toMatchSnapshot();
});
it('returns null without registryUrl + warning', async () => {
const res = await getPkgReleases({
datasource,
depName: testLookupName,
lookupName: testLookupName,
});
expect(logger.warn).toHaveBeenCalledTimes(1);
expect(logger.warn).toHaveBeenCalledWith(
{ lookupName: 'project' },
'artifactory datasource requires custom registryUrl. Skipping datasource'
);
expect(res).toBeNull();
});
it('returns null for empty 200 OK', async () => {
httpMock
.scope(testRegistryUrl)
.get(getPath(testLookupName))
.reply(200, '<html>\n<h1>Header wo. nodes</h1>\n<hmtl/>');
expect(
await getPkgReleases({
...testConfig,
datasource,
lookupName: testLookupName,
})
).toBeNull();
});
it('404 returns null', async () => {
httpMock.scope(testRegistryUrl).get(getPath(testLookupName)).reply(404);
expect(
await getPkgReleases({
...testConfig,
datasource,
lookupName: testLookupName,
})
).toBeNull();
expect(logger.warn).toHaveBeenCalledTimes(1);
expect(logger.warn).toHaveBeenCalledWith(
{
lookupName: 'project',
registryUrl: 'https://jfrog.company.com/artifactory',
},
'artifactory: `Not Found` error'
);
});
it('throws for error diff than 404', async () => {
httpMock.scope(testRegistryUrl).get(getPath(testLookupName)).reply(502);
await expect(
getPkgReleases({
...testConfig,
datasource,
lookupName: testLookupName,
})
).rejects.toThrow(EXTERNAL_HOST_ERROR);
});
it('throws no Http error', async () => {
httpMock
.scope(testRegistryUrl)
.get(getPath(testLookupName))
.replyWithError('unknown error');
const res = await getPkgReleases({
...testConfig,
datasource,
lookupName: testLookupName,
});
expect(res).toBeNull();
});
});
});
import { logger } from '../../logger';
import { cache } from '../../util/cache/package/decorator';
import { parse } from '../../util/html';
import { HttpError } from '../../util/http/types';
import { joinUrlParts } from '../../util/url';
import { Datasource } from '../datasource';
import type { GetReleasesConfig, Release, ReleaseResult } from '../types';
import { datasource } from './common';
export class ArtifactoryDatasource extends Datasource {
static readonly id = datasource;
constructor() {
super(datasource);
}
override readonly customRegistrySupport = true;
override readonly caching = true;
override readonly registryStrategy = 'merge';
@cache({
namespace: `datasource-${datasource}`,
key: ({ registryUrl, lookupName }: GetReleasesConfig) =>
`${registryUrl}:${lookupName}`,
})
async getReleases({
lookupName,
registryUrl,
}: GetReleasesConfig): Promise<ReleaseResult | null> {
if (!registryUrl) {
logger.warn(
{ lookupName },
'artifactory datasource requires custom registryUrl. Skipping datasource'
);
return null;
}
const url = joinUrlParts(registryUrl, lookupName);
const result: ReleaseResult = {
releases: [],
};
try {
const response = await this.http.get(url);
const body = parse(response.body, {
blockTextElements: {
script: true,
noscript: true,
style: true,
},
});
const nodes = body.querySelectorAll('a');
nodes
.filter(
// filter out hyperlink to navigate to parent folder
(node) => node.innerHTML !== '../' && node.innerHTML !== '..'
)
.forEach(
// extract version and published time for each node
(node) => {
const version: string =
node.innerHTML.slice(-1) === '/'
? node.innerHTML.slice(0, -1)
: node.innerHTML;
const published = ArtifactoryDatasource.parseReleaseTimestamp(
node.nextSibling?.text
);
const thisRelease: Release = {
version,
releaseTimestamp: published,
};
result.releases.push(thisRelease);
}
);
if (result.releases.length) {
logger.trace(
{ registryUrl, lookupName, versions: result.releases.length },
'artifactory: Found versions'
);
} else {
logger.trace(
{ registryUrl, lookupName },
'artifactory: No versions found'
);
}
} catch (err) {
// istanbul ignore else: not testable with nock
if (err instanceof HttpError) {
if (err.response?.statusCode === 404) {
logger.warn(
{ registryUrl, lookupName },
'artifactory: `Not Found` error'
);
return null;
}
}
this.handleGenericErrors(err);
}
return result.releases.length ? result : null;
}
private static parseReleaseTimestamp(rawText: string): string {
return rawText.trim().replace(/ ?-$/, '');
}
}
Artifactory is the recommended registry for Conan packages.
This datasource returns releases from given custom `registryUrl`(s).
The target URL is composed by the `registryUrl` and the `lookupName`, which defaults to `depName` when `lookupName` is not defined.
...@@ -13,4 +13,22 @@ describe('util/html', () => { ...@@ -13,4 +13,22 @@ describe('util/html', () => {
const body = parse(''); const body = parse('');
expect(body.childNodes).toHaveLength(0); expect(body.childNodes).toHaveLength(0);
}); });
it('parses HTML: PRE block hides child nodes', () => {
const body = parse('<div>Hello, world!</div>\n<pre><a>node A</a></pre>');
const childNodesA = body.querySelectorAll('a');
expect(childNodesA).toHaveLength(0);
});
it('parses HTML: use additional options to discover child nodes on PRE blocks', () => {
const body = parse('<div>Hello, world!</div>\n<pre><a>node A</a></pre>', {
blockTextElements: {},
});
const childNodesA = body.querySelectorAll('a');
expect(childNodesA).toHaveLength(1);
const div = childNodesA[0];
expect(div.tagName).toBe('A');
expect(div.textContent).toBe('node A');
expect(div instanceof HTMLElement).toBe(true);
});
}); });
import { HTMLElement, parse as _parse } from 'node-html-parser'; import { HTMLElement, Options, parse as _parse } from 'node-html-parser';
export { HTMLElement }; export { HTMLElement };
export function parse(html: string): HTMLElement { export function parse(html: string, config?: Partial<Options>): HTMLElement {
if (typeof config !== 'undefined') {
return _parse(html, config);
}
return _parse(html); return _parse(html);
} }
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment