feat(internal): apply host-rules in http module (#6501)

lib/datasource/docker/__snapshots__/index.spec.ts.snap

@@ -152,6 +152,7 @@ exports[`api/docker getReleases adds library/ prefix for Docker Hub (implicit) 1
     Array [
       "https://index.docker.io/v2/",
       Object {
+        "auth": "some-username:some-password",
         "headers": Object {
           "user-agent": "https://github.com/renovatebot/renovate",
         },
@@ -416,6 +417,7 @@ exports[`api/docker getReleases uses custom registry in depName 1`] = `
     Array [
       "https://registry.company.com/v2/",
       Object {
+        "auth": "some-username:some-password",
         "headers": Object {
           "user-agent": "https://github.com/renovatebot/renovate",
         },
@@ -432,6 +434,7 @@ exports[`api/docker getReleases uses custom registry in depName 1`] = `
     Array [
       "https://registry.company.com/v2/node/tags/list?n=10000",
       Object {
+        "auth": "some-username:some-password",
         "headers": Object {
           "user-agent": "https://github.com/renovatebot/renovate",
         },
@@ -448,6 +451,7 @@ exports[`api/docker getReleases uses custom registry in depName 1`] = `
     Array [
       "https://registry.company.com/v2/",
       Object {
+        "auth": "some-username:some-password",
         "headers": Object {
           "user-agent": "https://github.com/renovatebot/renovate",
         },
@@ -493,6 +497,7 @@ Array [
   Array [
     "https://registry.company.com/v2/",
     Object {
+      "auth": "some-username:some-password",
       "headers": Object {
         "user-agent": "https://github.com/renovatebot/renovate",
       },
@@ -509,6 +514,7 @@ Array [
   Array [
     "https://registry.company.com/v2/node/tags/list?n=10000",
     Object {
+      "auth": "some-username:some-password",
       "headers": Object {
         "user-agent": "https://github.com/renovatebot/renovate",
       },
@@ -525,6 +531,7 @@ Array [
   Array [
     "https://api.github.com/user/9287/repos?page=3&per_page=100",
     Object {
+      "auth": "some-username:some-password",
       "headers": Object {
         "user-agent": "https://github.com/renovatebot/renovate",
       },
@@ -541,6 +548,7 @@ Array [
   Array [
     "https://registry.company.com/v2/",
     Object {
+      "auth": "some-username:some-password",
       "headers": Object {
         "user-agent": "https://github.com/renovatebot/renovate",
       },
@@ -557,6 +565,7 @@ Array [
   Array [
     "https://registry.company.com/v2/node/manifests/latest",
     Object {
+      "auth": "some-username:some-password",
       "headers": Object {
         "accept": "application/vnd.docker.distribution.manifest.v2+json",
         "user-agent": "https://github.com/renovatebot/renovate",
@@ -579,6 +588,7 @@ exports[`api/docker getReleases uses lower tag limit for ECR deps 1`] = `
     Array [
       "https://123456789.dkr.ecr.us-east-1.amazonaws.com/v2/",
       Object {
+        "auth": "some-username:some-password",
         "headers": Object {
           "user-agent": "https://github.com/renovatebot/renovate",
         },
@@ -595,6 +605,7 @@ exports[`api/docker getReleases uses lower tag limit for ECR deps 1`] = `
     Array [
       "https://123456789.dkr.ecr.us-east-1.amazonaws.com/v2/node/tags/list?n=1000",
       Object {
+        "auth": "some-username:some-password",
         "headers": Object {
           "user-agent": "https://github.com/renovatebot/renovate",
         },
@@ -611,6 +622,7 @@ exports[`api/docker getReleases uses lower tag limit for ECR deps 1`] = `
     Array [
       "https://123456789.dkr.ecr.us-east-1.amazonaws.com/v2/",
       Object {
+        "auth": "some-username:some-password",
         "headers": Object {
           "user-agent": "https://github.com/renovatebot/renovate",
         },

lib/datasource/maven/index.spec.ts

@@ -37,6 +37,7 @@ describe('datasource/maven', () => {
       hostName: 'frontend_for_private_s3_repository',
       username: 'username',
       password: 'password',
+      timeout: 20000,
     });
     jest.resetAllMocks();
     nock.cleanAll();

lib/manager/cargo/artifacts.spec.ts

@@ -12,6 +12,7 @@ import * as cargo from './artifacts';
 jest.mock('fs-extra');
 jest.mock('child_process');
 jest.mock('../../util/exec/env');
+jest.mock('../../util/http');
 
 const fs: jest.Mocked<typeof _fs> = _fs as any;
 const exec: jest.Mock<typeof _exec> = _exec as any;

lib/manager/gomod/artifacts.spec.ts

@@ -15,6 +15,7 @@ jest.mock('fs-extra');
 jest.mock('child_process');
 jest.mock('../../util/exec/env');
 jest.mock('../../util/host-rules');
+jest.mock('../../util/http');
 
 const fs: jest.Mocked<typeof _fs> = _fs as any;
 const exec: jest.Mock<typeof _exec> = _exec as any;

lib/manager/pipenv/artifacts.spec.ts

@@ -14,6 +14,7 @@ jest.mock('fs-extra');
 jest.mock('child_process');
 jest.mock('../../util/exec/env');
 jest.mock('../../util/host-rules');
+jest.mock('../../util/http');
 
 const fs: jest.Mocked<typeof _fs> = _fs as any;
 const exec: jest.Mock<typeof _exec> = _exec as any;

lib/util/got/__snapshots__/index.spec.ts.snap

@@ -90,281 +90,3 @@ Object {
   },
 }
 `;
-
-exports[`util/got/index uses  basic auth 1`] = `
-Object {
-  "body": Object {},
-  "options": Object {
-    "auth": ":test",
-    "baseUrl": "https://api.github.com/",
-    "cache": false,
-    "decompress": true,
-    "followRedirect": true,
-    "form": false,
-    "gotTimeout": Object {
-      "request": 60000,
-    },
-    "hash": "",
-    "headers": Object {
-      "accept": "application/json",
-      "accept-encoding": "gzip, deflate",
-      "user-agent": "got/9.6.0 (https://github.com/sindresorhus/got)",
-    },
-    "hooks": Object {
-      "afterResponse": Array [],
-      "beforeError": Array [],
-      "beforeRedirect": Array [],
-      "beforeRequest": Array [],
-      "beforeRetry": Array [],
-      "init": Array [],
-    },
-    "hostname": "api.github.com",
-    "href": "https://api.github.com/some",
-    "json": true,
-    "method": "GET",
-    "path": "/some",
-    "pathname": "/some",
-    "protocol": "https:",
-    "retry": Object {
-      "errorCodes": Set {},
-      "methods": Set {},
-      "retries": [Function],
-      "statusCodes": Set {},
-    },
-    "search": "",
-    "stream": false,
-    "throwHttpErrors": true,
-    "useElectronNet": false,
-  },
-}
-`;
-
-exports[`util/got/index uses  basic auth 2`] = `
-Object {
-  "body": Object {},
-  "options": Object {
-    "auth": ":test",
-    "baseUrl": "https://api.github.com/",
-    "cache": false,
-    "decompress": true,
-    "followRedirect": true,
-    "form": false,
-    "gotTimeout": Object {
-      "request": 60000,
-    },
-    "hash": "",
-    "headers": Object {
-      "accept": "application/json",
-      "accept-encoding": "gzip, deflate",
-      "user-agent": "got/9.6.0 (https://github.com/sindresorhus/got)",
-    },
-    "hooks": Object {
-      "afterResponse": Array [],
-      "beforeError": Array [],
-      "beforeRedirect": Array [],
-      "beforeRequest": Array [],
-      "beforeRetry": Array [],
-      "init": Array [],
-    },
-    "hostname": "api.github.com",
-    "href": "https://api.github.com/some",
-    "json": true,
-    "method": "GET",
-    "path": "/some",
-    "pathname": "/some",
-    "protocol": "https:",
-    "retry": Object {
-      "errorCodes": Set {},
-      "methods": Set {},
-      "retries": [Function],
-      "statusCodes": Set {},
-    },
-    "search": "",
-    "stream": false,
-    "throwHttpErrors": true,
-    "useElectronNet": false,
-  },
-}
-`;
-
-exports[`util/got/index uses  bearer auth 1`] = `
-Object {
-  "body": Object {},
-  "options": Object {
-    "baseUrl": "https://api.github.com/",
-    "cache": false,
-    "decompress": true,
-    "followRedirect": true,
-    "form": false,
-    "hash": "",
-    "headers": Object {
-      "accept": "application/json",
-      "accept-encoding": "gzip, deflate",
-      "authorization": "Bearer XXX",
-      "user-agent": "got/9.6.0 (https://github.com/sindresorhus/got)",
-    },
-    "hooks": Object {
-      "afterResponse": Array [],
-      "beforeError": Array [],
-      "beforeRedirect": Array [],
-      "beforeRequest": Array [],
-      "beforeRetry": Array [],
-      "init": Array [],
-    },
-    "hostname": "api.github.com",
-    "href": "https://api.github.com/some",
-    "json": true,
-    "method": "GET",
-    "path": "/some",
-    "pathname": "/some",
-    "protocol": "https:",
-    "retry": Object {
-      "errorCodes": Set {},
-      "methods": Set {},
-      "retries": [Function],
-      "statusCodes": Set {},
-    },
-    "search": "",
-    "stream": false,
-    "throwHttpErrors": true,
-    "useElectronNet": false,
-  },
-}
-`;
-
-exports[`util/got/index uses  bearer auth 2`] = `
-Object {
-  "body": Object {},
-  "options": Object {
-    "baseUrl": "https://api.github.com/",
-    "cache": false,
-    "decompress": true,
-    "followRedirect": true,
-    "form": false,
-    "hash": "",
-    "headers": Object {
-      "accept": "application/json",
-      "accept-encoding": "gzip, deflate",
-      "authorization": "Bearer XXX",
-      "user-agent": "got/9.6.0 (https://github.com/sindresorhus/got)",
-    },
-    "hooks": Object {
-      "afterResponse": Array [],
-      "beforeError": Array [],
-      "beforeRedirect": Array [],
-      "beforeRequest": Array [],
-      "beforeRetry": Array [],
-      "init": Array [],
-    },
-    "hostname": "api.github.com",
-    "href": "https://api.github.com/some",
-    "json": true,
-    "method": "GET",
-    "path": "/some",
-    "pathname": "/some",
-    "protocol": "https:",
-    "retry": Object {
-      "errorCodes": Set {},
-      "methods": Set {},
-      "retries": [Function],
-      "statusCodes": Set {},
-    },
-    "search": "",
-    "stream": false,
-    "throwHttpErrors": true,
-    "useElectronNet": false,
-  },
-}
-`;
-
-exports[`util/got/index uses private-token auth 1`] = `
-Object {
-  "body": Object {},
-  "options": Object {
-    "baseUrl": "https://api.github.com/",
-    "cache": false,
-    "decompress": true,
-    "followRedirect": true,
-    "form": false,
-    "hash": "",
-    "headers": Object {
-      "Private-token": "XXX",
-      "accept": "application/json",
-      "accept-encoding": "gzip, deflate",
-      "user-agent": "got/9.6.0 (https://github.com/sindresorhus/got)",
-    },
-    "hooks": Object {
-      "afterResponse": Array [],
-      "beforeError": Array [],
-      "beforeRedirect": Array [],
-      "beforeRequest": Array [],
-      "beforeRetry": Array [],
-      "init": Array [],
-    },
-    "hostType": "gitlab",
-    "hostname": "api.github.com",
-    "href": "https://api.github.com/some",
-    "json": true,
-    "method": "GET",
-    "path": "/some",
-    "pathname": "/some",
-    "protocol": "https:",
-    "retry": Object {
-      "errorCodes": Set {},
-      "methods": Set {},
-      "retries": [Function],
-      "statusCodes": Set {},
-    },
-    "search": "",
-    "stream": false,
-    "throwHttpErrors": true,
-    "useElectronNet": false,
-  },
-}
-`;
-
-exports[`util/got/index uses token auth 1`] = `
-Object {
-  "body": Object {},
-  "options": Object {
-    "baseUrl": "https://api.github.com/",
-    "cache": false,
-    "decompress": true,
-    "followRedirect": true,
-    "form": false,
-    "hash": "",
-    "headers": Object {
-      "accept": "application/json",
-      "accept-encoding": "gzip, deflate",
-      "authorization": "token XXX",
-      "user-agent": "got/9.6.0 (https://github.com/sindresorhus/got)",
-    },
-    "hooks": Object {
-      "afterResponse": Array [],
-      "beforeError": Array [],
-      "beforeRedirect": Array [],
-      "beforeRequest": Array [],
-      "beforeRetry": Array [],
-      "init": Array [],
-    },
-    "hostType": "gitea",
-    "hostname": "api.github.com",
-    "href": "https://api.github.com/some",
-    "json": true,
-    "method": "GET",
-    "path": "/some",
-    "pathname": "/some",
-    "protocol": "https:",
-    "retry": Object {
-      "errorCodes": Set {},
-      "methods": Set {},
-      "retries": [Function],
-      "statusCodes": Set {},
-    },
-    "search": "",
-    "stream": false,
-    "throwHttpErrors": true,
-    "useElectronNet": false,
-  },
-}
-`;

lib/util/got/host-rules.ts

-/* eslint-disable no-param-reassign */
-import { logger } from '../../logger';
-import * as hostRules from '../host-rules';
-import { create } from './util';
-
-// Apply host rules to requests
-
-export default create({
-  options: {},
-  handler: (options, next) => {
-    // istanbul ignore if: never happen?
-    if (!options.hostname) {
-      return next(options);
-    }
-    const { username, password, token, timeout } = hostRules.find({
-      hostType: options.hostType,
-      url: options.href,
-    });
-    if (options.headers.authorization || options.auth || options.token) {
-      logger.trace('Authorization already set for host: ' + options.hostname);
-    } else if (password) {
-      logger.trace(
-        'Applying Basic authentication for host ' + options.hostname
-      );
-      options.auth = `${username || ''}:${password}`;
-    } else if (token) {
-      logger.trace(
-        'Applying Bearer authentication for host ' + options.hostname
-      );
-      options.token = token;
-    }
-    if (timeout) {
-      options.gotTimeout = { request: timeout };
-    }
-    return next(options);
-  },
-});

lib/util/got/index.spec.ts

 import nock from 'nock';
 import { getName } from '../../../test/util';
-import {
-  PLATFORM_TYPE_GITEA,
-  PLATFORM_TYPE_GITHUB,
-  PLATFORM_TYPE_GITLAB,
-} from '../../constants/platforms';
+import { PLATFORM_TYPE_GITHUB } from '../../constants/platforms';
 import * as hostRules from '../host-rules';
 import { GotJSONOptions } from './common';
 import { api } from '.';
@@ -36,39 +32,6 @@ describe(getName(__filename), () => {
     return nock(baseUrl, opts).get('/some').times(times).reply(200, {});
   }
 
-  it('uses  bearer auth', async () => {
-    const req = mock({ reqheaders: { authorization: 'Bearer XXX' } }, 2);
-    hostRules.add({ baseUrl, token: 'XXX' });
-
-    expect(await got()).toMatchSnapshot();
-    expect(await got({ token: 'XXX' })).toMatchSnapshot();
-    expect(req.isDone()).toBe(true);
-  });
-
-  it('uses  basic auth', async () => {
-    const req = mock({ reqheaders: { authorization: 'Basic OnRlc3Q=' } }, 2);
-
-    hostRules.add({ password: 'test', timeout: 60000 });
-
-    expect(await got()).toMatchSnapshot();
-    expect(await got({ auth: ':test' })).toMatchSnapshot();
-    expect(req.isDone()).toBe(true);
-  });
-
-  it('uses token auth', async () => {
-    const req = mock({ reqheaders: { authorization: 'token XXX' } });
-    hostRules.add({ baseUrl, token: 'XXX' });
-    expect(await got({ hostType: PLATFORM_TYPE_GITEA })).toMatchSnapshot();
-    expect(req.isDone()).toBe(true);
-  });
-
-  it('uses private-token auth', async () => {
-    const req = mock({ reqheaders: { 'private-token': 'XXX' } });
-    hostRules.add({ baseUrl, token: 'XXX' });
-    expect(await got({ hostType: PLATFORM_TYPE_GITLAB })).toMatchSnapshot();
-    expect(req.isDone()).toBe(true);
-  });
-
   it('gets', async () => {
     const req = mock({})
       .head('/some')

lib/util/got/index.ts

 import got from 'got';
 import auth from './auth';
-import hostRules from './host-rules';
 import { mergeInstances } from './util';
 
 export * from './common';
@@ -10,6 +9,6 @@ export * from './common';
  *  - Cache all GET requests for the lifetime of the repo
  *
  */
-export const api = mergeInstances(got, hostRules, auth);
+export const api = mergeInstances(got, auth);
 
 export default api;

lib/util/http/host-rules.ts

+import { logger } from '../../logger';
+import * as hostRules from '../host-rules';
+
+// Apply host rules to requests
+
+export function applyHostRules(url: string, inOptions: any): any {
+  const options = { ...inOptions };
+  const foundRules =
+    hostRules.find({
+      hostType: options.hostType,
+      url,
+    }) || {};
+  const { username, password, token, timeout } = foundRules;
+  if (options.headers?.authorization || options.auth || options.token) {
+    logger.trace('Authorization already set for host: ' + options.hostname);
+  } else if (password) {
+    logger.trace('Applying Basic authentication for host ' + options.hostname);
+    options.auth = `${username || ''}:${password}`;
+  } else if (token) {
+    logger.trace('Applying Bearer authentication for host ' + options.hostname);
+    options.token = token;
+  }
+  if (timeout) {
+    options.timeout = timeout;
+  }
+  return options;
+}

lib/util/http/index.ts

@@ -4,6 +4,7 @@ import { GotPromise } from 'got';
 import * as runCache from '../cache/run';
 import { clone } from '../clone';
 import got from '../got';
+import { applyHostRules } from './host-rules';
 
 interface OutgoingHttpHeaders {
   [header: string]: number | string | string[] | undefined;
@@ -55,7 +56,7 @@ export class Http<GetOptions = HttpOptions, PostOptions = HttpPostOptions> {
       url = URL.resolve(httpOptions.baseUrl, url);
     }
     // TODO: deep merge in order to merge headers
-    const options: any = {
+    let options: any = {
       method: 'get',
       ...this.options,
       hostType: this.hostType,
@@ -88,6 +89,8 @@ export class Http<GetOptions = HttpOptions, PostOptions = HttpPostOptions> {
         'https://github.com/renovatebot/renovate',
     };
 
+    options = applyHostRules(url, options);
+
     // Cache GET requests unless useCache=false
     let promisedRes: GotPromise<any>;
     if (options.method === 'get') {