diff --git a/docs/usage/configuration-options.md b/docs/usage/configuration-options.md index 27c09e582920807c10ec8d718423d2483d7cea88..a20aa4275f7896ef1eb1be1b49973c6e0bcb9125 100644 --- a/docs/usage/configuration-options.md +++ b/docs/usage/configuration-options.md @@ -1179,6 +1179,26 @@ Example config: Use an exact host for `matchHost` and not a domain (e.g. `api.github.com` as shown above and not `github.com`). Do not combine with `hostType` in the same rule or it won't work. +### maxRequestsPerSecond + +In addition to `concurrentRequestLimit`, you can limit the maximum number of requests that can be made per one second. +It can be used to set minimal delay between two requests to the same host. +Fractional values are allowed, e.g. `0.25` means 1 request per 4 seconds. +Default value `0` means no limit. + +Example config: + +```json +{ + "hostRules": [ + { + "matchHost": "api.github.com", + "maxRequestsPerSecond": 2 + } + ] +} +``` + ### dnsCache Enable got [dnsCache](https://github.com/sindresorhus/got/blob/v11.5.2/readme.md#dnsCache) support. diff --git a/lib/config/options/index.ts b/lib/config/options/index.ts index 71413beedcae4d5df3c5a0429317241d2947923b..931f6f556ae74e588d21eea15bc69c3dc8010c7f 100644 --- a/lib/config/options/index.ts +++ b/lib/config/options/index.ts @@ -2100,6 +2100,16 @@ const options: RenovateOptions[] = [ cli: false, env: false, }, + { + name: 'maxRequestsPerSecond', + description: 'Limit requests rate per host.', + type: 'integer', + stage: 'repository', + parent: 'hostRules', + default: 0, + cli: false, + env: false, + }, { name: 'authType', description: diff --git a/lib/modules/platform/github/index.spec.ts b/lib/modules/platform/github/index.spec.ts index 211719a4dfc689e1dbd5959f2d12faa4d181d88c..b34539a9913ad00e731fc9c9235a7a2ff45fafdf 100644 --- a/lib/modules/platform/github/index.spec.ts +++ b/lib/modules/platform/github/index.spec.ts @@ -2235,7 +2235,7 @@ describe('modules/platform/github/index', () => { await github.createPr(prConfig); expect(logger.logger.debug).toHaveBeenNthCalledWith( - 10, + 11, { prNumber: 123 }, 'GitHub-native automerge: not supported on this version of GHE. Use 3.3.0 or newer.' ); diff --git a/lib/types/host-rules.ts b/lib/types/host-rules.ts index 5cc234b37f4477c9500fd7acf2014376be27caa1..d5b23a80ac246bf092b404c793b66f43afa6fabf 100644 --- a/lib/types/host-rules.ts +++ b/lib/types/host-rules.ts @@ -10,6 +10,7 @@ export interface HostRuleSearchResult { enabled?: boolean; enableHttp2?: boolean; concurrentRequestLimit?: number; + maxRequestsPerSecond?: number; dnsCache?: boolean; keepalive?: boolean; diff --git a/lib/util/http/host-rules.ts b/lib/util/http/host-rules.ts index ede097e2ea0bf77cf97d98ddf045113e92ef3747..f52bc5d15de20336644303c386f54d9c64d4cfdb 100644 --- a/lib/util/http/host-rules.ts +++ b/lib/util/http/host-rules.ts @@ -1,3 +1,4 @@ +import is from '@sindresorhus/is'; import { BITBUCKET_API_USING_HOST_TYPES, GITHUB_API_USING_HOST_TYPES, @@ -120,10 +121,16 @@ export function applyHostRules(url: string, inOptions: GotOptions): GotOptions { return options; } -export function getRequestLimit(url: string): number | null { - const hostRule = hostRules.find({ - url, - }); - const limit = hostRule.concurrentRequestLimit; - return typeof limit === 'number' && limit > 0 ? limit : null; +export function getConcurrentRequestsLimit(url: string): number | null { + const { concurrentRequestLimit } = hostRules.find({ url }); + return is.number(concurrentRequestLimit) && concurrentRequestLimit > 0 + ? concurrentRequestLimit + : null; +} + +export function getThrottleIntervalMs(url: string): number | null { + const { maxRequestsPerSecond } = hostRules.find({ url }); + return is.number(maxRequestsPerSecond) && maxRequestsPerSecond > 0 + ? Math.ceil(1000 / maxRequestsPerSecond) + : null; } diff --git a/lib/util/http/index.spec.ts b/lib/util/http/index.spec.ts index dcd4f8165d3e9ef6c3d951231c690229c792a0fe..00f964975f18a26dcb8c10d92907eea9bf718c08 100644 --- a/lib/util/http/index.spec.ts +++ b/lib/util/http/index.spec.ts @@ -9,6 +9,7 @@ import * as memCache from '../cache/memory'; import * as hostRules from '../host-rules'; import { reportErrors } from '../schema'; import * as queue from './queue'; +import * as throttle from './throttle'; import type { HttpResponse } from './types'; import { Http } from '.'; @@ -21,6 +22,7 @@ describe('util/http/index', () => { http = new Http('dummy'); hostRules.clear(); queue.clear(); + throttle.clear(); }); it('get', async () => { @@ -433,4 +435,36 @@ describe('util/http/index', () => { }); }); }); + + describe('Throttling', () => { + afterEach(() => { + jest.useRealTimers(); + }); + + it('works without throttling', async () => { + jest.useFakeTimers({ advanceTimers: 1 }); + httpMock.scope(baseUrl).get('/foo').twice().reply(200, 'bar'); + + const t1 = Date.now(); + await http.get('http://renovate.com/foo'); + await http.get('http://renovate.com/foo'); + const t2 = Date.now(); + + expect(t2 - t1).toBeLessThan(100); + }); + + it('limits request rate by host', async () => { + jest.useFakeTimers({ advanceTimers: true }); + httpMock.scope(baseUrl).get('/foo').twice().reply(200, 'bar'); + hostRules.add({ matchHost: 'renovate.com', maxRequestsPerSecond: 0.25 }); + + const t1 = Date.now(); + await http.get('http://renovate.com/foo'); + jest.advanceTimersByTime(4000); + await http.get('http://renovate.com/foo'); + const t2 = Date.now(); + + expect(t2 - t1).toBeGreaterThanOrEqual(4000); + }); + }); }); diff --git a/lib/util/http/index.ts b/lib/util/http/index.ts index d12e6a313c96a7a848f1147941cc9a79d4c91aee..dc774463df55214eec4d0971d028c8d997833827 100644 --- a/lib/util/http/index.ts +++ b/lib/util/http/index.ts @@ -1,5 +1,5 @@ import merge from 'deepmerge'; -import got, { Options, RequestError, Response } from 'got'; +import got, { Options, RequestError } from 'got'; import hasha from 'hasha'; import { infer as Infer, ZodSchema } from 'zod'; import { HOST_DISABLED } from '../../constants/error-messages'; @@ -14,6 +14,7 @@ import { applyAuthorization, removeAuthorization } from './auth'; import { hooks } from './hooks'; import { applyHostRules } from './host-rules'; import { getQueue } from './queue'; +import { getThrottle } from './throttle'; import type { GotJSONOptions, GotOptions, @@ -33,6 +34,8 @@ type JsonArgs<T extends HttpOptions> = { schema?: ZodSchema | undefined; }; +type Task<T> = () => Promise<HttpResponse<T>>; + function cloneResponse<T extends Buffer | string | any>( response: HttpResponse<T> ): HttpResponse<T> { @@ -66,7 +69,7 @@ async function gotTask<T>( url: string, options: GotOptions, requestStats: Omit<RequestStats, 'duration' | 'statusCode'> -): Promise<Response<T>> { +): Promise<HttpResponse<T>> { logger.trace({ url, options }, 'got request'); let duration = 0; @@ -155,7 +158,7 @@ export class Http<Opts extends HttpOptions = HttpOptions> { method: options.method, }), ]); - let resPromise; + let resPromise: Promise<HttpResponse<T>> | null = null; // Cache GET requests unless useCache=false if ( @@ -168,7 +171,7 @@ export class Http<Opts extends HttpOptions = HttpOptions> { // istanbul ignore else: no cache tests if (!resPromise) { const startTime = Date.now(); - const httpTask = (): Promise<Response<T>> => { + const httpTask: Task<T> = () => { const queueDuration = Date.now() - startTime; return gotTask(url, options, { method: options.method ?? 'get', @@ -177,11 +180,16 @@ export class Http<Opts extends HttpOptions = HttpOptions> { }); }; - const queue = getQueue(url); - const queuedTask = queue - ? () => queue.add<Response<T>>(httpTask) + const throttle = getThrottle(url); + const throttledTask: Task<T> = throttle + ? () => throttle.add<HttpResponse<T>>(httpTask) : httpTask; + const queue = getQueue(url); + const queuedTask: Task<T> = queue + ? () => queue.add<HttpResponse<T>>(throttledTask) + : throttledTask; + resPromise = queuedTask(); if (options.method === 'get' || options.method === 'head') { diff --git a/lib/util/http/queue.spec.ts b/lib/util/http/queue.spec.ts index b3a20cb6bc483426b344c89290c25cc995cbd649..69069522393daa303faee2c623f36249a842ae7d 100644 --- a/lib/util/http/queue.spec.ts +++ b/lib/util/http/queue.spec.ts @@ -1,15 +1,14 @@ -import { mocked } from '../../../test/util'; -import * as _hostRules from './host-rules'; +import * as hostRules from '../host-rules'; import { clear, getQueue } from './queue'; -jest.mock('./host-rules'); - -const hostRules = mocked(_hostRules); - describe('util/http/queue', () => { beforeEach(() => { - hostRules.getRequestLimit.mockReturnValue(143); clear(); + hostRules.clear(); + hostRules.add({ + matchHost: 'https://example.com', + concurrentRequestLimit: 143, + }); }); it('returns null for invalid URL', () => { diff --git a/lib/util/http/queue.ts b/lib/util/http/queue.ts index 731fecd1acd2d23ffbaa38b6527f33ccbe124dad..f53abb175493b15c0a99b7c6b7aa9a7972207c57 100644 --- a/lib/util/http/queue.ts +++ b/lib/util/http/queue.ts @@ -1,7 +1,7 @@ import PQueue from 'p-queue'; import { logger } from '../../logger'; import { parseUrl } from '../url'; -import { getRequestLimit } from './host-rules'; +import { getConcurrentRequestsLimit } from './host-rules'; const hostQueues = new Map<string, PQueue | null>(); @@ -16,7 +16,7 @@ export function getQueue(url: string): PQueue | null { let queue = hostQueues.get(host); if (queue === undefined) { queue = null; // null represents "no queue", as opposed to undefined - const concurrency = getRequestLimit(url); + const concurrency = getConcurrentRequestsLimit(url); if (concurrency) { logger.debug(`Using queue: host=${host}, concurrency=${concurrency}`); queue = new PQueue({ concurrency }); diff --git a/lib/util/http/throttle.spec.ts b/lib/util/http/throttle.spec.ts new file mode 100644 index 0000000000000000000000000000000000000000..5163f23defb177e9d1f6405ca456221beefe1e89 --- /dev/null +++ b/lib/util/http/throttle.spec.ts @@ -0,0 +1,36 @@ +import * as hostRules from '../host-rules'; +import { clear, getThrottle } from './throttle'; + +describe('util/http/throttle', () => { + beforeEach(() => { + clear(); + hostRules.clear(); + hostRules.add({ + matchHost: 'https://example.com', + maxRequestsPerSecond: 143, + }); + }); + + it('returns null for invalid URL', () => { + expect(getThrottle('$#@!')).toBeNull(); + }); + + it('returns throttle for valid url', () => { + const t1a = getThrottle('https://example.com'); + const t1b = getThrottle('https://example.com'); + + const t2a = getThrottle('https://example.com:8080'); + const t2b = getThrottle('https://example.com:8080'); + + expect(t1a).not.toBeNull(); + expect(t1a).toBe(t1b); + + expect(t2a).not.toBeNull(); + expect(t2a).toBe(t2b); + + expect(t1a).not.toBe(t2a); + expect(t1a).not.toBe(t2b); + expect(t1b).not.toBe(t2a); + expect(t1b).not.toBe(t2b); + }); +}); diff --git a/lib/util/http/throttle.ts b/lib/util/http/throttle.ts new file mode 100644 index 0000000000000000000000000000000000000000..5cee1569317c97eb23fca730ac13c71eec012654 --- /dev/null +++ b/lib/util/http/throttle.ts @@ -0,0 +1,52 @@ +import pThrottle from 'p-throttle'; +import { logger } from '../../logger'; +import { parseUrl } from '../url'; +import { getThrottleIntervalMs } from './host-rules'; + +const hostThrottles = new Map<string, Throttle | null>(); + +class Throttle { + private throttle: ReturnType<typeof pThrottle>; + + constructor(interval: number) { + this.throttle = pThrottle({ + strict: true, + limit: 1, + interval, + }); + } + + add<T>(task: () => Promise<T>): Promise<T> { + const throttledTask = this.throttle(task); + return throttledTask(); + } +} + +export function getThrottle(url: string): Throttle | null { + const host = parseUrl(url)?.host; + if (!host) { + // should never happen + logger.debug({ url }, 'No host'); + return null; + } + + let throttle = hostThrottles.get(host); + if (throttle === undefined) { + throttle = null; // null represents "no throttle", as opposed to undefined + const throttleOptions = getThrottleIntervalMs(url); + if (throttleOptions) { + const intervalMs = throttleOptions; + logger.debug({ intervalMs, host }, 'Using throttle'); + throttle = new Throttle(intervalMs); + } else { + logger.debug({ host }, 'No throttle'); + } + } + hostThrottles.set(host, throttle); + + return throttle; +} + +export function clear(): void { + hostThrottles.clear(); +} diff --git a/lib/workers/global/index.ts b/lib/workers/global/index.ts index 88919445eefc3083a2f64b6e4bda2becf38812cb..4b6b80a8ef02d1fa3563119a95a8e13c51680f6f 100644 --- a/lib/workers/global/index.ts +++ b/lib/workers/global/index.ts @@ -19,6 +19,7 @@ import { instrument } from '../../instrumentation'; import { getProblems, logger, setMeta } from '../../logger'; import * as hostRules from '../../util/host-rules'; import * as queue from '../../util/http/queue'; +import * as throttle from '../../util/http/throttle'; import * as repositoryWorker from '../repository'; import { autodiscoverRepositories } from './autodiscover'; import { parseConfigs } from './config/parse'; @@ -167,6 +168,7 @@ export async function start(): Promise<number> { // host rules can change concurrency queue.clear(); + throttle.clear(); await repositoryWorker.renovateRepository(repoConfig); setMeta({}); diff --git a/lib/workers/repository/index.ts b/lib/workers/repository/index.ts index d59d4bab40f6af725e8aae19dc093be8f1daf170..8bcb684080cd97c6e8f9370e4e4331da8b5402d0 100644 --- a/lib/workers/repository/index.ts +++ b/lib/workers/repository/index.ts @@ -10,6 +10,7 @@ import { deleteLocalFile, privateCacheDir } from '../../util/fs'; import { isCloned } from '../../util/git'; import { clearDnsCache, printDnsStats } from '../../util/http/dns'; import * as queue from '../../util/http/queue'; +import * as throttle from '../../util/http/throttle'; import * as schemaUtil from '../../util/schema'; import { addSplit, getSplits, splitInit } from '../../util/split'; import { setBranchCache } from './cache'; @@ -37,6 +38,7 @@ export async function renovateRepository( logger.trace({ config }); let repoResult: ProcessResult | undefined; queue.clear(); + throttle.clear(); const localDir = GlobalConfig.get('localDir')!; try { await fs.ensureDir(localDir); diff --git a/lib/workers/repository/init/merge.ts b/lib/workers/repository/init/merge.ts index f1211ea1f388e28f501cc7718d30308b60d503c3..cf4dd8106b9b55723173d73c0209e90637dac754 100644 --- a/lib/workers/repository/init/merge.ts +++ b/lib/workers/repository/init/merge.ts @@ -22,6 +22,7 @@ import { readLocalFile } from '../../../util/fs'; import { getFileList } from '../../../util/git'; import * as hostRules from '../../../util/host-rules'; import * as queue from '../../../util/http/queue'; +import * as throttle from '../../../util/http/throttle'; import type { RepoFileConfig } from './types'; async function detectConfigFile(): Promise<string | null> { @@ -262,6 +263,7 @@ export async function mergeRenovateConfig( } // host rules can change concurrency queue.clear(); + throttle.clear(); delete resolvedConfig.hostRules; } returnConfig = mergeChildConfig(returnConfig, resolvedConfig); diff --git a/package.json b/package.json index e6874a7a1f8a871e1cc6522ae53cc49fc0ff3f8e..811c647336e73b27a91876b084557408fd25a900 100644 --- a/package.json +++ b/package.json @@ -214,6 +214,7 @@ "p-all": "3.0.0", "p-map": "4.0.0", "p-queue": "6.6.2", + "p-throttle": "4.1.1", "parse-link-header": "2.0.0", "prettier": "2.7.1", "quick-lru": "5.1.1", diff --git a/yarn.lock b/yarn.lock index aec126b9294bbb9c7e367e672484544fe0014e1d..4a2f82cba7a37dbb8bbd5cf4c74d6ddd9549fa69 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8005,6 +8005,11 @@ p-retry@^4.0.0: "@types/retry" "0.12.0" retry "^0.13.1" +p-throttle@4.1.1: + version "4.1.1" + resolved "https://registry.yarnpkg.com/p-throttle/-/p-throttle-4.1.1.tgz#80b1fbd358af40a8bfa1667f9dc8b72b714ad692" + integrity sha512-TuU8Ato+pRTPJoDzYD4s7ocJYcNSEZRvlxoq3hcPI2kZDZ49IQ1Wkj7/gDJc3X7XiEAAvRGtDzdXJI0tC3IL1g== + p-timeout@^3.2.0: version "3.2.0" resolved "https://registry.yarnpkg.com/p-timeout/-/p-timeout-3.2.0.tgz#c7e17abc971d2a7962ef83626b35d635acf23dfe"