feat: centralised cache (#2472)

Closes #2466 

lib/.eslintrc.js

@@ -2,5 +2,6 @@ module.exports = {
   globals: {
     logger: true,
     platform: true,
+    renovateCache: true,
   },
 };

lib/datasource/github.js

-const cacache = require('cacache/en');
-const os = require('os');
-const { DateTime } = require('luxon');
-
 const ghGot = require('../platform/github/gh-got-wrapper');
 const versioning = require('../versioning');
 
 module.exports = {
   getPreset,
   getPkgReleases,
-  rmAllCache,
 };
 
-const datasourceCache =
-  (process.env.RENOVATE_TMPDIR || os.tmpdir()) +
-  '/renovate-gh-datasource-cache-v1';
-
-async function getCachedResult(repo, type) {
-  try {
-    const cacheVal = await cacache.get(datasourceCache, `${repo}-${type}`);
-    const cachedResult = JSON.parse(cacheVal.data.toString());
-    if (cachedResult) {
-      if (DateTime.local() < DateTime.fromISO(cachedResult.expiry)) {
-        logger.debug(
-          { repo, type },
-          'Returning cached github datasource result'
-        );
-        delete cachedResult.expiry;
-        return cachedResult;
-      }
-      // istanbul ignore next
-      logger.debug('Cache expiry');
-    }
-  } catch (err) {
-    logger.debug('Cache miss');
-  }
-  return null;
-}
-
-async function setCachedResult(repo, type, res) {
-  logger.debug({ repo, type }, 'Saving cached github datasource');
-  await cacache.put(
-    datasourceCache,
-    `${repo}-${type}`,
-    JSON.stringify({ ...res, expiry: DateTime.local().plus({ minutes: 10 }) })
-  );
-}
-
-async function rmAllCache() {
-  await cacache.rm.all(datasourceCache);
-}
-
 const map = new Map();
 
 async function getPreset(pkgName, presetName = 'default') {
@@ -75,11 +31,20 @@ async function getPreset(pkgName, presetName = 'default') {
   }
 }
 
+const cacheNamespace = 'datasource-github';
+function getCacheKey(repo, type) {
+  return `${repo}:${type}`;
+}
+
 async function getPkgReleases(purl, config) {
   const { versionScheme } = config || {};
   const { fullname: repo, qualifiers: options } = purl;
+  options.ref = options.ref || 'tags';
   let versions;
-  const cachedResult = await getCachedResult(repo, options.ref);
+  const cachedResult = await renovateCache.get(
+    cacheNamespace,
+    getCacheKey(repo, options.ref)
+  );
   if (cachedResult) {
     return cachedResult;
   }
@@ -117,6 +82,12 @@ async function getPkgReleases(purl, config) {
     version: options.sanitize === 'true' ? isVersion(version) : version,
     gitRef: version,
   }));
-  setCachedResult(repo, options.ref, dependency);
+  const cacheMinutes = 10;
+  await renovateCache.set(
+    cacheNamespace,
+    getCacheKey(repo, options.ref),
+    dependency,
+    cacheMinutes
+  );
   return dependency;
 }

lib/workers/global/cache.js

+const cacache = require('cacache/en');
+const os = require('os');
+const { DateTime } = require('luxon');
+
+module.exports = {
+  init,
+};
+
+function getKey(namespace, key) {
+  return `${namespace}-${key}`;
+}
+
+const renovateCache =
+  (process.env.RENOVATE_TMPDIR || os.tmpdir()) + '/renovate-cache-v1';
+
+async function get(namespace, key) {
+  try {
+    const res = await cacache.get(renovateCache, getKey(namespace, key));
+    const cachedValue = JSON.parse(res.data.toString());
+    if (cachedValue) {
+      if (DateTime.local() < DateTime.fromISO(cachedValue.expiry)) {
+        logger.debug({ namespace, key }, 'Returning cached value');
+        return cachedValue.value;
+      }
+      // istanbul ignore next
+      await rm(namespace, key);
+    }
+  } catch (err) {
+    logger.debug({ namespace, key }, 'Cache miss');
+  }
+  return null;
+}
+
+async function set(namespace, key, value, ttlMinutes) {
+  logger.debug({ namespace, key }, 'Saving cached value');
+  await cacache.put(
+    renovateCache,
+    getKey(namespace, key),
+    JSON.stringify({
+      value,
+      expiry: DateTime.local().plus({ minutes: ttlMinutes }),
+    })
+  );
+}
+
+// istanbul ignore next
+async function rm(namespace, key) {
+  logger.debug({ namespace, key }, 'Removing cache entry');
+  await cacache.rm.entry(renovateCache, getKey(namespace, key));
+}
+
+async function rmAll() {
+  await cacache.rm.all(renovateCache);
+}
+
+function init() {
+  global.renovateCache = global.renovateCache || { get, set, rm, rmAll };
+}

lib/workers/global/index.js

@@ -2,6 +2,7 @@ const is = require('@sindresorhus/is');
 const { initLogger } = require('../../logger');
 const configParser = require('../../config');
 const repositoryWorker = require('../repository');
+const cache = require('./cache');
 
 module.exports = {
   start,
@@ -10,6 +11,7 @@ module.exports = {
 
 async function start() {
   initLogger();
+  cache.init();
   try {
     const config = await configParser.parseConfigs(process.env, process.argv);
     delete process.env.GITHUB_TOKEN;

lib/workers/pr/changelog/index.js

 const versioning = require('../../../versioning');
 const { addReleaseNotes } = require('../release-notes');
 
-const sourceCache = require('./source-cache');
 const sourceGithub = require('./source-github');
 
 module.exports = {
   getChangeLogJSON,
 };
 
+const cacheNamespace = 'changelog';
+function getCacheKey({
+  versionScheme,
+  fromVersion,
+  toVersion,
+  repositoryUrl,
+  releases,
+}) {
+  return `${repositoryUrl}-${versionScheme}-${fromVersion}-${toVersion}-${
+    releases ? releases.map(release => release.version).join('-') : ''
+  }`;
+}
+
 async function getChangeLogJSON(args) {
   const { repositoryUrl, versionScheme, fromVersion, toVersion } = args;
   if (!repositoryUrl) {
@@ -20,16 +32,24 @@ async function getChangeLogJSON(args) {
   if (!fromVersion || equals(fromVersion, toVersion)) {
     return null;
   }
-  const cachedResult = await sourceCache.getChangeLogJSON(args);
+  const cachedResult = await renovateCache.get(
+    cacheNamespace,
+    getCacheKey(args)
+  );
   if (cachedResult) {
-    logger.debug('Returning cached changelog');
     return cachedResult;
   }
 
   try {
     const res = await sourceGithub.getChangeLogJSON({ ...args });
     const output = await addReleaseNotes(res);
-    await sourceCache.setChangeLogJSON(args, output);
+    const cacheMinutes = 60;
+    await renovateCache.set(
+      cacheNamespace,
+      getCacheKey(args),
+      output,
+      cacheMinutes
+    );
     return output;
   } catch (err) /* istanbul ignore next */ {
     logger.error(

lib/workers/pr/changelog/source-cache.js

-const os = require('os');
-const cacache = require('cacache/en');
-const { DateTime } = require('luxon');
-
-module.exports = {
-  getChangeLogJSON,
-  setChangeLogJSON,
-  rmAllCache,
-};
-
-function getCache({ manager, depName, fromVersion, toVersion, releases }) {
-  const tmpdir = process.env.RENOVATE_TMPDIR || os.tmpdir();
-  const cachePath = tmpdir + '/renovate-cache-changelog-v3';
-  const cacheKey = `${manager}-${depName}-${fromVersion}-${toVersion}-${
-    releases ? releases.map(release => release.version).join('-') : ''
-  }`;
-  return [cachePath, cacheKey];
-}
-
-async function getChangeLogJSON(args) {
-  const cache = getCache(args);
-  const { depName } = args;
-  try {
-    const cacheVal = await cacache.get(...cache);
-    logger.trace(`Returning cached version of ${depName}`);
-    const cachedResult = JSON.parse(cacheVal.data.toString());
-    if (cachedResult) {
-      if (DateTime.local() < DateTime.fromISO(cachedResult.expiry)) {
-        logger.debug('Cache hit');
-        delete cachedResult.expiry;
-        return cachedResult;
-      }
-      // istanbul ignore next
-      logger.debug('Cache expiry');
-    }
-  } catch (err) {
-    logger.debug('Cache miss');
-  }
-  return null;
-}
-
-async function setChangeLogJSON(args, res) {
-  const cache = getCache(args);
-  await cacache.put(
-    ...cache,
-    JSON.stringify({ ...res, expiry: DateTime.local().plus({ hours: 1 }) })
-  );
-}
-
-async function rmAllCache() {
-  const cache = getCache({});
-  await cacache.rm.all(cache[0]);
-}

test/datasource/github.spec.js

@@ -39,7 +39,7 @@ describe('datasource/github', () => {
     });
   });
   describe('getPkgReleases', () => {
-    beforeAll(() => github.rmAllCache());
+    beforeAll(() => global.renovateCache.rmAll());
     it('returns cleaned tags', async () => {
       const body = [
         { name: 'a' },

test/globals.js

 jest.mock('gh-got');
 jest.mock('gl-got');
 
+const cache = require('../lib/workers/global/cache');
+
 global.platform = jest.genMockFromModule('../lib/platform/github');
 global.logger = require('./_fixtures/logger');
+
+cache.init();

test/workers/pr/changelog.spec.js

@@ -6,9 +6,6 @@ const endpoints = require('../../../lib/util/endpoints');
 const ghGot = require('../../../lib/platform/github/gh-got-wrapper');
 
 const { getChangeLogJSON } = require('../../../lib/workers/pr/changelog');
-const {
-  rmAllCache,
-} = require('../../../lib/workers/pr/changelog/source-cache');
 
 const upgrade = {
   depName: 'renovate',
@@ -39,7 +36,7 @@ describe('workers/pr/changelog', () => {
         platform: 'github',
         endpoint: 'https://api.github.com/',
       });
-      await rmAllCache();
+      await global.renovateCache.rmAll();
     });
     it('returns null if no fromVersion', async () => {
       expect(