.dockerignore

@@ -3,3 +3,4 @@
 !yarn.lock
 !hooks
 !lib
+!bin/yarn*

.eslintrc.js

@@ -9,6 +9,7 @@ module.exports = {
     'no-use-before-define': 0,
     'no-restricted-syntax': 0,
     'no-await-in-loop': 0,
+    'prefer-destructuring': 'off',
     'prefer-template': 'off',
     'promise/always-return': 'error',
     'promise/no-return-wrap': 'error',

.gitattributes

-* text=auto
-*.js text eol=lf
+* text=auto eol=lf

.github/ISSUE_TEMPLATE/3-Bug_report.md

@@ -9,6 +9,9 @@ Renovate CLI, Renovate GitHub App, or Renovate Pro?
 **Describe the bug**
 A clear and concise description of what the bug is.
 
+**Did you see anything helpful in debug logs?**
+If you are using the Renovate App, log into https://renovatebot.com/dashboard and locate the correct job log for when the problem occurred (e.g. when the PR was created).
+
 **To Reproduce**
 Steps to reproduce the behavior:
 

.github/issue_template.md

@@ -19,6 +19,6 @@
 - [ ] The Renovate GitHub App
 - [ ] Self-hosted GitHub
 - [ ] Self-hosted GitLab
-- [ ] Self-hosted VSTS
+- [ ] Self-hosted Azure DevOps
 
 #### Please describe the issue:

.gitignore

@@ -9,3 +9,5 @@
 /.vscode
 /.idea
 package-lock.json
+
+*.pyc

.prettierignore

@@ -10,3 +10,4 @@
 /*.log
 package.json
 test/_fixtures/
+bin/yarn*

.prettierrc

+singleQuote: true
+trailingComma: "es5"

.travis.yml

@@ -4,7 +4,7 @@ notifications:
 language: node_js
 
 node_js:
-  - "8.11.1"
+  - '10.15.0'
 
 sudo: false
 
@@ -12,22 +12,25 @@ branches:
   only:
     - master
     - stable
+    - v13
 
 if: tag IS blank
 
 before_install:
   - curl -o- -L https://yarnpkg.com/install.sh | bash
   - export PATH="$HOME/.yarn/bin:$PATH"
+  - python --version
 
 install:
   - yarn install --frozen-lockfile
+  - pip install --user -r requirements.txt
 
 cache:
   yarn: true
   directories:
-    - ".cache"
-    - ".eslintcache"
-    - "node_modules"
+    - '.cache'
+    - '.eslintcache'
+    - 'node_modules'
 
 script:
   - yarn prettier --list-different

Dockerfile

-FROM node:8.11.4-alpine@sha256:cf4ea9156ef964eaf0c4df65da3f4fed7358dbe31149ca105c7684a5858195d8
+FROM amd64/ubuntu:18.04@sha256:eb5d7eda6804359e4fc5223a31a2d9caa4c8ea590b14060d81c8bc05b22ca04e
 
 LABEL maintainer="Rhys Arkins <rhys@arkins.net>"
 LABEL name="renovate"
 
 WORKDIR /usr/src/app/
 
-RUN apk add --quiet --no-cache git openssh-client
+ENV DEBIAN_FRONTEND noninteractive
+ENV LC_ALL C.UTF-8
+ENV LANG C.UTF-8
+
+RUN apt-get update && apt-get install -y gpg curl wget unzip xz-utils git openssh-client && apt-get clean -y
+
+## Node.js
+
+# START copy Node.js from https://github.com/nodejs/docker-node/blob/master/10/jessie/Dockerfile
+
+ENV NODE_VERSION 10.15.0
+
+RUN ARCH= && dpkgArch="$(dpkg --print-architecture)" \
+  && case "${dpkgArch##*-}" in \
+    amd64) ARCH='x64';; \
+    ppc64el) ARCH='ppc64le';; \
+    s390x) ARCH='s390x';; \
+    arm64) ARCH='arm64';; \
+    armhf) ARCH='armv7l';; \
+    i386) ARCH='x86';; \
+    *) echo "unsupported architecture"; exit 1 ;; \
+  esac \
+  # gpg keys listed at https://github.com/nodejs/node#release-keys
+  && set -ex \
+  && for key in \
+    94AE36675C464D64BAFA68DD7434390BDBE9B9C5 \
+    FD3A5288F042B6850C66B31F09FE44734EB7990E \
+    71DCFD284A79C3B38668286BC97EC7A07EDE3FC1 \
+    DD8F2338BAE7501E3DD5AC78C273792F7D83545D \
+    C4F0DFFF4E8C1A8236409D08E73BC641CC11F4C8 \
+    B9AE9905FFD7803F25714661B63B535A4C206CA9 \
+    77984A986EBC2AA786BC0F66B01FBB92821C587A \
+    8FCCA13FEF1D0C2E91008E09770F7A9A5AE15600 \
+    4ED778F539E3634C779C87C6D7062848A1AB005C \
+    A48C2BEE680E841632CD4E44F07496B3EB3C1762 \
+    B9E2F5981AA6E0CD28160D9FF13993A75599653C \
+  ; do \
+    gpg --batch --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys "$key" || \
+    gpg --batch --keyserver hkp://ipv4.pool.sks-keyservers.net --recv-keys "$key" || \
+    gpg --batch --keyserver hkp://pgp.mit.edu:80 --recv-keys "$key" ; \
+  done \
+  && curl -fsSLO --compressed "https://nodejs.org/dist/v$NODE_VERSION/node-v$NODE_VERSION-linux-$ARCH.tar.xz" \
+  && curl -fsSLO --compressed "https://nodejs.org/dist/v$NODE_VERSION/SHASUMS256.txt.asc" \
+  && gpg --batch --decrypt --output SHASUMS256.txt SHASUMS256.txt.asc \
+  && grep " node-v$NODE_VERSION-linux-$ARCH.tar.xz\$" SHASUMS256.txt | sha256sum -c - \
+  && tar -xJf "node-v$NODE_VERSION-linux-$ARCH.tar.xz" -C /usr/local --strip-components=1 --no-same-owner \
+  && rm "node-v$NODE_VERSION-linux-$ARCH.tar.xz" SHASUMS256.txt.asc SHASUMS256.txt \
+  && ln -s /usr/local/bin/node /usr/local/bin/nodejs
+
+## END copy Node.js
+
+# PHP Composer
+
+RUN apt-get update && apt-get install -y php-cli php-mbstring && apt-get clean
+
+ENV COMPOSER_VERSION=1.7.2
+
+RUN php -r "copy('https://github.com/composer/composer/releases/download/$COMPOSER_VERSION/composer.phar', '/usr/local/bin/composer');"
+
+RUN chmod +x /usr/local/bin/composer
+
+# Go Modules
+
+ENV GOLANG_VERSION 1.11.1
+
+RUN wget -q -O go.tgz "https://golang.org/dl/go${GOLANG_VERSION}.linux-amd64.tar.gz" && \
+	tar -C /usr/local -xzf go.tgz && \
+	rm go.tgz && \
+	export PATH="/usr/local/go/bin:$PATH"
+
+ENV GOPATH /go
+ENV PATH $GOPATH/bin:/usr/local/go/bin:$PATH
+
+RUN mkdir -p "$GOPATH/src" "$GOPATH/bin" && chmod -R 777 "$GOPATH"
+
+ENV CGO_ENABLED=0
+
+# Set up ubuntu user
+
+RUN groupadd --gid 1000 ubuntu \
+  && useradd --uid 1000 --gid ubuntu --shell /bin/bash --create-home ubuntu
+
+RUN chmod -R a+rw /usr
+
+USER ubuntu
+
+ENV YARN_VERSION=1.13.0
+
+RUN npm i -g yarn@${YARN_VERSION}
+
 COPY package.json .
 COPY yarn.lock .
 RUN yarn install --production && yarn cache clean
 COPY lib lib
-USER node
+COPY bin bin
 
 ENTRYPOINT ["node", "/usr/src/app/lib/renovate.js"]
 CMD ["--help"]

Procfile

-renovate: yarn start
-web: node bin/heroku/web.js

bin/clean-cache.js

@@ -2,6 +2,5 @@ const fs = require('fs-extra');
 const os = require('os');
 
 (async () => {
-  await fs.remove(os.tmpdir() + '/renovate-cache-changelog-v3');
-  await fs.remove(os.tmpdir() + '/renovate-npm-cache');
+  await fs.remove(os.tmpdir() + '/renovate');
 })();

bin/config-validator.js

@@ -4,6 +4,7 @@ const fs = require('fs-extra');
 const { validateConfig } = require('../lib/config/validation');
 const { massageConfig } = require('../lib/config/massage');
 const { initLogger } = require('../lib/logger');
+const { configFileNames } = require('../lib/config/app-strings');
 
 initLogger();
 
@@ -28,12 +29,7 @@ async function validate(desc, config, isPreset = false) {
 }
 
 (async () => {
-  const renovateConfigFiles = [
-    'renovate.json',
-    '.renovaterc',
-    '.renovaterc.json',
-  ];
-  for (const file of renovateConfigFiles) {
+  for (const file of configFileNames.filter(name => name !== 'package.json')) {
     try {
       const rawContent = fs.readFileSync(file, 'utf8');
       console.log(`Validating ${file}`);

bin/heroku/web.js

-/* eslint-disable no-console */
-const http = require('http');
-
-const port = process.env.PORT || '3000';
-
-const requestHandler = (request, response) => {
-  // Redirect users to Heroku dashboard
-  const appName = request.headers.host.split(':')[0].split('.')[0];
-  response.writeHead(302, {
-    Location: `https://dashboard.heroku.com/apps/${appName}/logs`,
-  });
-  response.end();
-};
-
-http.createServer(requestHandler).listen(port, err => {
-  if (err) {
-    console.log('Failed to start web server', err);
-    return;
-  }
-  console.log(`Web server is listening on ${port}`);
-});

docs/adding-a-package-manager.md

@@ -21,16 +21,16 @@ Each manager needs its own subdirectory under `lib/managers` and to be added to
 
 The manager's `index.js` file supports the following values/functions:
 
-- extractDependencies
+- extractPackageFile
+- extractAllPackageFiles
 - getRangeStrategy (optional)
 - language (optional)
-- postExtract (optional)
 - supportsLockFileMaintenance (optional)
 - updateDependency
 
-##### `extractDependencies(content, packageFile, config)` (async, mandatory)
+##### `extractPackageFile(content, packageFile, config)` (async, semi-mandatory)
 
-This function is mandatory. It takes a file content and optionally the packageFile name/config, and returns an array of detected/extracted dependencies, including:
+This function is mandatory unless you use `extractAllPackageFiles` instead. It takes as arguments the file's content and optionally the file's full file pathname and config, and returns an array of detected/extracted dependencies, including:
 
 - dependency name
 - dependency type (e.g. dependencies, devDependencies, etc)
@@ -41,9 +41,19 @@ The fields returned here can be customised to suit the package manager, e.g. Doc
 
 This function doesn't necessarily need to _understand_ the file or even syntax that it is passed, instead it just needs to understand enough to extract the list of dependencies.
 
-As a general approach, we want to extract _all_ dependencies from each dependency file, even if they contain values we don't support. For any that have unsupported values that we cannot renovate, this `extractDependencies` function should set a `skipReason` to a value that would be helpful to someone reading the logs.
+As a general approach, we want to extract _all_ dependencies from each dependency file, even if they contain values we don't support. For any that have unsupported values that we cannot renovate, this `extractPackageFile` function should set a `skipReason` to a value that would be helpful to someone reading the logs.
 
-Also, if a file is passed to `extractDependencies` that is a "false match" (e.g. not an actual package file, or contains no dependencies) then this function can return `null` to have it ignored and removed from the list of package files. A common case for this is in Meteor, where its `package.js` file name is not unique and there many be many non-Meteor paojects using that filename.
+Also, if a file is passed to `extractPackageFile` that is a "false match" (e.g. not an actual package file, or contains no dependencies) then this function can return `null` to have it ignored and removed from the list of package files. A common case for this is in Meteor, where its `package.js` file name is not unique and there many be many non-Meteor projects using that filename.
+
+##### `extractAllPackageFiles(packageFiles)` (async, optional)
+
+You can use this function instead of `extractPackageFile` if the package manager cannot parse/extract all package files in parallel.
+
+For example, npm/yarn needs to correlate package files together for features such as Lerna and Workspaces, so it's necessary to iterate through them all together after initial parsing.
+
+As another example, gradle needs to write out all files and call a command via child process in order to extract dependencies, so that must be done first.
+
+This function takes an array of filenames as input and returns an array of filenames and dependencies as a result.
 
 #### `getRangeStrategy(config)` (optional)
 
@@ -55,14 +65,6 @@ If left undefined, then a default `getRangeStrategy` will be used that always re
 
 This is used when more than one package manager share settings from a common language. e.g. docker-compose, circleci and gitlabci all specify "docker" as their language and inherit all config settings from there.
 
-#### `postExtract(packageFiles)` (async, optional)
-
-This function takes an array of package files (extracted earlier using `extractDependencies`) and is useful if some form of "correlation" is required between the files.
-
-For example, Yarn Workspaces and Lerna are tools for working with multiple package files at once, including generating a single lock file instead of one per package file. It is therefore necessary to have a "full view" of all package files to determine if such logic is necessary, because the `extractDependencies` function only sees each package file in isolation.
-
-Currently `npm` is the only package manager using this function, because all other ones are able to extract enough data from package files in isolation.
-
 #### `supportsLockFileMaintenance` (optional)
 
 Set to true if this package manager needs to update lock files in addition to package files.

docs/branches-commits.md

@@ -22,6 +22,6 @@ For GitHub, we use the low-level `git`-based API to manually make the commits an
 
 In GitLab, Merge Request are not automatically closed if you delete the associated branch, so that gives us more flexibility. Therefore the way we update a branch is simply to delete it and then create the branch + commits again, and GitLab immediately reassociates the new (single) commit with the existing PR.
 
-#### VSTS
+#### Azure DevOps
 
-VSTS is implemented similarly to GitLab.
+Azure DevOps is implemented similarly to GitLab.

docs/configuration.md

@@ -7,7 +7,7 @@ Configuration is supported via any or all of the below:
 - Configuration file
 - Environment
 - CLI
-- `renovate.json`, `.renovaterc.json`, or `.renovaterc` in target repository
+- `renovate.json`, `.github/renovate.json`, `.renovaterc.json`, or `.renovaterc` in target repository
 - `renovate` field of `package.json` in target repository
 
 The above are listed in reverse order of preference. i.e. `package.json`

docs/deployment.md

@@ -18,52 +18,3 @@ would be to specify both via CLI. Alternatively, configure the token via
 Environment Variable if you don't want it to show in any cron logs.
 
 Running daily should suit most people. At most, hourly.
-
-## Heroku
-
-Heroku free dynos provide a good way to host this for free. Set it up with the
-following commands:
-
-### Installation
-
-The best way to deploy to Heroku is via git and Heroku CLI.
-
-```
-$ git clone https://github.com/renovatebot/renovate
-$ cd renovate
-$ heroku create [app name]
-$ git push heroku master
-```
-
-### Configuration
-
-You now need to set the token.
-
-```
-$ heroku config:set GITHUB_TOKEN=[YourGitHubToken]
-```
-
-(or use `GITLAB_TOKEN` if appropriate)
-
-You should also set any other [Configuration Options](configuration.md) you
-need.
-
-The app should now be ready for testing.
-
-```
-$ heroku run renovate [your/repo]
-```
-
-Once you've verified the script ran successfully, it's time to set it up for
-automatic scheduling.
-
-```
-$ heroku addons:create scheduler:standard
-$ heroku addons:open scheduler
-```
-
-At this point you should have the Heroku Scheduler Dashboard open. Click "Add
-new job" and enter the same command as you ran previously (e.g. `renovate [your/repo]`). Adjust the frequency to hourly if you prefer, then click Save.
-
-You can run `heroku logs` to check execution logs. Consider adjusting the
-scripts log level if you have problems (info -> verbose -> debug -> silly).

docs/local-development.md

@@ -7,7 +7,7 @@ Please submit PRs to improve it if you think anything is unclear or you can thin
 
 #### Fork and Clone
 
-If you will contribute to the project, you should first "fork" it using the GitHub Website and then clone your fork.
+If you will contribute to the project, you should first "fork" the main project using the GitHub Website and then clone your fork locally.
 
 #### Node version
 
@@ -27,37 +27,41 @@ FATAL: Renovate fatal error: You need to supply a GitHub token.
 
 ## Platform Account Setup
 
-Although it's possible to make small changes without testing against a real repository, in most cases it's important that you run a "real" test on a repository before you submit a feature or fix.
-It's possible to do this against GitHub or GitLab public hosts, and you can also use both.
+Although it's possible to make small source code improvements without testing against a real repository, in most cases it's important that you run a "real" test on a repository before you submit a feature or fix. It's possible to do this against GitHub, GitLab or Bitbucket public servers.
 
 #### Register new account (optional)
 
-It's recommended that you set up a dedicated test account on GitHub or GitLab, so that you minimise the risk that you accidentally cause problems when testing out Renovate.
+If you're going to be doing a lot of Renovate development then it's recommended that you set up a dedicated test account on GitHub or GitLab, so that you reduce the risk that you accidentally cause problems when testing out Renovate.
+
 e.g. if your GitHub username is "alex88" then maybe you register "alex88-testing" for use with Renovate.
 
 #### Generate platform token
 
-Once you have decided on your platform and account, log in and generate a "Personal Access Token" that can be used to authenticate Renovate.
+Once you have decided on your platform and account, log in and [generate a "Personal Access Token"](https://help.github.com/articles/creating-a-personal-access-token-for-the-command-line/) that can be used to authenticate Renovate.
 
 #### Export platform token
 
-Although you can specify a token to Renovate using `--token=`, it is annoying if you need to include this every time.
-You are better off to instead export an Environment Variable for this.
-If your platform of choice is GitHub, then export GITHUB_TOKEN, and if it's GitLab then export GITLAB_TOKEN.
-It's also find to export both so that you can switch between platforms.
+Although you can specify a token to Renovate using `--token=`, it can be inconvenient if you need to include this every time.
+You are better off to instead export the Environment Variable `RENOVATE_TOKEN` for this.
+
+#### Run against a real repo
+
+To make sure everything is working, create a test repo in your account, e.g. like `https://github.com/r4harry/testrepo1`. Now, add a file called `.nvmrc` with the content `8.13.0`. Now run against the test repo you created, e.g. `yarn start r4harry/testrepo1`. If your token is set up correctly, you should find that it added a "Configure Renovate" PR inside the repo.
+
+If this is working then in future you can create other test repos to verify your code changes against.
 
 ## Tests
 
-You can run `yarn test` locally to test your code. We test all PRs using the same tests, run on TravisCI. `yarn test` runs an `eslint` check, a `prettier check`, and then all the unit tests using `jest`.
+You can run `yarn test` locally to test your code. We test all PRs using the same tests, run on TravisCI. `yarn test` runs an `eslint` check, a `prettier` check, and then all the unit tests using `jest`.
 
 ## Jest
 
-You can run just the Jest unit tests by running `yarn jest`. You can also run just a subset of the Jest tests using file matching, e.g. `yarn jest lock-files` or `yarn jest workers/branch`. If you get a test failure due to a "snapshot" mismatch, and you are sure that you need to update the snapshot, then you can append `-u` to the end. e.g. `yarn jest lock-files -u` would update the saved Snapshots for _all_ tests in `test/workers/branch/lock-files.spec.js`.
+You can run just the Jest unit tests by running `yarn jest`. You can also run just a subset of the Jest tests using file matching, e.g. `yarn jest composer` or `yarn jest workers/branch`. If you get a test failure due to a "snapshot" mismatch, and you are sure that you need to update the snapshot, then you can append `-u` to the end. e.g. `yarn jest composer -u` would update the saved Snapshots for _all_ tests in `test/manager/composer/*`.
 
 #### Coverage
 
 The Renovate project maintains 100% test coverage, so any Pull Request will fail if it does not contain full coverage for code.
-Using `// instanbul ignore` is not ideal but sometimes is a pragmatic solution if an additional test wouldn't really prove anything.
+Using `// istanbul ignore` is not ideal but sometimes is a pragmatic solution if an additional test wouldn't really prove anything.
 
 To view the current test coverage locally, open up `coverage/lcov-report/index.html` in your browser.
 
@@ -66,18 +70,27 @@ Also, it can be good to submit your PR as a work in progress (WIP) without tests
 
 #### Linting and formatting
 
-We use [Prettier](https://github.com/prettier/prettier) for code formatting. If
-your code fails `yarn test` due to a `prettier` rule then you should find that the offending file will be updated automatically and pass the second time you run `yarn test` because each time you run it, it includes the `--fix` command automatically. You usually shouldn't need to fix any prettier errors manually.
+We use [Prettier](https://github.com/prettier/prettier) for code formatting. If your code fails `yarn test` due to a `prettier` rule then run `yarn lint-fix` to fix it or most `eslint` errors automatically before running `yarn test` again. You usually shouldn't need to fix any prettier errors manually.
+
+## Keeping your Renovate fork up to date
+
+First of all, never commit to `master` of your fork - always use a branch like `feat/1234-add-yarn-parsing`.
+
+Then, make sure your fork is up to date with `master` each time before creating a new branch. To do this, see these GitHub guides:
+
+[Configuring a remote for a fork](https://help.github.com/articles/configuring-a-remote-for-a-fork/)
+
+[Syncing a fork](https://help.github.com/articles/syncing-a-fork/)
 
 ## Tips and tricks
 
-#### Forked repositories
+#### Runnign Renovate against forked repositories
 
 Quite often, the quickest way for you to test or fix something is to fork an existing repository.
 However, by default Renovate skips over repositories that are forked.
-To override this default, you need to specify the setting `renovateFork` as `true`.
+To override this default, you need to specify the setting `includeForks` as `true`.
 
-Option 1: Add `"renovateFork": true` to the `renovate.json` of the repository
+Option 1: Add `"includeForks": true` to the `renovate.json` of the repository
 Option 2: Run Renovate with the CLI flag `--renovate-fork=true`
 
 #### Log files
@@ -99,3 +112,12 @@ the code configurable, so most new functionality should be controllable via
 configuration options.
 
 If you wish to add one, add it to `lib/config/definitions.js` and then add documentation to `website/docs/_posts/2017-10-05-configuration-options.md`.
+
+## Debugging
+
+It's really easy to debug Renovate using Chrome's inspect tool. Try like this:
+
+1. Open `chrome://inspect` in Chrome, then click on "Open dedicated DevTools for Node"
+2. Add a `debugger;` statement somewhere in the source code where you want to start debugging
+3. Run Renovate using `yarn debug ...` instead of `yarn start ...`
+4. Click "Resume script execution" in Chrome DevTools and wait for your break point to be triggered