diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000000000000000000000000000000000000..cc13f1f2d744e896a64129648dc79fc3223943f3 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,6 @@ +# 1.0.0 (2024-04-09) + + +### Features + +* initial version ([df08495](https://gitlab.com/to-be-continuous/docker-compose/commit/df0849597058a7594093f7ce2e2d4d302891205b)) diff --git a/README.md b/README.md index b1a4c8de647dc87adbbd58c961bfce98c6729227..517909d26b67893fce6e0523c85bb8e982d49d52 100644 --- a/README.md +++ b/README.md @@ -19,9 +19,9 @@ include: inputs: # ⚠ this is only an example base-app-name: wonderapp - review-project: "prj-12345" # enable review env - staging-project: "prj-12345" # enable staging env - prod-project: "prj-12345" # enable production env + review-docker-host: "ssh://user@192.168.64.5" # enable review env + staging-docker-host: "ssh://user@192.168.64.6" # enable staging env + prod-docker-host: "ssh://user@192.168.64.7" # enable production env ``` ### Use as a CI/CD template (legacy) @@ -39,9 +39,9 @@ variables: # 2: set/override template variables # ⚠ this is only an example DCMP_BASE_APP_NAME: wonderapp - DCMP_REVIEW_PROJECT: "prj-12345" # enable review env - DCMP_STAGING_PROJECT: "prj-12345" # enable staging env - DCMP_PROD_PROJECT: "prj-12345" # enable production env + DCMP_REVIEW_DOCKER_HOST: "ssh://user@192.168.64.5" # enable review env + DCMP_STAGING_DOCKER_HOST: "ssh://user@192.168.64.6" # enable staging env + DCMP_PROD_DOCKER_HOST: "ssh://user@192.168.64.7" # enable production env ``` ## Understand @@ -93,18 +93,22 @@ You're free to enable whichever or both, and you can also choose your deployment ### Supported authentication methods -The Docker Compose template supports the following authentication methods: +The Docker Compose template supports [deployment on remote Docker hosts](https://www.docker.com/blog/how-to-deploy-on-remote-docker-hosts-with-docker-compose/), using +dedicated variables: -* TODO (document) +* `DCMP_REVIEW_DOCKER_HOST`, `DCMP_INTEG_DOCKER_HOST`, `DCMP_STAGING_DOCKER_HOST` and `DCMP_PROD_DOCKER_HOST` to both enable and configure the target Docker host for each environment (ex: `ssh://user@192.168.64.5`), +* :lock: `DCMP_SSH_PRIVATE_KEY`, :lock: `DCMP_REVIEW_SSH_PRIVATE_KEY`, :lock: `DCMP_INTEG_SSH_PRIVATE_KEY`, :lock: `DCMP_STAGING_SSH_PRIVATE_KEY` and :lock: `DCMP_PROD_SSH_PRIVATE_KEY` to provide the global or per env SSH private key (in case SSH authentication is required). ### Deployment context variables In order to manage the various deployment environments, this template provides a couple of **dynamic variables** -that you might use in your hook scripts, deployment manifests and other deployment resources: +that you might use in your hook scripts or [Docker Compose files](https://docs.docker.com/compose/compose-application-model/#the-compose-file): * `${environment_type}`: the current deployment environment type (`review`, `integration`, `staging` or `production`) * `${environment_name}`: a generated application name to use for the current deployment environment (ex: `myapp-review-fix-bug-12` or `myapp-staging`) - _details below_ +> :information_source: the `${environment_name}` is used by the Docker Compose template as the [Docker Compose project name](https://docs.docker.com/compose/project-name/). + #### Generated environment name The `${environment_name}` variable is generated to designate each deployment environment with a unique and meaningful application name. @@ -112,13 +116,13 @@ By construction, it is suitable for inclusion in DNS, URLs, Kubernetes labels... It is built from: * the application _base name_ (defaults to `$CI_PROJECT_NAME` but can be overridden globally and/or per deployment environment - _see configuration variables_) -* GitLab predefined `$CI_ENVIRONMENT_SLUG` variable ([sluggified](https://en.wikipedia.org/wiki/Clean_URL#Slug) name, truncated to 24 characters) +* GitLab predefined `$CI_ENVIRONMENT_SLUG` variable ([slugified](https://en.wikipedia.org/wiki/Clean_URL#Slug) name, truncated to 24 characters) The `${environment_name}` variable is then evaluated as: * `<app base name>` for the production environment * `<app base name>-$CI_ENVIRONMENT_SLUG` for all other deployment environments -* :bulb: `${environment_name}` can also be overriden per environment with the appropriate configuration variable +* :bulb: `${environment_name}` can also be overridden per environment with the appropriate configuration variable Examples (with an application's base name `myapp`): @@ -131,38 +135,116 @@ Examples (with an application's base name `myapp`): ### Deployment and cleanup -> TODO: explain here the supported techniques to deploy and cleanup the environments. +The Docker Compose template requires you to provide: + +* a _required_ [Compose file](https://docs.docker.com/compose/compose-application-model/#the-compose-file) that implements your application deployment and cleanup, +* _optional_ [`.override.` Compose files](https://docs.docker.com/compose/multiple-compose-files/merge/) defining common and/or per-env override, +* _optional_ [dotenv files](https://docs.docker.com/compose/environment-variables/env-file/) defining common and/or per-env configuration, +* _optional_ hook scripts (shell) to implement logic that can't be implemented with Docker Compose. + +#### Compose files lookup strategy + +Unless you have explicitly set the [`COMPOSE_FILES`](https://docs.docker.com/compose/environment-variables/envvars/#compose_file) variable, the Docker Compose template will handle it and +implement the following [Compose file(s)](https://docs.docker.com/compose/compose-application-model/#the-compose-file) lookup strategy: + +| Lookup order | Compose file | Additional `.override.` files (optional) | +|:------------:|--------------|------------------------------------------------------------------------------------------------------------------------------------------| +| 1. | **environment specific** (`compose-${environment_type}.yaml`) | **environment specific** override (`compose-${environment_type}.override.yaml`) | +| 2. | **default** (`compose.yaml`) | 1. **default** override (`compose.override.yaml`)<br/>2. **environment specific** override (`compose-${environment_type}.override.yaml`) | + +> :information_source: Important: > -> You should also explained clearly what is expected from the template user and what is the lookup policy in case the template -> implements one. +> * Compose file base name `docker-compose` is also supported as an alternative to `compose`, +> * Compose file extension `.yml` is also supported as an alternative to `.yaml`, +> * `.override.` files must match the same base name (`compose` or `docker-compose`) and extension (`yaml` or `yml`) as the found Compose file. + +Examples with different combinations of files: + +<table> + <thead> + <tr> + <th>Files in your project</th> + <th>Compose files for Review</th> + <th>Compose files for Integration</th> + <th>Compose files for Staging</th> + <th>Compose files for Production</th> + </tr> + </thead> + <tfoot> + <tr> + <td>- <code>compose.yaml</code></td> + <td colspan="4" align=center><code>compose.yaml</code></td> + </tr> + <tr> + <td>- <code>compose.yaml</code><br/>- <code>compose-production.yaml</code></td> + <td colspan="3" align=center><code>compose.yaml</code></td> + <td align=center><code>compose-production.yaml</code></td> + </tr> + <tr> + <td>- <code>docker-compose.yml</code><br/>- <code>docker-compose-production.override.yml</code></td> + <td colspan="3" align=center><code>docker-compose.yml</code></td> + <td align=center><code>docker-compose.yml</code> + <code>docker-compose-production.override.yml</code> (merged)</td> + </tr> + <tr> + <td>- <code>compose.yml</code><br/>- <code>compose.override.yml</code><br/>- <code>compose-review.override.yml</code><br/>- <code>compose-production.yml</code></td> + <td align=center><code>compose.yml</code> + <code>compose.override.yml</code> + <code>compose-review.override.yml</code> (merged)</td> + <td colspan="2" align=center><code>compose.yml</code>+ <code>compose.override.yml</code> (merged)</td> + <td align=center><code>compose-production.yml</code></td> + </tr> + </tfoot> + <tbody> + </tbody> +</table> + +#### Dotenv files lookup strategy + +Unless you have explicitly set the [`COMPOSE_ENV_FILES`](https://docs.docker.com/compose/environment-variables/envvars/#compose_env_files) variable, the Docker Compose template will handle it +and implement the following [dotenv files](https://docs.docker.com/compose/environment-variables/env-file/) lookup strategy: + + 1. a `.env` file defining defaults for all environments, + 2. a `${environment_type}.env` file that might redefine or override defaults for a specific environment (e.g. `staging.env`). + +#### Deployment + +The **deployment** is processed as follows by the template: + +1. _optionally_ executes the `pre-compose-up.sh` script found in your project to perform pre-deployment stuff (for e.g. create required services), +2. runs [`docker-compose up`](https://docs.docker.com/reference/cli/docker/compose/up/), +3. _optionally_ executes the `post-compose-up.sh` script found in your project to perform post-deployment stuff, + +> :information_source: Important: > -> Example: +> * Compose files, dotenv files and hook scripts are searched in the `$DCMP_SCRIPTS_DIR` directory (configurable), +> * Hook scripts need to be executable; you can add the execution flag with `git update-index --chmod=+x pre-compose-up.sh`. -The Docker Compose template requires you to provide a shell script that fully implements your application -deployment and cleanup using the `docker-compose` CLI and all other tools available in the selected Docker image. +#### Cleanup -The deployment script is searched as follows: +The **cleanup** is processed as follows by the template: -1. look for a specific `dcmp-deploy-$environment_type.sh` in the `$DCMP_SCRIPTS_DIR` directory in your project (e.g. `dcmp-deploy-staging.sh` for staging environment), -2. if not found: look for a default `dcmp-deploy.sh` in the `$DCMP_SCRIPTS_DIR` directory in your project, -3. if not found: the deployment job will fail. +1. _optionally_ executes the `pre-compose-down.sh` script found in your project to perform pre-cleanup stuff, +2. runs [`docker-compose down`](https://docs.docker.com/reference/cli/docker/compose/down/), +3. _optionally_ executes the `post-compose-down.sh` script found in your project to perform post-cleanup stuff, -The cleanup script is searched as follows: +> :information_source: Important: +> +> * Compose files, dotenv files and hook scripts are searched in the `$DCMP_SCRIPTS_DIR` directory (configurable), +> * Hook scripts need to be executable; you can add the execution flag with `git update-index --chmod=+x pre-compose-up.sh`. -1. look for a specific `dcmp-cleanup-$environment_type.sh` in the `$DCMP_SCRIPTS_DIR` directory in your project (e.g. `dcmp-cleanup-staging.sh` for staging environment), -2. if not found: look for a default `dcmp-cleanup.sh` in the `$DCMP_SCRIPTS_DIR` directory in your project, -3. if not found: the cleanup job will fail. +#### Using Variables -> :information_source: Your deployment (and cleanup) scripts have to be able to cope with various environments, each with different application names, exposed routes, settings, ... -> Part of this complexity can be handled by the lookup policies described above (ex: one script per env) and also by using available environment variables: -> -> 1. [deployment context variables](#deployment-context-variables) provided by the template: -> * `${environment_type}`: the current environment type (`review`, `integration`, `staging` or `production`) -> * `${environment_name}`: the application name to use for the current environment (ex: `myproject-review-fix-bug-12` or `myproject-staging`) -> * `${hostname}`: the environment hostname, extracted from the current environment url (after late variable expansion - see below) -> 2. any [GitLab CI variable](https://docs.gitlab.com/ee/ci/variables/predefined_variables.html) -> 3. any [custom variable](https://docs.gitlab.com/ee/ci/variables/#add-a-cicd-variable-to-a-project) -> (ex: `${SECRET_TOKEN}` that you have set in your project CI/CD variables) +Your deployment (and cleanup) scripts have to be able to cope with various environments, each with different application names, exposed routes, settings, ... +Part of this complexity can be handled by the lookup strategies described above (ex: one file per env) and also by using available environment variables: + +1. [deployment context variables](#deployment-context-variables) provided by the template: + * `${environment_type}`: the current environment type (`review`, `integration`, `staging` or `production`) + * `${environment_name}`: the application name to use for the current environment (ex: `myproject-review-fix-bug-12` or `myproject-staging`) + * `${hostname}`: the environment hostname, extracted from the current environment url (after late variable expansion - see below) +2. any [GitLab CI variable](https://docs.gitlab.com/ee/ci/variables/predefined_variables.html) +3. any [custom variable](https://docs.gitlab.com/ee/ci/variables/#add-a-cicd-variable-to-a-docker-host) + (ex: `${SECRET_TOKEN}` that you have set in your project CI/CD variables) + +Be aware that environment variables may be freely used and substituted in [dotenv files](https://docs.docker.com/compose/environment-variables/env-file/) +using the appropriate [interpolation syntax](https://docs.docker.com/compose/environment-variables/env-file/#interpolation). ### Environments URL management @@ -193,7 +275,7 @@ The **static way** can be implemented simply by setting the appropriate configur > DCMP_REVIEW_ENVIRONMENT_URL: "https://wonderapp-review.nonprod.acme.domain/%{environment_name}" > ``` -To implement the **dynamic way**, your deployment script shall simply generate a `environment_url.txt` file in the working directory, containing only +To implement the **dynamic way**, your post deployment hook script shall simply generate a `environment_url.txt` file in the working directory, containing only the dynamically generated url. When detected by the template, it will use it as the newly deployed environment url. ### Deployment output variables @@ -206,7 +288,7 @@ Each deployment job produces _output variables_ that are propagated to downstrea Those variables may be freely used in downstream jobs (for instance to run acceptance tests against the latest deployed environment). -You may also add and propagate your own custom variables, by pushing them to the `docker-compose.env` file in your [deployment scripts or hooks](#deployment-and-cleanup). +You may also add and propagate your own custom variables, by pushing them to the `docker-compose.out.env` file in your [deployment scripts or hooks](#deployment-and-cleanup). ## Configuration reference @@ -214,7 +296,7 @@ You may also add and propagate your own custom variables, by pushing them to the Here are some advices about your **secrets** (variables marked with a :lock:): -1. Manage them as [project or group CI/CD variables](https://docs.gitlab.com/ee/ci/variables/#add-a-cicd-variable-to-a-project): +1. Manage them as [project or group CI/CD variables](https://docs.gitlab.com/ee/ci/variables/#add-a-cicd-variable-to-a-docker-host): * [**masked**](https://docs.gitlab.com/ee/ci/variables/#mask-a-cicd-variable) to prevent them from being inadvertently displayed in your job logs, * [**protected**](https://docs.gitlab.com/ee/ci/variables/#protected-cicd-variables) if you want to secure some secrets @@ -230,45 +312,46 @@ The Docker Compose template uses some global configuration used throughout all j | Input / Variable | Description | Default value | | ------------------------ | -------------------------------------- | ----------------- | -| `image` / `DCMP_IMAGE` | the Docker image used to run Docker Compose CLI commands | `registry.hub.docker.com/docker-compose:latest` | -| `base-app-name` / `DCMP_BASE_APP_NAME` | Base application name | `$CI_PROJECT_NAME` ([see GitLab doc](https://docs.gitlab.com/ee/ci/variables/predefined_variables.html)) | -| `api-url` / `DCMP_API_URL` | Default Docker Compose API url | _none_ | -| :lock: `DCMP_API_TOKEN` | Default Docker Compose API token | _none_ | -| `environment-url` / `DCMP_ENVIRONMENT_URL` | Default environments url _(only define for static environment URLs declaration)_<br/>_supports late variable expansion (ex: `https://%{environment_name}.docker-compose.acme.com`)_ | _none_ | -| `scripts-dir` / `DCMP_SCRIPTS_DIR` | Directory where deploy & cleanup scripts are located | `.` _(root project dir)_ | +| `image` / `DCMP_IMAGE` | the Docker image used to run Docker Compose CLI commands | `registry.hub.docker.com/library/docker:latest` | +| `cmd` / `DCMP_CMD` | The docker compose command (`docker compose` or `docker-compose`) | _none_ (auto) | +| `base-app-name` / `DCMP_BASE_APP_NAME`| Base application name | `$CI_PROJECT_NAME` ([see GitLab doc](https://docs.gitlab.com/ee/ci/variables/predefined_variables.html)) | +| `environment-url` / `DCMP_ENVIRONMENT_URL`| Default environments url _(only define for static environment URLs declaration)_<br/>_supports late variable expansion (ex: `https://%{environment_name}.docker-compose.acme.com`)_ | _none_ | +| `scripts-dir` / `DCMP_SCRIPTS_DIR`| Directory where Compose files, dotenv files and hook scripts are located | `.` _(root project dir)_ | +| `up-opts` / `DCMP_UP_OPTS` | [`compose up` options](https://docs.docker.com/reference/cli/docker/compose/up/#options) | `--no-build --remove-orphans --wait --wait-timeout 180` | +| `down-opts`/ `DCMP_DOWN_OPTS` | [`compose down` options](https://docs.docker.com/reference/cli/docker/compose/down/#options) | `--volumes --remove-orphans --rmi all` | +| :lock: `DCMP_SSH_PRIVATE_KEY` | Default SSH key to use when connecting to Docker hosts over SSH (can be overridden per env) | _none_ | +| `ssh-known-hosts` / `DCMP_SSH_KNOWN_HOSTS` | SSH `known_hosts` (file or text variable) | _none_ | ### Review environments configuration Review environments are dynamic and ephemeral environments to deploy your _ongoing developments_ (a.k.a. _feature_ or _topic_ branches). -They are **disabled by default** and can be enabled by setting the `DCMP_REVIEW_PROJECT` variable (see below). +They are **disabled by default** and can be enabled by setting the `DCMP_REVIEW_DOCKER_HOST` variable (see below). Here are variables supported to configure review environments: -| Input / Variable | Description | Default value | -| ------------------------ | -------------------------------------- | ----------------- | -| `review-project` / `DCMP_REVIEW_PROJECT` | Project ID for `review` env | _none_ (disabled) | -| `review-app-name` / `DCMP_REVIEW_APP_NAME` | Application name for `review` env | `"${DCMP_BASE_APP_NAME}-${CI_ENVIRONMENT_SLUG}"` (ex: `myproject-review-fix-bug-12`) | -| `review-api-url` / `DCMP_REVIEW_API_URL` | API url for `review` env _(only define to override default)_ | `$DCMP_API_URL` | -| :lock: `DCMP_REVIEW_API_TOKEN` | API token for `review` env _(only define to override default)_ | `$DCMP_API_TOKEN` | +| Input / Variable | Description | Default value | +| ------------------------ |-------------------------------------------------------------------------------------------------------------------| ----------------- | +| `review-docker-host` / `DCMP_REVIEW_DOCKER_HOST` | Docker Host for `review` env (ex: `ssh://user@docker-host-for-review`) | _none_ (disabled) | +| :lock: `DCMP_REVIEW_SSH_PRIVATE_KEY` | `review` env specific SSH key to use when connecting to Docker Host over SSH | `$DCMP_SSH_PRIVATE_KEY` | +| `review-app-name` / `DCMP_REVIEW_APP_NAME` | Application name for `review` env | `"${DCMP_BASE_APP_NAME}-${CI_ENVIRONMENT_SLUG}"` (ex: `myproject-review-fix-bug-12`) | | `review-environment-url` / `DCMP_REVIEW_ENVIRONMENT_URL`| The review environments url _(only define for static environment URLs declaration and if different from default)_ | `$DCMP_ENVIRONMENT_URL` | -| `review-autostop-duration` / `DCMP_REVIEW_AUTOSTOP_DURATION`| The amount of time before GitLab will automatically stop `review` environments | `4 hours` | +| `review-autostop-duration` / `DCMP_REVIEW_AUTOSTOP_DURATION`| The amount of time before GitLab will automatically stop `review` environments | `4 hours` | ### Integration environment configuration The integration environment is the environment associated to your integration branch (`develop` by default). -It is **disabled by default** and can be enabled by setting the `DCMP_INTEG_PROJECT` variable (see below). +It is **disabled by default** and can be enabled by setting the `DCMP_INTEG_DOCKER_HOST` variable (see below). Here are variables supported to configure the integration environment: -| Input / Variable | Description | Default value | -| ------------------------ | -------------------------------------- | ----------------- | -| `integ-project` / `DCMP_INTEG_PROJECT` | Project ID for `integration` env | _none_ (disabled) | -| `integ-app-name` / `DCMP_INTEG_APP_NAME` | Application name for `integration` env | `${DCMP_BASE_APP_NAME}-integration` | -| `integ-api-url` / `DCMP_INTEG_API_URL` | API url for `integration` env _(only define to override default)_ | `$DCMP_API_URL` | -| :lock: `DCMP_INTEG_API_TOKEN` | API token for `integration` env _(only define to override default)_ | `$DCMP_API_TOKEN` | +| Input / Variable | Description | Default value | +| ------------------------ |----------------------------------------------------------------------------------------------------------------------| ----------------- | +| `integ-docker-host` / `DCMP_INTEG_DOCKER_HOST` | Docker Host for `integration` env (ex: `ssh://user@docker-host-for-integ`) | _none_ (disabled) | +| :lock: `DCMP_INTEG_SSH_PRIVATE_KEY` | `integration` env specific SSH key to use when connecting to Docker Host over SSH | `$DCMP_SSH_PRIVATE_KEY` | +| `integ-app-name` / `DCMP_INTEG_APP_NAME` | Application name for `integration` env | `${DCMP_BASE_APP_NAME}-integration` | | `integ-environment-url` / `DCMP_INTEG_ENVIRONMENT_URL`| The integration environment url _(only define for static environment URLs declaration and if different from default)_ | `$DCMP_ENVIRONMENT_URL` | ### Staging environment configuration @@ -276,31 +359,43 @@ Here are variables supported to configure the integration environment: The staging environment is an iso-prod environment meant for testing and validation purpose associated to your production branch (`main` or `master` by default). -It is **disabled by default** and can be enabled by setting the `DCMP_STAGING_PROJECT` variable (see below). +It is **disabled by default** and can be enabled by setting the `DCMP_STAGING_DOCKER_HOST` variable (see below). Here are variables supported to configure the staging environment: -| Input / Variable | Description | Default value | -| ------------------------ | -------------------------------------- | ----------------- | -| `staging-project` / `DCMP_STAGING_PROJECT` | Project ID for `staging` env | _none_ (disabled) | -| `staging-app-name` / `DCMP_STAGING_APP_NAME` | Application name for `staging` env | `${DCMP_BASE_APP_NAME}-staging` | -| `staging-api-url` / `DCMP_STAGING_API_URL` | API url for `staging` env _(only define to override default)_ | `$DCMP_API_URL` | -| :lock: `DCMP_STAGING_API_TOKEN` | API token for `staging` env _(only define to override default)_ | `$DCMP_API_TOKEN` | +| Input / Variable | Description | Default value | +| ------------------------ |-------------------------------------------------------------------------------------------------------------------| ----------------- | +| `staging-docker-host` / `DCMP_STAGING_DOCKER_HOST` | Docker Host for `staging` env (ex: `ssh://user@docker-host-for-staging`) | _none_ (disabled) | +| :lock: `DCMP_STAGING_SSH_PRIVATE_KEY` | `staging` env specific SSH key to use when connecting to Docker Host over SSH | `$DCMP_SSH_PRIVATE_KEY` | +| `staging-app-name` / `DCMP_STAGING_APP_NAME` | Application name for `staging` env | `${DCMP_BASE_APP_NAME}-staging` | | `staging-environment-url` / `DCMP_STAGING_ENVIRONMENT_URL`| The staging environment url _(only define for static environment URLs declaration and if different from default)_ | `$DCMP_ENVIRONMENT_URL` | ### Production environment configuration The production environment is the final deployment environment associated with your production branch (`main` or `master` by default). -It is **disabled by default** and can be enabled by setting the `DCMP_PROD_PROJECT` variable (see below). +It is **disabled by default** and can be enabled by setting the `DCMP_PROD_DOCKER_HOST` variable (see below). Here are variables supported to configure the production environment: -| Input / Variable | Description | Default value | -| ------------------------- | -------------------------------------- | ----------------- | -| `prod-project` / `DCMP_PROD_PROJECT` | Project ID for `production` env | _none_ (disabled) | -| `prod-app-name` / `DCMP_PROD_APP_NAME` | Application name for `production` env | `$DCMP_BASE_APP_NAME` | -| `prod-api-url` / `DCMP_PROD_API_URL` | API url for `production` env _(only define to override default)_ | `$DCMP_API_URL` | -| :lock: `DCMP_PROD_API_TOKEN` | API token for `production` env _(only define to override default)_ | `$DCMP_API_TOKEN` | +| Input / Variable | Description | Default value | +| ------------------------- |---------------------------------------------------------------------------------------------------------------------| ----------------- | +| `prod-docker-host` / `DCMP_PROD_DOCKER_HOST` | Docker Host for `production` env (ex: `ssh://user@docker-host-for-prod`) | _none_ (disabled) | +| :lock: `DCMP_PROD_SSH_PRIVATE_KEY` | `production` env specific SSH key to use when connecting to Docker Host over SSH | `$DCMP_SSH_PRIVATE_KEY` | +| `prod-app-name` / `DCMP_PROD_APP_NAME` | Application name for `production` env | `$DCMP_BASE_APP_NAME` | | `prod-environment-url` / `DCMP_PROD_ENVIRONMENT_URL`| The production environment url _(only define for static environment URLs declaration and if different from default)_ | `$DCMP_ENVIRONMENT_URL` | -| `prod-deploy-strategy` / `DCMP_PROD_DEPLOY_STRATEGY`| Defines the deployment to production strategy. One of `manual` (i.e. _one-click_) or `auto`. | `manual` | +| `prod-deploy-strategy` / `DCMP_PROD_DEPLOY_STRATEGY`| Defines the deployment to production strategy. One of `manual` (i.e. _one-click_) or `auto`. | `manual` | + +### Compose Config job + +The Docker Compose template enables running [Compose Config](https://docs.docker.com/reference/cli/docker/compose/config/), thus enabling detection of syntax errors in your Compose or dotenv files. + +This job is mapped to the `package-test` stage and is **active** by default. + +Here are its parameters: + +| Input / Variable | Description | Default value | +| ----------------------- | ----------------------------------------- | ----------------------------- | +| `config-disabled` / `DCMP_CONFIG_DISABLED` | Set to `true` to disable `compose config` | _none_ (enabled) | +| `config-opts` / `DCMP_CONFIG_OPTS` | [`compose config` options](https://docs.docker.com/reference/cli/docker/compose/config/#options) | `--quiet` _(to avoid displaying secrets inadvertently)_ | + diff --git a/bumpversion.sh b/bumpversion.sh index ed44d7b68b0e09f6d2cf557f7a15e52553246341..329e866dac988c049574a0a9f26ba89979c523a8 100755 --- a/bumpversion.sh +++ b/bumpversion.sh @@ -33,7 +33,7 @@ if [[ "$curVer" ]]; then # replace in template and variants for tmpl in templates/*.yml do - sed -e "s/command: *\[\"--service\", \"\(.*\)\", \"$curVer\"\]/command: [\"--service\", \"\1\", \"$nextVer\"]/" "$tmpl" > "$tmpl.next" + sed -e "s/command: *\[ *\"--service\", *\"\(.*\)\", *\"$curVer\" *\]/command: [\"--service\", \"\1\", \"$nextVer\"]/" "$tmpl" > "$tmpl.next" mv -f "$tmpl.next" "$tmpl" done else diff --git a/kicker.json b/kicker.json index 7a722a527dfa81e95c9b8d985e45d027fe2840f8..b6371492fb6332688224e9be7c9cb3b27ccaa30f 100644 --- a/kicker.json +++ b/kicker.json @@ -10,17 +10,13 @@ { "name": "DCMP_IMAGE", "description": "The Docker image used to run Docker Compose CLI commands - **set the version required by your Docker Compose cluster**", - "default": "registry.hub.docker.com/docker-compose:latest" + "default": "registry.hub.docker.com/library/docker:latest" }, { - "name": "DCMP_API_URL", - "type": "url", - "description": "Default Docker Compose API url" - }, - { - "name": "DCMP_API_TOKEN", - "description": "Default Docker Compose API token", - "secret": true + "name": "DCMP_CMD", + "description": "The docker compose command (empty means _auto_)", + "values": ["", "docker compose", "docker-compose"], + "advanced": true }, { "name": "DCMP_BASE_APP_NAME", @@ -35,22 +31,60 @@ }, { "name": "DCMP_SCRIPTS_DIR", - "description": "Directory where deploy & cleanup scripts are located", + "description": "Directory where Compose files, dotenv files and hook scripts are located", "default": ".", "advanced": true + }, + { + "name": "DCMP_UP_OPTS", + "description": "[`compose up` options](https://docs.docker.com/reference/cli/docker/compose/up/#options)", + "default": "--no-build --remove-orphans --wait --wait-timeout 180" + }, + { + "name": "DCMP_DOWN_OPTS", + "description": "[`compose down` options](https://docs.docker.com/reference/cli/docker/compose/down/#options)", + "default": "--volumes --remove-orphans --rmi all" + }, + { + "name": "DCMP_SSH_PRIVATE_KEY", + "description": "Default SSH key to use when connecting to Docker hosts over SSH (can be overridden per env)", + "secret": true + }, + { + "name": "DCMP_SSH_KNOWN_HOSTS", + "description": "SSH `known_hosts` (file or text variable)" } - ], +], "features": [ + { + "id": "config", + "name": "Compose Config", + "description": "Runs [`compose config`](https://docs.docker.com/reference/cli/docker/compose/config/) to detect errors in your Compose file(s)", + "disable_with": "DCMP_CONFIG_DISABLED", + "variables": [ + { + "name": "DCMP_CONFIG_OPTS", + "description": "[`compose config` options](https://docs.docker.com/reference/cli/docker/compose/config/#options)", + "default": "--quiet", + "advanced": true + } + ] + }, { "id": "review", "name": "Review", "description": "Dynamic review environments for your topic branches (see GitLab [Review Apps](https://docs.gitlab.com/ee/ci/review_apps/))", "variables": [ { - "name": "DCMP_REVIEW_PROJECT", - "description": "Project ID for `review` env", + "name": "DCMP_REVIEW_DOCKER_HOST", + "description": "Docker Host for `review` env (ex: `ssh://docker@docker-host-for-review:2375`)", "mandatory": true }, + { + "name": "DCMP_REVIEW_SSH_PRIVATE_KEY", + "description": "`review` env specific SSH key to use when connecting to Docker Host over SSH", + "secret": true + }, { "name": "DCMP_REVIEW_APP_NAME", "description": "The application name for `review` env (only define to override default)", @@ -66,17 +100,6 @@ "type": "url", "description": "The `review` environments url _(only define for static environment URLs declaration and if different from default)_", "advanced": true - }, - { - "name": "DCMP_REVIEW_API_URL", - "type": "url", - "description": "API url for `review` env _(only define to override default)_", - "advanced": true - }, - { - "name": "DCMP_REVIEW_API_TOKEN", - "description": "API token for `review` env (only define to override default)", - "secret": true } ] }, @@ -86,10 +109,15 @@ "description": "A continuous-integration environment associated to your integration branch (`develop` by default)", "variables": [ { - "name": "DCMP_INTEG_PROJECT", - "description": "Project ID for `integration` env", + "name": "DCMP_INTEG_DOCKER_HOST", + "description": "Docker Host for `integration` env (ex: `ssh://docker@docker-host-for-integ:2375`)", "mandatory": true }, + { + "name": "DCMP_INTEG_SSH_PRIVATE_KEY", + "description": "`integration` env specific SSH key to use when connecting to Docker Host over SSH", + "secret": true + }, { "name": "DCMP_INTEG_APP_NAME", "description": "The application name for `integration` env (only define to override default)", @@ -100,17 +128,6 @@ "type": "url", "description": "The `integration` environment url _(only define for static environment URLs declaration and if different from default)_", "advanced": true - }, - { - "name": "DCMP_INTEG_API_URL", - "type": "url", - "description": "API url for `integration` env _(only define to override default)_", - "advanced": true - }, - { - "name": "DCMP_INTEG_API_TOKEN", - "description": "API token for `integration` env (only define to override default)", - "secret": true } ] }, @@ -120,10 +137,15 @@ "description": "An iso-prod environment meant for testing and validation purpose on your production branch (`main` or `master` by default)", "variables": [ { - "name": "DCMP_STAGING_PROJECT", - "description": "Project ID for `staging` env", + "name": "DCMP_STAGING_DOCKER_HOST", + "description": "Docker Host for `staging` env (ex: `ssh://docker@docker-host-for-staging:2375`)", "mandatory": true }, + { + "name": "DCMP_STAGING_SSH_PRIVATE_KEY", + "description": "`staging` env specific SSH key to use when connecting to Docker Host over SSH", + "secret": true + }, { "name": "DCMP_STAGING_APP_NAME", "description": "The application name for `staging` env (only define to override default)", @@ -134,17 +156,6 @@ "type": "url", "description": "The `staging` environment url _(only define for static environment URLs declaration and if different from default)_", "advanced": true - }, - { - "name": "DCMP_STAGING_API_URL", - "type": "url", - "description": "API url for `staging` env _(only define to override default)_", - "advanced": true - }, - { - "name": "DCMP_STAGING_API_TOKEN", - "description": "API token for `staging` env (only define to override default)", - "secret": true } ] }, @@ -154,10 +165,15 @@ "description": "The production environment", "variables": [ { - "name": "DCMP_PROD_PROJECT", - "description": "Project ID for `production` env", + "name": "DCMP_PROD_DOCKER_HOST", + "description": "Docker Host for `production` env (ex: `ssh://docker@docker-host-for-prod:2375`)", "mandatory": true }, + { + "name": "DCMP_PROD_SSH_PRIVATE_KEY", + "description": "`production` env specific SSH key to use when connecting to Docker Host over SSH", + "secret": true + }, { "name": "DCMP_PROD_APP_NAME", "description": "The application name for `production` env (only define to override default)", @@ -175,17 +191,6 @@ "type": "enum", "values": ["manual", "auto"], "default": "manual" - }, - { - "name": "DCMP_PROD_API_URL", - "type": "url", - "description": "API url for `production` env _(only define to override default)_", - "advanced": true - }, - { - "name": "DCMP_PROD_API_TOKEN", - "description": "API token for `production` env (only define to override default)", - "secret": true } ] } diff --git a/templates/gitlab-ci-docker-compose.yml b/templates/gitlab-ci-docker-compose.yml index 37408482f42d9f3aa25612994fe7dae814602e6b..7332d9b01dca76e8b1a815ca67b12fe2800fec91 100644 --- a/templates/gitlab-ci-docker-compose.yml +++ b/templates/gitlab-ci-docker-compose.yml @@ -1,5 +1,5 @@ # ========================================================================================= -# Copyright (C) 2021 Orange & contributors +# Copyright (C) 2024 Pierre Smeyers and contributors # # This program is free software; you can redistribute it and/or modify it under the terms # of the GNU Lesser General Public License as published by the Free Software Foundation; @@ -17,10 +17,14 @@ spec: inputs: image: description: The Docker image used to run Docker Compose CLI commands - **set the version required by your Docker Compose cluster** - default: registry.hub.docker.com/docker-compose:latest - api-url: - description: Default Docker Compose API url + default: registry.hub.docker.com/library/docker:latest + cmd: + description: "The docker compose command (empty means _auto_)" default: '' + options: + - '' + - docker compose + - docker-compose base-app-name: description: Base application name default: $CI_PROJECT_NAME @@ -31,10 +35,26 @@ spec: _supports late variable expansion (ex: `https://%{environment_name}.dcmp.acme.com`)_ default: '' scripts-dir: - description: Directory where deploy & cleanup scripts are located + description: Directory where Compose files, dotenv files and hook scripts are located default: . - review-project: - description: Project ID for `review` env + config-opts: + description: "[`compose config` options](https://docs.docker.com/reference/cli/docker/compose/config/#options)" + default: '--quiet' + config-disabled: + description: Disable Compose Config + type: boolean + default: false + up-opts: + description: "[`compose up` options](https://docs.docker.com/reference/cli/docker/compose/up/#options)" + default: "--no-build --remove-orphans --wait --wait-timeout 180" + down-opts: + description: "[`compose down` options](https://docs.docker.com/reference/cli/docker/compose/down/#options)" + default: "--volumes --remove-orphans --rmi all" + ssh-known-hosts: + description: SSH `known_hosts` (file or text variable) + default: '' + review-docker-host: + description: "Docker Host for `review` env (ex: `ssh://docker@docker-host-for-review:2375`)" default: '' review-app-name: description: The application name for `review` env (only define to override default) @@ -45,11 +65,8 @@ spec: review-environment-url: description: The `review` environments url _(only define for static environment URLs declaration and if different from default)_ default: '' - review-api-url: - description: API url for `review` env _(only define to override default)_ - default: '' - integ-project: - description: Project ID for `integration` env + integ-docker-host: + description: "Docker Host for `integration` env (ex: `ssh://docker@docker-host-for-integ:2375`)" default: '' integ-app-name: description: The application name for `integration` env (only define to override default) @@ -57,11 +74,8 @@ spec: integ-environment-url: description: The `integration` environment url _(only define for static environment URLs declaration and if different from default)_ default: '' - integ-api-url: - description: API url for `integration` env _(only define to override default)_ - default: '' - staging-project: - description: Project ID for `staging` env + staging-docker-host: + description: "Docker Host for `staging` env (ex: `ssh://docker@docker-host-for-staging:2375`)" default: '' staging-app-name: description: The application name for `staging` env (only define to override default) @@ -69,11 +83,8 @@ spec: staging-environment-url: description: The `staging` environment url _(only define for static environment URLs declaration and if different from default)_ default: '' - staging-api-url: - description: API url for `staging` env _(only define to override default)_ - default: '' - prod-project: - description: Project ID for `production` env + prod-docker-host: + description: "Docker Host for `production` env (ex: `ssh://docker@docker-host-for-prod:2375`)" default: '' prod-app-name: description: The application name for `production` env (only define to override default) @@ -81,9 +92,6 @@ spec: prod-environment-url: description: The `production` environment url _(only define for static environment URLs declaration and if different from default)_ default: '' - prod-api-url: - description: API url for `production` env _(only define to override default)_ - default: '' prod-deploy-strategy: description: Defines the deployment to `production` strategy. options: @@ -113,6 +121,24 @@ workflow: when: never - when: always +# test job prototype: implement adaptive pipeline rules +.test-policy: + rules: + # on tag: auto & failing + - if: $CI_COMMIT_TAG + # on ADAPTIVE_PIPELINE_DISABLED: auto & failing + - if: '$ADAPTIVE_PIPELINE_DISABLED == "true"' + # on production or integration branch(es): auto & failing + - if: '$CI_COMMIT_REF_NAME =~ $PROD_REF || $CI_COMMIT_REF_NAME =~ $INTEG_REF' + # early stage (dev branch, no MR): manual & non-failing + - if: '$CI_MERGE_REQUEST_ID == null && $CI_OPEN_MERGE_REQUESTS == null' + when: manual + allow_failure: true + # Draft MR: auto & non-failing + - if: '$CI_MERGE_REQUEST_TITLE =~ /^Draft:.*/' + allow_failure: true + # else (Ready MR): auto & failing + - when: on_success variables: # variabilized tracking image @@ -120,32 +146,33 @@ variables: # Default Docker image (use a public image - can be overridden) DCMP_IMAGE: $[[ inputs.image ]] + DCMP_CMD: $[[ inputs.cmd ]] DCMP_BASE_APP_NAME: $[[ inputs.base-app-name ]] - DCMP_API_URL: $[[ inputs.api-url ]] DCMP_ENVIRONMENT_URL: $[[ inputs.environment-url ]] DCMP_SCRIPTS_DIR: $[[ inputs.scripts-dir ]] + DCMP_SSH_KNOWN_HOSTS: $[[ inputs.ssh-known-hosts ]] + DCMP_CONFIG_OPTS: $[[ inputs.config-opts ]] + DCMP_CONFIG_DISABLED: $[[ inputs.config-disabled ]] + DCMP_UP_OPTS: $[[ inputs.up-opts ]] + DCMP_DOWN_OPTS: $[[ inputs.down-opts ]] - DCMP_REVIEW_PROJECT: $[[ inputs.review-project ]] + DCMP_REVIEW_DOCKER_HOST: $[[ inputs.review-docker-host ]] DCMP_REVIEW_APP_NAME: $[[ inputs.review-app-name ]] DCMP_REVIEW_ENVIRONMENT_URL: $[[ inputs.review-environment-url ]] - DCMP_REVIEW_API_URL: $[[ inputs.review-api-url ]] DCMP_REVIEW_AUTOSTOP_DURATION: $[[ inputs.review-autostop-duration ]] - DCMP_INTEG_PROJECT: $[[ inputs.integ-project ]] + DCMP_INTEG_DOCKER_HOST: $[[ inputs.integ-docker-host ]] DCMP_INTEG_APP_NAME: $[[ inputs.integ-app-name ]] DCMP_INTEG_ENVIRONMENT_URL: $[[ inputs.integ-environment-url ]] - DCMP_INTEG_API_URL: $[[ inputs.integ-api-url ]] - DCMP_STAGING_PROJECT: $[[ inputs.staging-project ]] + DCMP_STAGING_DOCKER_HOST: $[[ inputs.staging-docker-host ]] DCMP_STAGING_APP_NAME: $[[ inputs.staging-app-name ]] DCMP_STAGING_ENVIRONMENT_URL: $[[ inputs.staging-environment-url ]] - DCMP_STAGING_API_URL: $[[ inputs.staging-api-url ]] - DCMP_PROD_PROJECT: $[[ inputs.prod-project ]] + DCMP_PROD_DOCKER_HOST: $[[ inputs.prod-docker-host ]] DCMP_PROD_APP_NAME: $[[ inputs.prod-app-name ]] DCMP_PROD_ENVIRONMENT_URL: $[[ inputs.prod-environment-url ]] - DCMP_PROD_API_URL: $[[ inputs.prod-api-url ]] DCMP_PROD_DEPLOY_STRATEGY: $[[ inputs.prod-deploy-strategy ]] # default production ref name (pattern) @@ -165,7 +192,7 @@ stages: - infra-prod - production -.dcmp-scripts: &dcmp-scripts | +.compose-scripts: &compose-scripts | # BEGSCRIPT set -e @@ -404,28 +431,223 @@ stages: awk '{while(match($0,"[$%]{[^}]*}")) {var=substr($0,RSTART+2,RLENGTH-3);val=ENVIRON[var];gsub("&","\\\\&",val);gsub("[$%]{"var"}",val)}}1' } - # login to the hosting platform - function dcmp_login() { - api_url=${ENV_API_URL:-$DCMP_API_URL} - api_token=${ENV_API_TOKEN:-$DCMP_API_TOKEN} - project=$ENV_PROJECT - - assert_defined "$api_url" 'Missing required API url' - assert_defined "$api_token" 'Missing required API token' - assert_defined "$project" 'Missing required project' - - docker-compose login --url="$api_url" --token="$api_token" --project="$project" + function configure_network() { + # maybe install .netrc + if [[ -f ".netrc" ]] + then + log_info "--- \\e[32m.netrc\\e[0m file found: envsubst and install" + awkenvsubst < .netrc > ~/.netrc + chmod 0600 ~/.netrc + fi + + dh_proto=${DOCKER_HOST%%:*} + if [[ "$dh_proto" == "ssh" ]] + then + # setup SSH + log_info "--- SSH Docker Host detected: configure SSH..." + + # 1: maybe install SSH client + if ! command -v ssh-agent > /dev/null + then + log_info "... install SSH client" + apk add --no-cache openssh-client + fi + + # 2: maybe setup known hosts + if [[ "$DCMP_SSH_KNOWN_HOSTS" ]] + then + mkdir -m 700 ~/.ssh + if [[ -f "$DCMP_SSH_KNOWN_HOSTS" ]] + then + log_info "... add SSH known hosts (file)" + cp -f "$DCMP_SSH_KNOWN_HOSTS" ~/.ssh/known_hosts + else + log_info "... add SSH known hosts (text)" + echo "$DCMP_SSH_KNOWN_HOSTS" > ~/.ssh/known_hosts + fi + chmod 644 ~/.ssh/known_hosts + fi + + # start SSH agent + eval "$(ssh-agent -s)" + + # 3: maybe detect and add SSH private key (file or PEM content) + ssh_priv_key=${ENV_SSH_PRIVATE_KEY:-$DCMP_SSH_PRIVATE_KEY} + if [[ -f "$ssh_priv_key" ]] + then + log_info "... add SSH private key (file)" + tr -d '\r' < "$ssh_priv_key" | ssh-add - + elif [[ "$ssh_priv_key" ]] + then + log_info "... add SSH private key (text)" + echo "$ssh_priv_key" | tr -d '\r' | ssh-add - + else + log_warn "No SSH private key found in configuration" + fi + fi + } + + function configure_registries_auth() { + if [[ -f ".docker/config.json" ]] + then + log_info "--- \\e[32m.docker/config.json\\e[0m file found: envsubst and install" + mkdir -p ~/.docker + # special variable supported + TBC_CI_REGISTRY_TOKEN=$(echo -n "$CI_REGISTRY_USER:$CI_REGISTRY_PASSWORD" | base64 | tr -d '\n') + export TBC_CI_REGISTRY_TOKEN + awkenvsubst < .docker/config.json > ~/.docker/config.json + else + log_info "--- \\e[32m.docker/config.json\\e[0m file not found: looking for TBC built images..." + _image_vars=$(env | awk -F '=' "/^[A-Z]+_(SNAPSHOT|RELEASE)_IMAGE=/ {print \$1}" | sort | uniq) + if [[ -z "$_image_vars" ]] + then + log_info "... no TBC image detected: leave unconfigured Docker authentication" + else + # init Docker config JSON + _docker_cfg_json="{\"auths\":{" + _registry_hosts="," + for _image_var in $_image_vars + do + _image_url=$(eval echo "\$${_image_var}") + _registry_host=$(echo "$_image_url" | cut -d/ -f1) + if [[ "$_registry_host" ]] + then + log_info "... TBC image detected: \\e[33;1m${_image_var}\\e[0m=\\e[33;1m${_image_url}\\e[0m..." + if expr "${_registry_hosts}" : ".*,${_registry_host},.*" >/dev/null + then + log_info "... host \\e[33;1m${_registry_host}\\e[0m already configured: skip" + else + log_info "... add host \\e[33;1m${_registry_host}\\e[0m authentication" + _prefix=$(echo "$_image_var" | cut -d_ -f1) + _kind=$(echo "$_image_var" | cut -d_ -f2) + _specific_user=$(eval echo "\$${_prefix}_REGISTRY_\$${_kind}_USER") + _default_user=$(eval echo "\$${_prefix}_REGISTRY_USER") + _specific_password=$(eval echo "\$${_prefix}_REGISTRY_\$${_kind}_PASSWORD") + _default_password=$(eval echo "\$${_prefix}_REGISTRY_PASSWORD") + _authent_token=$(echo -n "${_specific_user:-${_default_user:-$CI_REGISTRY_USER}}:${_specific_password:-${_default_password:-$CI_REGISTRY_PASSWORD}}" | base64 | tr -d '\n') + if [[ "$_registry_hosts" != "," ]] + then + _docker_cfg_json="${_docker_cfg_json}," + fi + _docker_cfg_json="${_docker_cfg_json}\"$_registry_host\":{\"auth\":\"$_authent_token\"}" + _registry_hosts="$_registry_hosts$_registry_host," + fi + fi + done + # end Docker config JSON + _docker_cfg_json="${_docker_cfg_json}}}" + mkdir -p ~/.docker + echo "$_docker_cfg_json" > ~/.docker/config.json + fi + fi } - # application deployment function - function dcmp_deploy() { + function find_first() { + for file in "$@"; do + if [[ -f "$file" ]]; then + echo "$file" + break + fi + done + } + + # initialize context and authentication (SSH) + function compose_init() { export environment_type=$ENV_TYPE export environment_name=${ENV_APP_NAME:-${DCMP_BASE_APP_NAME}${ENV_APP_SUFFIX}} - environment_url=${ENV_URL:-$DCMP_ENVIRONMENT_URL} # also export environment_name in SCREAMING_SNAKE_CASE format (may be useful with Kubernetes env variables) environment_name_ssc=$(to_ssc "$environment_name") export environment_name_ssc + # auto-detect command + if [[ -z "$DCMP_CMD" ]] + then + if command -v docker-compose > /dev/null + then + log_info "... \\e[33;1mdocker-compose\\e[0m command found: will be used" + DCMP_CMD=docker-compose + else + log_info "... \\e[33;1mdocker-compose\\e[0m command not found: will use \\e[33;1mdocker compose\\e[0m instead" + DCMP_CMD="docker compose" + fi + fi + + # set COMPOSE_PROJECT_NAME (use env name) + if [[ -z "$COMPOSE_PROJECT_NAME" ]] + then + export COMPOSE_PROJECT_NAME=$environment_name + log_info "--- \$COMPOSE_PROJECT_NAME unset: use \\e[33;1m${COMPOSE_PROJECT_NAME}\\e[0m" + fi + + # compose file lookup + # see: https://docs.docker.com/compose/compose-application-model/#the-compose-file + if [[ -z "$COMPOSE_FILE" ]] + then + log_info "--- \$COMPOSE_FILE unset: lookup for Docker Compose files..." + base_compose_file=$(find_first "$DCMP_SCRIPTS_DIR/compose.yaml" "$DCMP_SCRIPTS_DIR/compose.yml" "$DCMP_SCRIPTS_DIR/docker-compose.yaml" "$DCMP_SCRIPTS_DIR/docker-compose.yml") + env_compose_file=$(find_first "$DCMP_SCRIPTS_DIR/compose-${environment_type}.yaml" "$DCMP_SCRIPTS_DIR/compose-${environment_type}.yml" "$DCMP_SCRIPTS_DIR/docker-compose-${environment_type}.yaml" "$DCMP_SCRIPTS_DIR/docker-compose-${environment_type}.yml") + if [[ -f "$env_compose_file" ]] + then + COMPOSE_FILE=$env_compose_file + log_info "... env-specific Docker Compose file found: \\e[33;1m${env_compose_file}\\e[0m" + file_no_ext="${env_compose_file%.*}" + ext="${env_compose_file##*.}" + # lookup for env-specific override + env_override_file="${file_no_ext}.override.${ext}" + if [[ -f "${env_override_file}" ]] + then + log_info "... env-specific Docker Compose override file found: \\e[33;1m${env_override_file}\\e[0m" + COMPOSE_FILE="$COMPOSE_FILE:$env_override_file" + fi + elif [[ -f "$base_compose_file" ]] + then + COMPOSE_FILE=$base_compose_file + # lookup for base override + file_no_ext="${base_compose_file%.*}" + ext="${base_compose_file##*.}" + log_info "... base Docker Compose file found: \\e[33;1m${base_compose_file}\\e[0m" + base_override_file="${file_no_ext}.override.${ext}" + if [[ -f "${base_override_file}" ]] + then + log_info "... base Docker Compose override file found: \\e[33;1m${base_override_file}\\e[0m" + COMPOSE_FILE="$COMPOSE_FILE:$base_override_file" + fi + # lookup for env-specific override + env_override_file="${file_no_ext}-${environment_type}.override.${ext}" + if [[ -f "${env_override_file}" ]] + then + log_info "... env-specific Docker Compose override file found: \\e[33;1m${env_override_file}\\e[0m" + COMPOSE_FILE="$COMPOSE_FILE:$env_override_file" + fi + else + log_error "... no Docker Compose file found in $DCMP_SCRIPTS_DIR: please refer to the template documentation" + exit 1 + fi + fi + + # dotenv files lookup + if [[ -z "$COMPOSE_ENV_FILES" ]] + then + log_info "--- \$COMPOSE_ENV_FILES unset: lookup for env files..." + # latest defined file takes precedence + dcmp_envs="" + if [[ -f "$DCMP_SCRIPTS_DIR/.env" ]] + then + dcmp_envs="$dcmp_envs,$DCMP_SCRIPTS_DIR/.env" + log_info "... env file found: \\e[33;1m$DCMP_SCRIPTS_DIR/.env\\e[0m" + fi + if [[ -f "$DCMP_SCRIPTS_DIR/${environment_type}.env" ]] + then + dcmp_envs="$dcmp_envs,$DCMP_SCRIPTS_DIR/${environment_type}.env" + log_info "... env file found: \\e[33;1m$DCMP_SCRIPTS_DIR/${environment_type}.env\\e[0m" + fi + export COMPOSE_ENV_FILES=${dcmp_envs:1} + fi + } + + # application deployment function + function compose_up() { + environment_url=${ENV_URL:-$DCMP_ENVIRONMENT_URL} # variables expansion in $environment_url environment_url=$(echo "$environment_url" | awkenvsubst) export environment_url @@ -440,10 +662,30 @@ stages: log_info "--- \$hostname: \\e[33;1m${hostname}\\e[0m" # unset any upstream deployment env & artifacts - rm -f docker-compose.env + rm -f docker-compose.out.env rm -f environment_url.txt - # TODO: implement the deployment here + # maybe execute pre compose-up script + prescript="$DCMP_SCRIPTS_DIR/pre-compose-up.sh" + if [[ -f "$prescript" ]]; then + log_info "--- \\e[32mpre-compose-up\\e[0m hook (\\e[33;1m${prescript}\\e[0m) found: execute" + exec_hook "$prescript" + else + log_info "--- \\e[32mpre-compose-up\\e[0m hook (\\e[33;1m${prescript}\\e[0m) not found: skip" + fi + + # up (--detach is mandatory and therefore not configurable) + # shellcheck disable=SC2086 + $DCMP_CMD up --detach $DCMP_UP_OPTS + + # maybe execute post compose-up script + postscript="$DCMP_SCRIPTS_DIR/post-compose-up.sh" + if [[ -f "$postscript" ]]; then + log_info "--- \\e[32mpost-compose-up\\e[0m hook (\\e[33;1m${postscript}\\e[0m) found: execute" + exec_hook "$postscript" + else + log_info "--- \\e[32mpost-compose-up\\e[0m hook (\\e[33;1m${postscript}\\e[0m) not found: skip" + fi # persist environment url if [[ -f environment_url.txt ]] @@ -454,23 +696,37 @@ stages: else echo "$environment_url" > environment_url.txt fi - echo -e "environment_type=$environment_type\\nenvironment_name=$environment_name\\nenvironment_url=$environment_url" >> docker-compose.env + echo -e "environment_type=$environment_type\\nenvironment_name=$environment_name\\nenvironment_url=$environment_url" >> docker-compose.out.env } # environment cleanup function - function dcmp_delete() { - export environment_type=$ENV_TYPE - export environment_name=${ENV_APP_NAME:-${DCMP_BASE_APP_NAME}${ENV_APP_SUFFIX}} - # also export environment_name in SCREAMING_SNAKE_CASE format (may be useful with Kubernetes env variables) - environment_name_ssc=$(to_ssc "$environment_name") - export environment_name_ssc - + function compose_down() { log_info "--- \\e[32mcleanup\\e[0m" log_info "--- \$environment_type: \\e[33;1m${environment_type}\\e[0m" log_info "--- \$environment_name: \\e[33;1m${environment_name}\\e[0m" log_info "--- \$environment_name_ssc: \\e[33;1m${environment_name_ssc}\\e[0m" - # TODO: implement the cleanup here + # maybe execute pre compose-down script + prescript="$DCMP_SCRIPTS_DIR/pre-compose-down.sh" + if [[ -f "$prescript" ]]; then + log_info "--- \\e[32mpre-compose-down\\e[0m hook (\\e[33;1m${prescript}\\e[0m) found: execute" + exec_hook "$prescript" + else + log_info "--- \\e[32mpre-compose-down\\e[0m hook (\\e[33;1m${prescript}\\e[0m) not found: skip" + fi + + # down + # shellcheck disable=SC2086 + $DCMP_CMD down $DCMP_DOWN_OPTS + + # maybe execute post compose-down script + postscript="$DCMP_SCRIPTS_DIR/post-compose-down.sh" + if [[ -f "$postscript" ]]; then + log_info "--- \\e[32mpost-compose-down\\e[0m hook (\\e[33;1m${postscript}\\e[0m) found: execute" + exec_hook "$postscript" + else + log_info "--- \\e[32mpost-compose-down\\e[0m hook (\\e[33;1m${postscript}\\e[0m) not found: skip" + fi } unscope_variables @@ -479,102 +735,139 @@ stages: # ENDSCRIPT # job prototype -# defines default Docker image, tracking probe, cache policy and tags +# defines default Docker image, services, cache policy and init scripts # Required vars for login: -# @var ENV_API_URL : env-specific Docker Compose API url -# @var ENV_API_TOKEN : env-specific Docker Compose API token -# @var ENV_PROJECT : env-specific project name -.dcmp-base: +# @var ENV_TYPE : environment type +# @var ENV_APP_NAME : env-specific application name +# @var ENV_APP_SUFFIX: env-specific application suffix +# @var DOCKER_HOST : env-specific DOCKER_HOST +.compose-base: image: $DCMP_IMAGE services: - name: "$TBC_TRACKING_IMAGE" command: ["--service", "docker-compose", "1.0.0"] before_script: - - !reference [.dcmp-scripts] + - !reference [.compose-scripts] - install_ca_certs "${CUSTOM_CA_CERTS:-$DEFAULT_CA_CERTS}" - - dcmp_login + - compose_init + +# base job for online compose operations +.compose-online: + extends: .compose-base + before_script: + - !reference [.compose-base, before_script] + - configure_network + - configure_registries_auth # Deploy job prototype # Can be extended to define a concrete environment -# -# @var ENV_TYPE : environment type -# @var ENV_APP_NAME : env-specific application name -# @var ENV_APP_SUFFIX: env-specific application suffix # @var ENV_URL : env-specific application url -.dcmp-deploy: - extends: .dcmp-base +.compose-deploy: + extends: .compose-online stage: deploy variables: ENV_APP_SUFFIX: "-$CI_ENVIRONMENT_SLUG" script: - - dcmp_deploy + - compose_up artifacts: name: "$ENV_TYPE env url for $CI_PROJECT_NAME on $CI_COMMIT_REF_SLUG" - # TODO: propagate deployed env url in a environment_url.txt file + # propagate deployed env url in a environment_url.txt file paths: - environment_url.txt reports: - # TODO: propagate deployed env info in a dotenv artifact - dotenv: docker-compose.env + # propagate deployed env info in a dotenv artifact + dotenv: docker-compose.out.env environment: url: "$environment_url" # can be either static or dynamic # Cleanup job prototype # Can be extended for each deletable environment -# -# @var ENV_TYPE : environment type -# @var ENV_APP_NAME : env-specific application name -# @var ENV_APP_SUFFIX: env-specific application suffix -.dcmp-cleanup: - extends: .dcmp-base +.compose-cleanup: + extends: .compose-online stage: deploy # force no dependencies dependencies: [] variables: ENV_APP_SUFFIX: "-$CI_ENVIRONMENT_SLUG" script: - - dcmp_delete + - compose_down environment: action: stop +# run compose config job as parallel matrix +compose-config: + extends: .compose-base + stage: package-test + script: + - $DCMP_CMD config $DCMP_CONFIG_OPTS + parallel: + matrix: + - ENV_TYPE: review + ENV_APP_SUFFIX: "-review-slug" + ENV_APP_NAME: "$DCMP_REVIEW_APP_NAME" + - ENV_TYPE: integration + ENV_APP_SUFFIX: "-integration" + ENV_APP_NAME: "$DCMP_INTEG_APP_NAME" + - ENV_TYPE: staging + ENV_APP_SUFFIX: "-staging" + ENV_APP_NAME: "$DCMP_STAGINGAPP_NAME" + - ENV_TYPE: production + ENV_APP_NAME: "$DCMP_PROD_APP_NAME" + rules: + # exclude tags + - if: $CI_COMMIT_TAG + when: never + # exclude when $DCMP_CONFIG_DISABLED is set + - if: '$DCMP_CONFIG_DISABLED == "true"' + when: never + # review: skip if $DCMP_REVIEW_DOCKER_HOST unset or integration branch or prod branch + - if: '$ENV_TYPE == "review" && ($DCMP_REVIEW_DOCKER_HOST == null || $DCMP_REVIEW_DOCKER_HOST == "" || $CI_COMMIT_REF_NAME =~ $INTEG_REF || $CI_COMMIT_REF_NAME =~ $PROD_REF)' + when: never + # integration: skip if $DCMP_INTEG_DOCKER_HOST unset or prod branch + - if: '$ENV_TYPE == "integration" && ($DCMP_INTEG_DOCKER_HOST == null || $DCMP_INTEG_DOCKER_HOST == "" || $CI_COMMIT_REF_NAME =~ $PROD_REF)' + when: never + # staging: skip if $DCMP_STAGING_DOCKER_HOST unset + - if: '$ENV_TYPE == "staging" && ($DCMP_STAGING_DOCKER_HOST == null || $DCMP_STAGING_DOCKER_HOST == "")' + when: never + # production: skip if $DCMP_PROD_DOCKER_HOST unset + - if: '$ENV_TYPE == "production" && ($DCMP_PROD_DOCKER_HOST == null || $DCMP_PROD_DOCKER_HOST == "")' + when: never + # test policy rules must come last + - !reference [.test-policy, rules] + # deploy to review env (only on feature branches) -# disabled by default, enable this job by setting $DCMP_REVIEW_PROJECT. -dcmp-review: - extends: .dcmp-deploy +# disabled by default, enable this job by setting $DCMP_REVIEW_DOCKER_HOST. +compose-review: + extends: .compose-deploy variables: + DOCKER_HOST: "$DCMP_REVIEW_DOCKER_HOST" + ENV_SSH_PRIVATE_KEY: "$DCMP_REVIEW_SSH_PRIVATE_KEY" ENV_TYPE: review ENV_APP_NAME: "$DCMP_REVIEW_APP_NAME" ENV_URL: "$DCMP_REVIEW_ENVIRONMENT_URL" - ENV_API_URL: "$DCMP_REVIEW_API_URL" - ENV_API_TOKEN: "$DCMP_REVIEW_API_TOKEN" - ENV_PROJECT: "$DCMP_REVIEW_PROJECT" environment: name: review/$CI_COMMIT_REF_NAME - on_stop: dcmp-cleanup-review + on_stop: compose-cleanup-review auto_stop_in: "$DCMP_REVIEW_AUTOSTOP_DURATION" resource_group: review/$CI_COMMIT_REF_NAME rules: # exclude tags - if: $CI_COMMIT_TAG when: never - # exclude if $CLEANUP_ALL_REVIEW set to 'force' - - if: '$CLEANUP_ALL_REVIEW == "force"' - when: never - # exclude if $DCMP_REVIEW_PROJECT not set - - if: '$DCMP_REVIEW_PROJECT == null || $DCMP_REVIEW_PROJECT == ""' + # exclude if $DCMP_REVIEW_DOCKER_HOST not set + - if: '$DCMP_REVIEW_DOCKER_HOST == null || $DCMP_REVIEW_DOCKER_HOST == ""' when: never # only on non-production, non-integration branches - if: '$CI_COMMIT_REF_NAME !~ $PROD_REF && $CI_COMMIT_REF_NAME !~ $INTEG_REF' # cleanup review env (automatically triggered once branches are deleted) -dcmp-cleanup-review: - extends: .dcmp-cleanup +compose-cleanup-review: + extends: .compose-cleanup variables: + DOCKER_HOST: "$DCMP_REVIEW_DOCKER_HOST" + ENV_SSH_PRIVATE_KEY: "$DCMP_REVIEW_SSH_PRIVATE_KEY" ENV_TYPE: review ENV_APP_NAME: "$DCMP_REVIEW_APP_NAME" - ENV_API_URL: "$DCMP_REVIEW_API_URL" - ENV_API_TOKEN: "$DCMP_REVIEW_API_TOKEN" - ENV_PROJECT: "$DCMP_REVIEW_PROJECT" environment: name: review/$CI_COMMIT_REF_NAME action: stop @@ -584,8 +877,8 @@ dcmp-cleanup-review: # exclude tags - if: $CI_COMMIT_TAG when: never - # exclude if $DCMP_REVIEW_PROJECT not set - - if: '$DCMP_REVIEW_PROJECT == null || $DCMP_REVIEW_PROJECT == ""' + # exclude if $DCMP_REVIEW_DOCKER_HOST not set + - if: '$DCMP_REVIEW_DOCKER_HOST == null || $DCMP_REVIEW_DOCKER_HOST == ""' when: never # only on non-production, non-integration branches - if: '$CI_COMMIT_REF_NAME !~ $PROD_REF && $CI_COMMIT_REF_NAME !~ $INTEG_REF' @@ -593,66 +886,63 @@ dcmp-cleanup-review: allow_failure: true # deploy to `integration` env (only on develop branch) -dcmp-integration: - extends: .dcmp-deploy +compose-integration: + extends: .compose-deploy variables: + DOCKER_HOST: "$DCMP_INTEG_DOCKER_HOST" + ENV_SSH_PRIVATE_KEY: "$DCMP_INTEG_SSH_PRIVATE_KEY" ENV_TYPE: integration ENV_APP_NAME: "$DCMP_INTEG_APP_NAME" ENV_URL: "$DCMP_INTEG_ENVIRONMENT_URL" - ENV_API_URL: "$DCMP_INTEG_API_URL" - ENV_API_TOKEN: "$DCMP_INTEG_API_TOKEN" - ENV_PROJECT: "$DCMP_INTEG_PROJECT" environment: name: integration # TODO: use resource group resource_group: integration rules: - # exclude if $DCMP_INTEG_PROJECT not set - - if: '$DCMP_INTEG_PROJECT == null || $DCMP_INTEG_PROJECT == ""' + # exclude if $DCMP_INTEG_DOCKER_HOST not set + - if: '$DCMP_INTEG_DOCKER_HOST == null || $DCMP_INTEG_DOCKER_HOST == ""' when: never # only on integration branch(es) - if: '$CI_COMMIT_REF_NAME =~ $INTEG_REF' # deploy to `staging` env (only on master branch) -dcmp-staging: - extends: .dcmp-deploy +compose-staging: + extends: .compose-deploy variables: + DOCKER_HOST: "$DCMP_STAGING_DOCKER_HOST" + ENV_SSH_PRIVATE_KEY: "$DCMP_STAGING_SSH_PRIVATE_KEY" ENV_TYPE: staging ENV_APP_NAME: "$DCMP_STAGING_APP_NAME" ENV_URL: "$DCMP_STAGING_ENVIRONMENT_URL" - ENV_API_URL: "$DCMP_STAGING_API_URL" - ENV_API_TOKEN: "$DCMP_STAGING_API_TOKEN" - ENV_PROJECT: "$DCMP_STAGING_PROJECT" environment: name: staging # TODO: use resource group resource_group: staging rules: - # exclude if $DCMP_STAGING_PROJECT not set - - if: '$DCMP_STAGING_PROJECT == null || $DCMP_STAGING_PROJECT == ""' + # exclude if $DCMP_STAGING_DOCKER_HOST not set + - if: '$DCMP_STAGING_DOCKER_HOST == null || $DCMP_STAGING_DOCKER_HOST == ""' when: never # only on production branch(es) - if: '$CI_COMMIT_REF_NAME =~ $PROD_REF' -# Deploy to production if on branch master and variable DCMP_PROD_PROJECT defined and AUTODEPLOY_TO_PROD is set -dcmp-production: - extends: .dcmp-deploy +# Deploy to production if on branch master and variable DCMP_PROD_DOCKER_HOST defined and AUTODEPLOY_TO_PROD is set +compose-production: + extends: .compose-deploy stage: production variables: + DOCKER_HOST: "$DCMP_PROD_DOCKER_HOST" + ENV_SSH_PRIVATE_KEY: "$DCMP_PROD_SSH_PRIVATE_KEY" ENV_TYPE: production ENV_APP_SUFFIX: "" # no suffix for prod ENV_APP_NAME: "$DCMP_PROD_APP_NAME" ENV_URL: "$DCMP_PROD_ENVIRONMENT_URL" - ENV_API_URL: "$DCMP_PROD_API_URL" - ENV_API_TOKEN: "$DCMP_PROD_API_TOKEN" - ENV_PROJECT: "$DCMP_PROD_PROJECT" environment: name: production # TODO: use resource group resource_group: production rules: - # exclude if $DCMP_PROD_PROJECT not set - - if: '$DCMP_PROD_PROJECT == null || $DCMP_PROD_PROJECT == ""' + # exclude if $DCMP_PROD_DOCKER_HOST not set + - if: '$DCMP_PROD_DOCKER_HOST == null || $DCMP_PROD_DOCKER_HOST == ""' when: never # exclude non-production branch(es) - if: '$CI_COMMIT_REF_NAME !~ $PROD_REF'