diff --git a/.gitlab/merge_request_templates/new_feature.md b/.gitlab/merge_request_templates/new_feature.md index 74abae94c94dc0768bb5c51fe51ad253fce113fe..491b7f98ded7e0da03d18c95978eafcb7d86619f 100644 --- a/.gitlab/merge_request_templates/new_feature.md +++ b/.gitlab/merge_request_templates/new_feature.md @@ -8,8 +8,8 @@ Closes #999 ## Checklist * General: - * [ ] use [rules](https://docs.gitlab.com/ee/ci/yaml/#rules) instead of [only/except](https://docs.gitlab.com/ee/ci/yaml/#onlyexcept-advanced) - * [ ] optimized [cache](https://docs.gitlab.com/ee/ci/caching/) configuration (wherever applicable) + * [ ] use [rules](https://docs.gitlab.com/ci/yaml/#rules) instead of [only/except](https://docs.gitlab.com/ci/yaml/#onlyexcept-advanced) + * [ ] optimized [cache](https://docs.gitlab.com/ci/caching/) configuration (wherever applicable) * Publicly usable: * [ ] untagged runners * [ ] no proxy configuration but support `http_proxy`/`https_proxy`/`no_proxy` diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b8dbe2400120e859d5402fd51da599685c0c45e..ab205310773bfe2e3f400ade79846bad747eec34 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,18 @@ -# [1.2.0](https://git.code.tecnalia.dev/smartdatalab/public/ci-cd-components/docker-compose/compare/1.1.0...1.2.0) (2025-01-29) +## [1.2.2](https://gitlab.com/to-be-continuous/docker-compose/compare/1.2.1...1.2.2) (2025-04-11) + + +### Bug Fixes + +* **envsubst:** leave lines with '# nosubst' unchanged when substituting (used to be simply dropped) ([392e60d](https://gitlab.com/to-be-continuous/docker-compose/commit/392e60d69d47130f7f1a249ef8001c1fe6b234d0)) + +## [1.2.1](https://gitlab.com/to-be-continuous/docker-compose/compare/1.2.0...1.2.1) (2025-02-01) + + +### Bug Fixes + +* homogenize new TBC envsubst mechanism ([63bca5c](https://gitlab.com/to-be-continuous/docker-compose/commit/63bca5c391b44614a0e34edcc4dcdc82f55e6c4d)) + +# [1.2.0](https://gitlab.com/to-be-continuous/docker-compose/compare/1.1.0...1.2.0) (2025-01-27) ### Features diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7414195e1bf2f16feb4762b1844a68016511d0eb..3d4325b1b3db820a5425a473df7a7a55d3c37128 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -61,7 +61,7 @@ To contribute: 1. Create an issue describing the bug or enhancement you want to propose (select the right issue template). 2. Make sure the issue has been reviewed and agreed. -3. Create a Merge Request, from your **own** fork (see [forking workflow](https://docs.gitlab.com/ee/user/project/repository/forking_workflow.html) documentation). +3. Create a Merge Request, from your **own** fork (see [forking workflow](https://docs.gitlab.com/user/project/repository/forking_workflow/) documentation). Don't hesitate to mark your MR as `Draft` as long as you think it's not ready to be reviewed. ### Git Commit Conventions diff --git a/README.md b/README.md index d9a511359d6ad0cd6fc8e3fb19259444ada3b8f8..b162bad45c3162897bd10f1f90b56d716802944d 100644 --- a/README.md +++ b/README.md @@ -4,8 +4,8 @@ This project implements a GitLab CI/CD template to deploy your application with ## Usage -This template can be used both as a [CI/CD component](https://docs.gitlab.com/ee/ci/components/#use-a-component) -or using the legacy [`include:project`](https://docs.gitlab.com/ee/ci/yaml/index.html#includeproject) syntax. +This template can be used both as a [CI/CD component](https://docs.gitlab.com/ci/components/#use-a-component) +or using the legacy [`include:project`](https://docs.gitlab.com/ci/yaml/#includeproject) syntax. ### Use as a CI/CD component @@ -14,7 +14,7 @@ Add the following to your `.gitlab-ci.yml`: ```yaml include: # 1: include the component - - component: $CI_SERVER_FQDN/to-be-continuous/docker-compose/gitlab-ci-docker-compose@1.2.0 + - component: $CI_SERVER_FQDN/to-be-continuous/docker-compose/gitlab-ci-docker-compose@1.2.2 # 2: set/override component inputs inputs: # ⚠ this is only an example @@ -32,7 +32,7 @@ Add the following to your `.gitlab-ci.yml`: include: # 1: include the template - project: 'to-be-continuous/docker-compose' - ref: '1.2.0' + ref: '1.2.2' file: '/templates/gitlab-ci-docker-compose.yml' variables: @@ -67,7 +67,7 @@ _ongoing developments_ (a.k.a. _feature_ or _topic_ branches). When enabled, it deploys the result from upstream build stages to a dedicated and temporary environment. It is only active for non-production, non-integration branches. -It is a strict equivalent of GitLab's [Review Apps](https://docs.gitlab.com/ee/ci/review_apps/) feature. +It is a strict equivalent of GitLab's [Review Apps](https://docs.gitlab.com/ci/review_apps/) feature. It also comes with a _cleanup_ job (accessible either from the _environments_ page, or from the pipeline view). @@ -244,8 +244,8 @@ Part of this complexity can be handled by the lookup strategies described above * `${environment_type}`: the current environment type (`review`, `integration`, `staging` or `production`) * `${environment_name}`: the application name to use for the current environment (ex: `myproject-review-fix-bug-12` or `myproject-staging`) * `${hostname}`: the environment hostname, extracted from the current environment url (after late variable expansion - see below) -2. any [GitLab CI variable](https://docs.gitlab.com/ee/ci/variables/predefined_variables.html) -3. any [custom variable](https://docs.gitlab.com/ee/ci/variables/#add-a-cicd-variable-to-a-docker-host) +2. any [GitLab CI variable](https://docs.gitlab.com/ci/variables/predefined_variables/) +3. any [custom variable](https://docs.gitlab.com/ci/variables/#add-a-cicd-variable-to-a-docker-host) (ex: `${SECRET_TOKEN}` that you have set in your project CI/CD variables) Be aware that environment variables may be freely used and substituted in [dotenv files](https://docs.docker.com/compose/environment-variables/env-file/) @@ -256,7 +256,7 @@ using the appropriate [interpolation syntax](https://docs.docker.com/compose/env The Docker Compose template supports two ways of providing your environments url: * a **static way**: when the environments url can be determined in advance, probably because you're exposing your routes through a DNS you manage, -* a [**dynamic way**](https://docs.gitlab.com/ee/ci/environments/#set-a-dynamic-environment-url): when the url cannot be known before the +* a [**dynamic way**](https://docs.gitlab.com/ci/environments/#set-a-dynamic-environment-url): when the url cannot be known before the deployment job is executed. The **static way** can be implemented simply by setting the appropriate configuration variable(s) depending on the environment (see environments configuration chapters): @@ -285,7 +285,7 @@ the dynamically generated url. When detected by the template, it will use it as ### Deployment output variables -Each deployment job produces _output variables_ that are propagated to downstream jobs (using [dotenv artifacts](https://docs.gitlab.com/ee/ci/yaml/artifacts_reports.html#artifactsreportsdotenv)): +Each deployment job produces _output variables_ that are propagated to downstream jobs (using [dotenv artifacts](https://docs.gitlab.com/ci/yaml/artifacts_reports/#artifactsreportsdotenv)): * `$environment_type`: set to the type of environment (`review`, `integration`, `staging` or `production`), * `$environment_name`: the application name (see below), @@ -301,12 +301,12 @@ You may also add and propagate your own custom variables, by pushing them to the Here are some advices about your **secrets** (variables marked with a :lock:): -1. Manage them as [project or group CI/CD variables](https://docs.gitlab.com/ee/ci/variables/#add-a-cicd-variable-to-a-docker-host): - * [**masked**](https://docs.gitlab.com/ee/ci/variables/#mask-a-cicd-variable) to prevent them from being inadvertently +1. Manage them as [project or group CI/CD variables](https://docs.gitlab.com/ci/variables/#add-a-cicd-variable-to-a-docker-host): + * [**masked**](https://docs.gitlab.com/ci/variables/#mask-a-cicd-variable) to prevent them from being inadvertently displayed in your job logs, - * [**protected**](https://docs.gitlab.com/ee/ci/variables/#protected-cicd-variables) if you want to secure some secrets + * [**protected**](https://docs.gitlab.com/ci/variables/#protected-cicd-variables) if you want to secure some secrets you don't want everyone in the project to have access to (for instance production secrets). -2. In case a secret contains [characters that prevent it from being masked](https://docs.gitlab.com/ee/ci/variables/#mask-a-cicd-variable), +2. In case a secret contains [characters that prevent it from being masked](https://docs.gitlab.com/ci/variables/#mask-a-cicd-variable), simply define its value as the [Base64](https://en.wikipedia.org/wiki/Base64) encoded value prefixed with `@b64@`: it will then be possible to mask it and the template will automatically decode it prior to using it. 3. Don't forget to escape special characters (ex: `$` -> `$$`). @@ -317,9 +317,9 @@ The Docker Compose template uses some global configuration used throughout all j | Input / Variable | Description | Default value | | ------------------------ |-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| ----------------- | -| `image` / `DCMP_IMAGE` | The Docker image used to run Docker Compose CLI commands | `registry.hub.docker.com/library/docker:latest` | +| `image` / `DCMP_IMAGE` | The Docker image used to run Docker Compose CLI commands | `registry.hub.docker.com/library/docker:latest` <br/>[](https://to-be-continuous.gitlab.io/doc/secu/trivy-DCMP_IMAGE) | | `cmd` / `DCMP_CMD` | The docker compose or stack command (`docker compose`, `docker-compose` or `docker stack`) | _none_ (auto) | -| `base-app-name` / `DCMP_BASE_APP_NAME`| Base application name | `$CI_PROJECT_NAME` ([see GitLab doc](https://docs.gitlab.com/ee/ci/variables/predefined_variables.html)) | +| `base-app-name` / `DCMP_BASE_APP_NAME`| Base application name | `$CI_PROJECT_NAME` ([see GitLab doc](https://docs.gitlab.com/ci/variables/predefined_variables/)) | | `environment-url` / `DCMP_ENVIRONMENT_URL`| Default environments url _(only define for static environment URLs declaration)_<br/>_supports late variable expansion (ex: `https://%{environment_name}.docker-compose.acme.com`)_ | _none_ | | `scripts-dir` / `DCMP_SCRIPTS_DIR`| Directory where Compose files, dotenv files and hook scripts are located | `.` _(root project dir)_ | | `up-opts` / `DCMP_UP_OPTS` | [`compose up` options](https://docs.docker.com/reference/cli/docker/compose/up/#options) (only when using Docker Compose) | `--no-build --remove-orphans --wait --wait-timeout 180` | diff --git a/kicker.json b/kicker.json index f0cfa1827451487272fdcc1a6b60a071e5f2bc48..88d9ac9f55bcb5cb89b7f30f72a7b3896d4b97b8 100644 --- a/kicker.json +++ b/kicker.json @@ -104,7 +104,7 @@ { "id": "review", "name": "Review", - "description": "Dynamic review environments for your topic branches (see GitLab [Review Apps](https://docs.gitlab.com/ee/ci/review_apps/))", + "description": "Dynamic review environments for your topic branches (see GitLab [Review Apps](https://docs.gitlab.com/ci/review_apps/))", "variables": [ { "name": "DCMP_REVIEW_DOCKER_HOST", diff --git a/templates/gitlab-ci-docker-compose.yml b/templates/gitlab-ci-docker-compose.yml index e0c1ec0c14a53c6c5dd98f44e22810955bd829df..1f955ba0d82795ade51191c6c2bf23e54721cb24 100644 --- a/templates/gitlab-ci-docker-compose.yml +++ b/templates/gitlab-ci-docker-compose.yml @@ -464,9 +464,82 @@ stages: echo "$1" | tr '[:lower:]' '[:upper:]' | tr '[:punct:]' '_' } - function awkenvsubst() { - # escapes '&' char in variables for gsub - awk '{while(match($0,"[$%]{[^}]*}")) {var=substr($0,RSTART+2,RLENGTH-3);val=ENVIRON[var];gsub("&","\\\\&",val);gsub("[$%]{"var"}",val)}}1' + function tbc_envsubst() { + awk ' + BEGIN { + count_replaced_lines = 0 + # ASCII codes + for (i=0; i<=255; i++) + char2code[sprintf("%c", i)] = i + } + # determine encoding (from env or from file extension) + function encoding() { + enc = ENVIRON["TBC_ENVSUBST_ENCODING"] + if (enc != "") + return enc + if (match(FILENAME, /\.(json|yaml|yml)$/)) + return "jsonstr" + return "raw" + } + # see: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/encodeURIComponent + function uriencode(str) { + len = length(str) + enc = "" + for (i=1; i<=len; i++) { + c = substr(str, i, 1); + if (index("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_.!~*'\''()", c)) + enc = enc c + else + enc = enc "%" sprintf("%02X", char2code[c]) + } + return enc + } + /# *nosubst/ { + print $0 + next + } + { + orig_line = $0 + line = $0 + count_repl_in_line = 0 + # /!\ 3rd arg (match) not supported in BusyBox awk + while (match(line, /[$%]\{([[:alnum:]_]+)\}/)) { + expr_start = RSTART + expr_len = RLENGTH + # get var name + var = substr(line, expr_start+2, expr_len-3) + # get var value (from env) + val = ENVIRON[var] + # check variable is set + if (val == "") { + printf("[\033[1;93mWARN\033[0m] Environment variable \033[33;1m%s\033[0m is not set or empty\n", var) > "/dev/stderr" + } else { + enc = encoding() + if (enc == "jsonstr") { + gsub(/["\\]/, "\\\\&", val) + gsub("\n", "\\n", val) + gsub("\r", "\\r", val) + gsub("\t", "\\t", val) + } else if (enc == "uricomp") { + val = uriencode(val) + } else if (enc == "raw") { + } else { + printf("[\033[1;93mWARN\033[0m] Unsupported encoding \033[33;1m%s\033[0m: ignored\n", enc) > "/dev/stderr" + } + } + # replace expression in line + line = substr(line, 1, expr_start - 1) val substr(line, expr_start + expr_len) + count_repl_in_line++ + } + if (count_repl_in_line) { + if (count_replaced_lines == 0) + printf("[\033[1;94mINFO\033[0m] Variable expansion occurred in file \033[33;1m%s\033[0m:\n", FILENAME) > "/dev/stderr" + count_replaced_lines++ + printf("> line %s: %s\n", NR, orig_line) > "/dev/stderr" + } + print line + } + ' "$@" } function configure_network() { @@ -474,7 +547,7 @@ stages: if [[ -f ".netrc" ]] then log_info "--- \\e[32m.netrc\\e[0m file found: envsubst and install" - awkenvsubst < .netrc > ~/.netrc + tbc_envsubst .netrc > ~/.netrc chmod 0600 ~/.netrc fi @@ -533,7 +606,7 @@ stages: # special variable supported TBC_CI_REGISTRY_TOKEN=$(echo -n "$CI_REGISTRY_USER:$CI_REGISTRY_PASSWORD" | base64 | tr -d '\n') export TBC_CI_REGISTRY_TOKEN - awkenvsubst < .docker/config.json > ~/.docker/config.json + tbc_envsubst .docker/config.json > ~/.docker/config.json else log_info "--- \\e[32m.docker/config.json\\e[0m file not found: looking for TBC built images..." _image_vars=$(env | awk -F '=' "/^[A-Z]+_(SNAPSHOT|RELEASE)_IMAGE=/ {print \$1}" | sort | uniq) @@ -688,7 +761,7 @@ stages: function compose_up() { environment_url=${ENV_URL:-$DCMP_ENVIRONMENT_URL} # variables expansion in $environment_url - environment_url=$(echo "$environment_url" | awkenvsubst) + environment_url=$(echo "$environment_url" | TBC_ENVSUBST_ENCODING=uricomp tbc_envsubst) export environment_url # extract hostname from $environment_url hostname=$(echo "$environment_url" | awk -F[/:] '{print $4}') @@ -801,7 +874,7 @@ stages: image: $DCMP_IMAGE services: - name: "$TBC_TRACKING_IMAGE" - command: ["--service", "docker-compose", "1.2.0"] + command: ["--service", "docker-compose", "1.2.2"] before_script: - !reference [.compose-scripts] - install_ca_certs "${CUSTOM_CA_CERTS:-$DEFAULT_CA_CERTS}"