diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index fb1ed91402a9ba400dbb69ccf4744b49ae0ea3df..0592efc9de3f8644ca0f141a555c389d6069359b 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -5,4 +5,4 @@ stages:
- build
variables:
- GITLAB_CI_FILES: "templates/*.yml"
+ GITLAB_CI_FILES: "templates/extract.yml templates/validation.yml"
diff --git a/README.md b/README.md
index c749a260f98c90906b27ff4f1e32a9a00079b4ff..1057a17bb3545db4e7c5a98723ede733862b8e06 100644
--- a/README.md
+++ b/README.md
@@ -4,7 +4,7 @@
This template provides a jobs to validate your template syntax.
-It uses the [GitLab CI Lint API](https://docs.gitlab.com/ee/api/lint.html).
+It uses the [GitLab CI Lint API](https://docs.gitlab.com/api/lint/).
## Template validation
@@ -24,3 +24,61 @@ stages:
variables:
GITLAB_CI_FILES: "templates/*.yml"
```
+## Variants
+
+The default validation template is designed to work on untagged runners, without any proxy configuration using Docker images
+from the internet and using Default Trusted Certificate Authorities.
+
+Nevertheless there are template variants available to cover specific cases.
+
+### Vault variant
+
+This variant allows delegating your secrets management to a [Vault](https://www.vaultproject.io/) server.
+
+#### Configuration
+
+In order to be able to communicate with the Vault server, the variant requires the additional configuration parameters:
+
+| Input / Variable | Description | Default value |
+| ----------------- | -------------------------------------- | ----------------- |
+| `TBC_VAULT_IMAGE` | The [Vault Secrets Provider](https://gitlab.com/to-be-continuous/tools/vault-secrets-provider) image to use (can be overridden) | `registry.gitlab.com/to-be-continuous/tools/vault-secrets-provider:latest` |
+| `vault-base-url` / `VAULT_BASE_URL` | The Vault server base API url | **must be defined** |
+| `vault-oidc-aud` / `VAULT_OIDC_AUD` | The `aud` claim for the JWT | `$CI_SERVER_URL` |
+| :lock: `VAULT_ROLE_ID` | The [AppRole](https://www.vaultproject.io/docs/auth/approle) RoleID | _none_ |
+| :lock: `VAULT_SECRET_ID` | The [AppRole](https://www.vaultproject.io/docs/auth/approle) SecretID | _none_ |
+
+By default, the variant will authentifacte using a [JWT ID token](https://docs.gitlab.com/ci/secrets/id_token_authentication/). To use [AppRole](https://www.vaultproject.io/docs/auth/approle) instead the `VAULT_ROLE_ID` and `VAULT_SECRET_ID` should be defined as secret project variables.
+
+#### Usage
+
+Then you may retrieve any of your secret(s) from Vault using the following syntax:
+
+```text
+@url@http://vault-secrets-provider/api/secrets/{secret_path}?field={field}
+```
+
+With:
+
+| Parameter | Description |
+| -------------------------------- | -------------------------------------- |
+| `secret_path` (_path parameter_) | this is your secret location in the Vault server |
+| `field` (_query parameter_) | parameter to access a single basic field from the secret JSON payload |
+
+#### Example
+
+```yaml
+include:
+ # main template
+ - component: $CI_SERVER_FQDN/to-be-continuous/tools/gitlab-ci/validation@master
+ # Vault variant
+ - component: $CI_SERVER_FQDN/to-be-continuous/tools/gitlab-ci/validation-vault@master
+ inputs:
+ # audience claim for JWT
+ vault-oidc-aud: "https://vault.acme.host"
+ vault-base-url: "https://vault.acme.host/v1"
+
+variables:
+ # Secrets managed by Vault
+ GITLAB_TOKEN: "@url@http://vault-secrets-provider/api/secrets/b7ecb6ebabc231/my-infra/gitlab?field=token"
+```
+
diff --git a/templates/validation-vault.yml b/templates/validation-vault.yml
new file mode 100644
index 0000000000000000000000000000000000000000..e76fe9d5a3be7f5b8489baca69eebce2d7908c24
--- /dev/null
+++ b/templates/validation-vault.yml
@@ -0,0 +1,31 @@
+ # ====================================================================================================================
+ # === Vault template variant
+ # ====================================================================================================================
+spec:
+ inputs:
+ vault-base-url:
+ description: The Vault server base API url
+ default: ''
+ vault-oidc-aud:
+ description: the `aud` claim for the JWT
+ default: $CI_SERVER_URL
+---
+variables:
+ # variabilized vault-secrets-provider image
+ TBC_VAULT_IMAGE: registry.gitlab.com/to-be-continuous/tools/vault-secrets-provider:latest
+ # variables have to be explicitly declared in the YAML to be exported to the service
+ VAULT_ROLE_ID: $VAULT_ROLE_ID
+ VAULT_SECRET_ID: $VAULT_SECRET_ID
+ VAULT_OIDC_AUD: $[[ inputs.vault-oidc-aud ]]
+
+ VAULT_BASE_URL: $[[ inputs.vault-base-url ]]
+
+.gitlab-ci-base:
+ services:
+ - name: "$TBC_VAULT_IMAGE"
+ alias: "vault-secrets-provider"
+ variables:
+ VAULT_JWT_TOKEN: "$VAULT_JWT_TOKEN"
+ id_tokens:
+ VAULT_JWT_TOKEN:
+ aud: "$VAULT_OIDC_AUD"
diff --git a/templates/validation.yml b/templates/validation.yml
index d1ab3583923a4c2b1c94385c6c3d109f5cfeca1d..4f2361fd09dc6cd3f044f962377f99c188a40893 100644
--- a/templates/validation.yml
+++ b/templates/validation.yml
@@ -33,6 +33,11 @@ spec:
echo -e "[\e[1;91mERROR\e[0m] $*" >&2
}
+ function fail() {
+ log_error "$*"
+ exit 1
+ }
+
function install_ca_certs() {
certs=$1
if [[ -z "$certs" ]]
@@ -51,6 +56,159 @@ spec:
fi
}
+ function unscope_variables() {
+ _scoped_vars=$(env | awk -F '=' "/^scoped__[a-zA-Z0-9_]+=/ {print \$1}" | sort)
+ if [[ -z "$_scoped_vars" ]]; then return; fi
+ log_info "Processing scoped variables..."
+ for _scoped_var in $_scoped_vars
+ do
+ _fields=${_scoped_var//__/:}
+ _condition=$(echo "$_fields" | cut -d: -f3)
+ case "$_condition" in
+ if) _not="";;
+ ifnot) _not=1;;
+ *)
+ log_warn "... unrecognized condition \\e[1;91m$_condition\\e[0m in \\e[33;1m${_scoped_var}\\e[0m"
+ continue
+ ;;
+ esac
+ _target_var=$(echo "$_fields" | cut -d: -f2)
+ _cond_var=$(echo "$_fields" | cut -d: -f4)
+ _cond_val=$(eval echo "\$${_cond_var}")
+ _test_op=$(echo "$_fields" | cut -d: -f5)
+ case "$_test_op" in
+ defined)
+ if [[ -z "$_not" ]] && [[ -z "$_cond_val" ]]; then continue;
+ elif [[ "$_not" ]] && [[ "$_cond_val" ]]; then continue;
+ fi
+ ;;
+ equals|startswith|endswith|contains|in|equals_ic|startswith_ic|endswith_ic|contains_ic|in_ic)
+ # comparison operator
+ # sluggify actual value
+ _cond_val=$(echo "$_cond_val" | tr '[:punct:]' '_')
+ # retrieve comparison value
+ _cmp_val_prefix="scoped__${_target_var}__${_condition}__${_cond_var}__${_test_op}__"
+ _cmp_val=${_scoped_var#"$_cmp_val_prefix"}
+ # manage 'ignore case'
+ if [[ "$_test_op" == *_ic ]]
+ then
+ # lowercase everything
+ _cond_val=$(echo "$_cond_val" | tr '[:upper:]' '[:lower:]')
+ _cmp_val=$(echo "$_cmp_val" | tr '[:upper:]' '[:lower:]')
+ fi
+ case "$_test_op" in
+ equals*)
+ if [[ -z "$_not" ]] && [[ "$_cond_val" != "$_cmp_val" ]]; then continue;
+ elif [[ "$_not" ]] && [[ "$_cond_val" == "$_cmp_val" ]]; then continue;
+ fi
+ ;;
+ startswith*)
+ if [[ -z "$_not" ]] && [[ "$_cond_val" != "$_cmp_val"* ]]; then continue;
+ elif [[ "$_not" ]] && [[ "$_cond_val" == "$_cmp_val"* ]]; then continue;
+ fi
+ ;;
+ endswith*)
+ if [[ -z "$_not" ]] && [[ "$_cond_val" != *"$_cmp_val" ]]; then continue;
+ elif [[ "$_not" ]] && [[ "$_cond_val" == *"$_cmp_val" ]]; then continue;
+ fi
+ ;;
+ contains*)
+ if [[ -z "$_not" ]] && [[ "$_cond_val" != *"$_cmp_val"* ]]; then continue;
+ elif [[ "$_not" ]] && [[ "$_cond_val" == *"$_cmp_val"* ]]; then continue;
+ fi
+ ;;
+ in*)
+ if [[ -z "$_not" ]] && [[ "__${_cmp_val}__" != *"__${_cond_val}__"* ]]; then continue;
+ elif [[ "$_not" ]] && [[ "__${_cmp_val}__" == *"__${_cond_val}__"* ]]; then continue;
+ fi
+ ;;
+ esac
+ ;;
+ *)
+ log_warn "... unrecognized test operator \\e[1;91m${_test_op}\\e[0m in \\e[33;1m${_scoped_var}\\e[0m"
+ continue
+ ;;
+ esac
+ # matches
+ _val=$(eval echo "\$${_target_var}")
+ log_info "... apply \\e[32m${_target_var}\\e[0m from \\e[32m\$${_scoped_var}\\e[0m${_val:+ (\\e[33;1moverwrite\\e[0m)}"
+ _val=$(eval echo "\$${_scoped_var}")
+ export "${_target_var}"="${_val}"
+ done
+ log_info "... done"
+ }
+
+ # evaluate and export a secret
+ # - $1: secret variable name
+ function eval_secret() {
+ name=$1
+ value=$(eval echo "\$${name}")
+ case "$value" in
+ @b64@*)
+ decoded=$(mktemp)
+ errors=$(mktemp)
+ if echo "$value" | cut -c6- | base64 -d > "${decoded}" 2> "${errors}"
+ then
+ # shellcheck disable=SC2086
+ export ${name}="$(cat ${decoded})"
+ log_info "Successfully decoded base64 secret \\e[33;1m${name}\\e[0m"
+ else
+ fail "Failed decoding base64 secret \\e[33;1m${name}\\e[0m:\\n$(sed 's/^/... /g' "${errors}")"
+ fi
+ ;;
+ @hex@*)
+ decoded=$(mktemp)
+ errors=$(mktemp)
+ if echo "$value" | cut -c6- | sed 's/\([0-9A-F]\{2\}\)/\\\\x\1/gI' | xargs printf > "${decoded}" 2> "${errors}"
+ then
+ # shellcheck disable=SC2086
+ export ${name}="$(cat ${decoded})"
+ log_info "Successfully decoded hexadecimal secret \\e[33;1m${name}\\e[0m"
+ else
+ fail "Failed decoding hexadecimal secret \\e[33;1m${name}\\e[0m:\\n$(sed 's/^/... /g' "${errors}")"
+ fi
+ ;;
+ @url@*)
+ url=$(echo "$value" | cut -c6-)
+ if command -v curl > /dev/null
+ then
+ decoded=$(mktemp)
+ errors=$(mktemp)
+ if curl -s -S -f --connect-timeout 5 -o "${decoded}" "$url" 2> "${errors}"
+ then
+ # shellcheck disable=SC2086
+ export ${name}="$(cat ${decoded})"
+ log_info "Successfully curl'd secret \\e[33;1m${name}\\e[0m"
+ else
+ log_warn "Failed getting secret \\e[33;1m${name}\\e[0m:\\n$(sed 's/^/... /g' "${errors}")"
+ fi
+ elif command -v wget > /dev/null
+ then
+ decoded=$(mktemp)
+ errors=$(mktemp)
+ if wget -T 5 -O "${decoded}" "$url" 2> "${errors}"
+ then
+ # shellcheck disable=SC2086
+ export ${name}="$(cat ${decoded})"
+ log_info "Successfully wget'd secret \\e[33;1m${name}\\e[0m"
+ else
+ log_warn "Failed getting secret \\e[33;1m${name}\\e[0m:\\n$(sed 's/^/... /g' "${errors}")"
+ fi
+ else
+ log_warn "Couldn't get secret \\e[33;1m${name}\\e[0m: no http client found"
+ fi
+ ;;
+ esac
+ }
+
+ function eval_all_secrets() {
+ encoded_vars=$(env | grep -Ev '(^|.*_ENV_)scoped__' | awk -F '=' '/^[a-zA-Z0-9_]*=@(b64|hex|url)@/ {print $1}')
+ for var in $encoded_vars
+ do
+ eval_secret "$var"
+ done
+ }
+
# validates an input GitLab CI YAML file
function ci_lint() {
rc=0
@@ -71,12 +229,18 @@ spec:
exit $rc
}
-gitlab-ci-lint:
- image: registry.hub.docker.com/badouralix/curl-jq:latest
- stage: build
+ unscope_variables
+ eval_all_secrets
+
+.gitlab-ci-base:
before_script:
- !reference [.lint-scripts]
- install_ca_certs "${CUSTOM_CA_CERTS:-$DEFAULT_CA_CERTS}"
+
+gitlab-ci-lint:
+ extends: .gitlab-ci-base
+ stage: build
+ image: registry.hub.docker.com/badouralix/curl-jq:latest
script:
- ci_lint
rules: