Merge branch 'master' into 'master'

feat: add vault variant for validation template

Closes #6

See merge request to-be-continuous/tools/gitlab-ci!4

.gitlab-ci.yml

@@ -5,4 +5,4 @@ stages:
   - build
 
 variables:
-  GITLAB_CI_FILES: "templates/*.yml"
+  GITLAB_CI_FILES: "templates/extract.yml templates/validation.yml"

README.md

@@ -24,3 +24,61 @@ stages:
 variables:
   GITLAB_CI_FILES: "templates/*.yml"
 ```
+## Variants
+
+The default validation template is designed to work on untagged runners, without any proxy configuration using Docker images
+from the internet and using Default Trusted Certificate Authorities.
+
+Nevertheless there are template variants available to cover specific cases.
+
+### Vault variant
+
+This variant allows delegating your secrets management to a [Vault](https://www.vaultproject.io/) server.
+
+#### Configuration
+
+In order to be able to communicate with the Vault server, the variant requires the additional configuration parameters:
+
+| Input / Variable | Description                            | Default value     |
+| ----------------- | -------------------------------------- | ----------------- |
+| `TBC_VAULT_IMAGE` | The [Vault Secrets Provider](https://gitlab.com/to-be-continuous/tools/vault-secrets-provider) image to use (can be overridden) | `registry.gitlab.com/to-be-continuous/tools/vault-secrets-provider:latest` |
+| `vault-base-url` / `VAULT_BASE_URL` | The Vault server base API url          | **must be defined** |
+| `vault-oidc-aud` / `VAULT_OIDC_AUD` | The `aud` claim for the JWT | `$CI_SERVER_URL` |
+| :lock: `VAULT_ROLE_ID`   | The [AppRole](https://www.vaultproject.io/docs/auth/approle) RoleID | _none_ |
+| :lock: `VAULT_SECRET_ID` | The [AppRole](https://www.vaultproject.io/docs/auth/approle) SecretID | _none_ |
+
+By default, the variant will authentifacte using a [JWT ID token](https://docs.gitlab.com/ee/ci/secrets/id_token_authentication.html). To use [AppRole](https://www.vaultproject.io/docs/auth/approle) instead the `VAULT_ROLE_ID` and `VAULT_SECRET_ID` should be defined as secret project variables.
+
+#### Usage
+
+Then you may retrieve any of your secret(s) from Vault using the following syntax:
+
+```text
+@url@http://vault-secrets-provider/api/secrets/{secret_path}?field={field}
+```
+
+With:
+
+| Parameter                        | Description                            |
+| -------------------------------- | -------------------------------------- |
+| `secret_path` (_path parameter_) | this is your secret location in the Vault server |
+| `field` (_query parameter_)      | parameter to access a single basic field from the secret JSON payload |
+
+#### Example
+
+```yaml
+include:
+  # main template
+  - component: $CI_SERVER_FQDN/to-be-continuous/tools/gitlab-ci/validation@master
+  # Vault variant
+  - component: $CI_SERVER_FQDN/to-be-continuous/tools/gitlab-ci/validation-vault@master
+    inputs:
+      # audience claim for JWT
+      vault-oidc-aud: "https://vault.acme.host"
+      vault-base-url: "https://vault.acme.host/v1"
+
+variables:
+  # Secrets managed by Vault
+  GITLAB_TOKEN: "@url@http://vault-secrets-provider/api/secrets/b7ecb6ebabc231/my-infra/gitlab?field=token"
+```
+

templates/validation-vault.yml

+ # ====================================================================================================================
+ # === Vault template variant
+ # ====================================================================================================================
+spec:
+  inputs:
+    vault-base-url:
+      description: The Vault server base API url
+      default: ''
+    vault-oidc-aud:
+      description: the `aud` claim for the JWT
+      default: $CI_SERVER_URL
+---
+variables:
+  # variabilized vault-secrets-provider image
+  TBC_VAULT_IMAGE: registry.gitlab.com/to-be-continuous/tools/vault-secrets-provider:latest
+  # variables have to be explicitly declared in the YAML to be exported to the service
+  VAULT_ROLE_ID: $VAULT_ROLE_ID
+  VAULT_SECRET_ID: $VAULT_SECRET_ID
+  VAULT_OIDC_AUD: $[[ inputs.vault-oidc-aud ]]
+
+  VAULT_BASE_URL: $[[ inputs.vault-base-url ]]
+
+.gitlab-ci-base:
+  services:
+    - name: "$TBC_VAULT_IMAGE"
+      alias: "vault-secrets-provider"
+  variables:
+    VAULT_JWT_TOKEN: "$VAULT_JWT_TOKEN"
+  id_tokens:
+    VAULT_JWT_TOKEN:
+      aud: "$VAULT_OIDC_AUD"

templates/validation.yml

@@ -12,6 +12,11 @@
     echo -e "[\e[1;91mERROR\e[0m] $*" >&2
   }
 
+  function fail() {
+    log_error "$*"
+    exit 1
+  }
+
   function install_ca_certs() {
     certs=$1
     if [[ -z "$certs" ]]
@@ -30,6 +35,159 @@
     fi
   }
 
+  function unscope_variables() {
+    _scoped_vars=$(env | awk -F '=' "/^scoped__[a-zA-Z0-9_]+=/ {print \$1}" | sort)
+    if [[ -z "$_scoped_vars" ]]; then return; fi
+    log_info "Processing scoped variables..."
+    for _scoped_var in $_scoped_vars
+    do
+      _fields=${_scoped_var//__/:}
+      _condition=$(echo "$_fields" | cut -d: -f3)
+      case "$_condition" in
+      if) _not="";;
+      ifnot) _not=1;;
+      *)
+        log_warn "... unrecognized condition \\e[1;91m$_condition\\e[0m in \\e[33;1m${_scoped_var}\\e[0m"
+        continue
+      ;;
+      esac
+      _target_var=$(echo "$_fields" | cut -d: -f2)
+      _cond_var=$(echo "$_fields" | cut -d: -f4)
+      _cond_val=$(eval echo "\$${_cond_var}")
+      _test_op=$(echo "$_fields" | cut -d: -f5)
+      case "$_test_op" in
+      defined)
+        if [[ -z "$_not" ]] && [[ -z "$_cond_val" ]]; then continue; 
+        elif [[ "$_not" ]] && [[ "$_cond_val" ]]; then continue; 
+        fi
+        ;;
+      equals|startswith|endswith|contains|in|equals_ic|startswith_ic|endswith_ic|contains_ic|in_ic)
+        # comparison operator
+        # sluggify actual value
+        _cond_val=$(echo "$_cond_val" | tr '[:punct:]' '_')
+        # retrieve comparison value
+        _cmp_val_prefix="scoped__${_target_var}__${_condition}__${_cond_var}__${_test_op}__"
+        _cmp_val=${_scoped_var#"$_cmp_val_prefix"}
+        # manage 'ignore case'
+        if [[ "$_test_op" == *_ic ]]
+        then
+          # lowercase everything
+          _cond_val=$(echo "$_cond_val" | tr '[:upper:]' '[:lower:]')
+          _cmp_val=$(echo "$_cmp_val" | tr '[:upper:]' '[:lower:]')
+        fi
+        case "$_test_op" in
+        equals*)
+          if [[ -z "$_not" ]] && [[ "$_cond_val" != "$_cmp_val" ]]; then continue; 
+          elif [[ "$_not" ]] && [[ "$_cond_val" == "$_cmp_val" ]]; then continue; 
+          fi
+          ;;
+        startswith*)
+          if [[ -z "$_not" ]] && [[ "$_cond_val" != "$_cmp_val"* ]]; then continue; 
+          elif [[ "$_not" ]] && [[ "$_cond_val" == "$_cmp_val"* ]]; then continue; 
+          fi
+          ;;
+        endswith*)
+          if [[ -z "$_not" ]] && [[ "$_cond_val" != *"$_cmp_val" ]]; then continue; 
+          elif [[ "$_not" ]] && [[ "$_cond_val" == *"$_cmp_val" ]]; then continue; 
+          fi
+          ;;
+        contains*)
+          if [[ -z "$_not" ]] && [[ "$_cond_val" != *"$_cmp_val"* ]]; then continue; 
+          elif [[ "$_not" ]] && [[ "$_cond_val" == *"$_cmp_val"* ]]; then continue; 
+          fi
+          ;;
+        in*)
+          if [[ -z "$_not" ]] && [[ "__${_cmp_val}__" != *"__${_cond_val}__"* ]]; then continue; 
+          elif [[ "$_not" ]] && [[ "__${_cmp_val}__" == *"__${_cond_val}__"* ]]; then continue; 
+          fi
+          ;;
+        esac
+        ;;
+      *)
+        log_warn "... unrecognized test operator \\e[1;91m${_test_op}\\e[0m in \\e[33;1m${_scoped_var}\\e[0m"
+        continue
+        ;;
+      esac
+      # matches
+      _val=$(eval echo "\$${_target_var}")
+      log_info "... apply \\e[32m${_target_var}\\e[0m from \\e[32m\$${_scoped_var}\\e[0m${_val:+ (\\e[33;1moverwrite\\e[0m)}"
+      _val=$(eval echo "\$${_scoped_var}")
+      export "${_target_var}"="${_val}"
+    done
+    log_info "... done"
+  }
+
+  # evaluate and export a secret
+  # - $1: secret variable name
+  function eval_secret() {
+    name=$1
+    value=$(eval echo "\$${name}")
+    case "$value" in
+    @b64@*)
+      decoded=$(mktemp)
+      errors=$(mktemp)
+      if echo "$value" | cut -c6- | base64 -d > "${decoded}" 2> "${errors}"
+      then
+        # shellcheck disable=SC2086
+        export ${name}="$(cat ${decoded})"
+        log_info "Successfully decoded base64 secret \\e[33;1m${name}\\e[0m"
+      else
+        fail "Failed decoding base64 secret \\e[33;1m${name}\\e[0m:\\n$(sed 's/^/... /g' "${errors}")"
+      fi
+      ;;
+    @hex@*)
+      decoded=$(mktemp)
+      errors=$(mktemp)
+      if echo "$value" | cut -c6- | sed 's/\([0-9A-F]\{2\}\)/\\\\x\1/gI' | xargs printf > "${decoded}" 2> "${errors}"
+      then
+        # shellcheck disable=SC2086
+        export ${name}="$(cat ${decoded})"
+        log_info "Successfully decoded hexadecimal secret \\e[33;1m${name}\\e[0m"
+      else
+        fail "Failed decoding hexadecimal secret \\e[33;1m${name}\\e[0m:\\n$(sed 's/^/... /g' "${errors}")"
+      fi
+      ;;
+    @url@*)
+      url=$(echo "$value" | cut -c6-)
+      if command -v curl > /dev/null
+      then
+        decoded=$(mktemp)
+        errors=$(mktemp)
+        if curl -s -S -f --connect-timeout 5 -o "${decoded}" "$url" 2> "${errors}"
+        then
+          # shellcheck disable=SC2086
+          export ${name}="$(cat ${decoded})"
+          log_info "Successfully curl'd secret \\e[33;1m${name}\\e[0m"
+        else
+          log_warn "Failed getting secret \\e[33;1m${name}\\e[0m:\\n$(sed 's/^/... /g' "${errors}")"
+        fi
+      elif command -v wget > /dev/null
+      then
+        decoded=$(mktemp)
+        errors=$(mktemp)
+        if wget -T 5 -O "${decoded}" "$url" 2> "${errors}"
+        then
+          # shellcheck disable=SC2086
+          export ${name}="$(cat ${decoded})"
+          log_info "Successfully wget'd secret \\e[33;1m${name}\\e[0m"
+        else
+          log_warn "Failed getting secret \\e[33;1m${name}\\e[0m:\\n$(sed 's/^/... /g' "${errors}")"
+        fi
+      else
+        log_warn "Couldn't get secret \\e[33;1m${name}\\e[0m: no http client found"
+      fi
+      ;;
+    esac
+  }
+
+  function eval_all_secrets() {
+    encoded_vars=$(env | grep -Ev '(^|.*_ENV_)scoped__' | awk -F '=' '/^[a-zA-Z0-9_]*=@(b64|hex|url)@/ {print $1}')
+    for var in $encoded_vars
+    do
+      eval_secret "$var"
+    done
+  }
+
   # validates an input GitLab CI YAML file
   function ci_lint() {
     rc=0
@@ -50,12 +208,18 @@
     exit $rc
   }
 
-gitlab-ci-lint:
-  image: registry.hub.docker.com/badouralix/curl-jq:latest
-  stage: build
+  unscope_variables
+  eval_all_secrets
+
+.gitlab-ci-base:
   before_script:
     - !reference [.lint-scripts]
     - install_ca_certs "${CUSTOM_CA_CERTS:-$DEFAULT_CA_CERTS}"
+
+gitlab-ci-lint:
+  extends: .gitlab-ci-base
+  stage: build
+  image: registry.hub.docker.com/badouralix/curl-jq:latest
   script:
     - ci_lint
   rules: