From bf678c4d300c77f3805b016f9a74a29b4c520af5 Mon Sep 17 00:00:00 2001
From: Xavier FRANCOIS <xavier.francois@orange.com>
Date: Thu, 2 Nov 2023 17:50:10 +0000
Subject: [PATCH] feat: add vault variant
---
README.md | 60 +++++++++++++++++++++++
kicker.json | 33 +++++++++++++
templates/gitlab-ci-python-vault.yml | 22 +++++++++
templates/gitlab-ci-python.yml | 73 ++++++++++++++++++++++++++++
4 files changed, 188 insertions(+)
create mode 100644 templates/gitlab-ci-python-vault.yml
diff --git a/README.md b/README.md
index c2400ea..cbb70e6 100644
--- a/README.md
+++ b/README.md
@@ -387,3 +387,63 @@ variables:
PIP_INDEX_URL: "https://cache.corp/repository/pypi/simple"
PIP_EXTRA_INDEX_URL: "${CI_SERVER_PROTOCOL}://gitlab-ci-token:${CI_JOB_TOKEN}@${CI_SERVER_HOST}:${CI_SERVER_PORT}/api/v4/groups/<group-id>/-/packages/pypi/simple"
```
+
+
+## Variants
+
+The Python template can be used in conjunction with template variants to cover specific cases.
+
+### Vault variant
+
+This variant allows delegating your secrets management to a [Vault](https://www.vaultproject.io/) server.
+
+#### Configuration
+
+In order to be able to communicate with the Vault server, the variant requires the additional configuration parameters:
+
+| Name | Description | Default value |
+| ----------------- | -------------------------------------- | ----------------- |
+| `TBC_VAULT_IMAGE` | The [Vault Secrets Provider](https://gitlab.com/to-be-continuous/tools/vault-secrets-provider) image to use (can be overridden) | `$CI_REGISTRY/to-be-continuous/tools/vault-secrets-provider:master` |
+| `VAULT_BASE_URL` | The Vault server base API url | _none_ |
+| `VAULT_OIDC_AUD` | The `aud` claim for the JWT | `$CI_SERVER_URL` |
+| :lock: `VAULT_ROLE_ID` | The [AppRole](https://www.vaultproject.io/docs/auth/approle) RoleID | **must be defined** |
+| :lock: `VAULT_SECRET_ID` | The [AppRole](https://www.vaultproject.io/docs/auth/approle) SecretID | **must be defined** |
+
+#### Usage
+
+Then you may retrieve any of your secret(s) from Vault using the following syntax:
+
+```text
+@url@http://vault-secrets-provider/api/secrets/{secret_path}?field={field}
+```
+
+With:
+
+| Name | Description |
+| -------------------------------- | -------------------------------------- |
+| `secret_path` (_path parameter_) | this is your secret location in the Vault server |
+| `field` (_query parameter_) | parameter to access a single basic field from the secret JSON payload |
+
+#### Example
+
+```yaml
+include:
+ # main template
+ - project: 'to-be-continuous/python'
+ ref: '6.3.5'
+ file: '/templates/gitlab-ci-python.yml'
+ # Vault variant
+ - project: 'to-be-continuous/python'
+ ref: '6.3.5'
+ file: '/templates/gitlab-ci-python-vault.yml'
+
+variables:
+ # audience claim for JWT
+ VAULT_OIDC_AUD: "https://vault.acme.host"
+ # Secrets managed by Vault
+ GIT_PASSWORD: "@url@http://vault-secrets-provider/api/secrets/b7ecb6ebabc231/git/semantic-release?field=group-access-token"
+ GIT_PRIVATE_KEY: "@url@http://vault-secrets-provider/api/secrets/b7ecb6ebabc231/git/semantic-release?field=private-key"
+ PYTHON_REPOSITORY_PASSWORD: "@url@http://vault-secrets-provider/api/secrets/b7ecb6ebabc231/pip-repo/repository?field=password"
+ VAULT_BASE_URL: "https://vault.acme.host/v1"
+ # $VAULT_ROLE_ID and $VAULT_SECRET_ID defined as a secret CI/CD variable
+```
diff --git a/kicker.json b/kicker.json
index 0aa1a7c..3a2046d 100644
--- a/kicker.json
+++ b/kicker.json
@@ -241,5 +241,38 @@
}
]
}
+ ],
+ "variants": [
+ {
+ "id": "vault",
+ "name": "Vault",
+ "description": "Retrieve secrets from a [Vault](https://www.vaultproject.io/) server",
+ "template_path": "templates/gitlab-ci-python-vault.yml",
+ "variables": [
+ {
+ "name": "TBC_VAULT_IMAGE",
+ "description": "The [Vault Secrets Provider](https://gitlab.com/to-be-continuous/tools/vault-secrets-provider) image to use",
+ "default": "$CI_REGISTRY/to-be-continuous/tools/vault-secrets-provider:master",
+ "advanced": true
+ },
+ {
+ "name": "VAULT_BASE_URL",
+ "description": "The Vault server base API url",
+ "mandatory": true
+ },
+ {
+ "name": "VAULT_ROLE_ID",
+ "description": "The [AppRole](https://www.vaultproject.io/docs/auth/approle) RoleID",
+ "mandatory": true,
+ "secret": true
+ },
+ {
+ "name": "VAULT_SECRET_ID",
+ "description": "The [AppRole](https://www.vaultproject.io/docs/auth/approle) SecretID",
+ "mandatory": true,
+ "secret": true
+ }
+ ]
+ }
]
}
diff --git a/templates/gitlab-ci-python-vault.yml b/templates/gitlab-ci-python-vault.yml
new file mode 100644
index 0000000..eb8a84d
--- /dev/null
+++ b/templates/gitlab-ci-python-vault.yml
@@ -0,0 +1,22 @@
+# =====================================================================================================================
+# === Vault template variant
+# =====================================================================================================================
+variables:
+ # variabilized vault-secrets-provider image
+ TBC_VAULT_IMAGE: "$CI_REGISTRY/to-be-continuous/tools/vault-secrets-provider:master"
+ # variables have to be explicitly declared in the YAML to be exported to the service
+ VAULT_ROLE_ID: "$VAULT_ROLE_ID"
+ VAULT_SECRET_ID: "$VAULT_SECRET_ID"
+ VAULT_OIDC_AUD: "$CI_SERVER_URL"
+
+.python-base:
+ services:
+ - name: "$TBC_TRACKING_IMAGE"
+ command: ["--service", "python", "6.3.5"]
+ - name: "$TBC_VAULT_IMAGE"
+ alias: "vault-secrets-provider"
+ variables:
+ VAULT_JWT_TOKEN: "$VAULT_JWT_TOKEN"
+ id_tokens:
+ VAULT_JWT_TOKEN:
+ aud: "$VAULT_OIDC_AUD"
diff --git a/templates/gitlab-ci-python.yml b/templates/gitlab-ci-python.yml
index b473f97..4941a8d 100644
--- a/templates/gitlab-ci-python.yml
+++ b/templates/gitlab-ci-python.yml
@@ -263,6 +263,78 @@ variables:
log_info "... done"
}
+ # evaluate and export a secret
+ # - $1: secret variable name
+ function eval_secret() {
+ name=$1
+ value=$(eval echo "\$${name}")
+ case "$value" in
+ @b64@*)
+ decoded=$(mktemp)
+ errors=$(mktemp)
+ if echo "$value" | cut -c6- | base64 -d > "${decoded}" 2> "${errors}"
+ then
+ # shellcheck disable=SC2086
+ export ${name}="$(cat ${decoded})"
+ log_info "Successfully decoded base64 secret \\e[33;1m${name}\\e[0m"
+ else
+ fail "Failed decoding base64 secret \\e[33;1m${name}\\e[0m:\\n$(sed 's/^/... /g' "${errors}")"
+ fi
+ ;;
+ @hex@*)
+ decoded=$(mktemp)
+ errors=$(mktemp)
+ if echo "$value" | cut -c6- | sed 's/\([0-9A-F]\{2\}\)/\\\\x\1/gI' | xargs printf > "${decoded}" 2> "${errors}"
+ then
+ # shellcheck disable=SC2086
+ export ${name}="$(cat ${decoded})"
+ log_info "Successfully decoded hexadecimal secret \\e[33;1m${name}\\e[0m"
+ else
+ fail "Failed decoding hexadecimal secret \\e[33;1m${name}\\e[0m:\\n$(sed 's/^/... /g' "${errors}")"
+ fi
+ ;;
+ @url@*)
+ url=$(echo "$value" | cut -c6-)
+ if command -v curl > /dev/null
+ then
+ decoded=$(mktemp)
+ errors=$(mktemp)
+ if curl -s -S -f --connect-timeout 5 -o "${decoded}" "$url" 2> "${errors}"
+ then
+ # shellcheck disable=SC2086
+ export ${name}="$(cat ${decoded})"
+ log_info "Successfully curl'd secret \\e[33;1m${name}\\e[0m"
+ else
+ log_warn "Failed getting secret \\e[33;1m${name}\\e[0m:\\n$(sed 's/^/... /g' "${errors}")"
+ fi
+ elif command -v wget > /dev/null
+ then
+ decoded=$(mktemp)
+ errors=$(mktemp)
+ if wget -T 5 -O "${decoded}" "$url" 2> "${errors}"
+ then
+ # shellcheck disable=SC2086
+ export ${name}="$(cat ${decoded})"
+ log_info "Successfully wget'd secret \\e[33;1m${name}\\e[0m"
+ else
+ log_warn "Failed getting secret \\e[33;1m${name}\\e[0m:\\n$(sed 's/^/... /g' "${errors}")"
+ fi
+ else
+ log_warn "Couldn't get secret \\e[33;1m${name}\\e[0m: no http client found"
+ fi
+ ;;
+ esac
+ }
+
+ function eval_all_secrets() {
+ encoded_vars=$(env | grep -v '^scoped__' | awk -F '=' '/^[a-zA-Z0-9_]*=@(b64|hex|url)@/ {print $1}')
+ for var in $encoded_vars
+ do
+ eval_secret "$var"
+ done
+ }
+
+
function guess_build_system() {
case "${PYTHON_BUILD_SYSTEM:-auto}" in
auto)
@@ -538,6 +610,7 @@ variables:
}
unscope_variables
+ eval_all_secrets
# ENDSCRIPT
--
GitLab