diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3c8636d746428e7b0fb962a85fa602673a0d2e89..8fc38591aa49fd6269a6bf673d5021548c728932 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -12,22 +12,23 @@ include: inputs: kicker-validation-job-tags: ["docker"] schema-base-url: "https://git.code.tecnalia.com/api/v4/projects/smartdatalab%2Fpublic%2Fci-cd-components%2Fkicker/repository/files" + yajsv-image: cicd-docker-dev.artifact.tecnalia.com/yajsv:latest - component: git.code.tecnalia.com/smartdatalab/public/ci-cd-components/bash/gitlab-ci-bash@master inputs: bash-shellcheck-job-tags: ["docker"] + shellcheck-files: "*.sh" - component: git.code.tecnalia.com/smartdatalab/public/ci-cd-components/semantic-release/gitlab-ci-semrel@master inputs: semantic-release-job-tags: ["docker"] -stages: - - build - - publish - variables: GITLAB_CI_FILES: "templates/gitlab-ci-k8s.yml" - BASH_SHELLCHECK_FILES: "*.sh" GIT_STRATEGY: clone +stages: + - build + - publish + semantic-release: rules: # on production branch(es): auto if SEMREL_AUTO_RELEASE_ENABLED diff --git a/CHANGELOG.md b/CHANGELOG.md index cdfc72495f2e9666a9997de8c51e9ef0609716ab..8ff4ffc39d62e39bdc3a84f000e94adf8d6471e7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,9 @@ -## [6.1.4](https://git.code.tecnalia.com/smartdatalab/public/ci-cd-components/kubernetes/compare/6.1.3...6.1.4) (2024-06-05) +# [6.2.0](https://gitlab.com/to-be-continuous/kubernetes/compare/6.1.4...6.2.0) (2024-07-12) -### Bug Fixes +### Features -* skip k8s-jobs when *_SPACE variables are empty ([83404bf](https://git.code.tecnalia.com/smartdatalab/public/ci-cd-components/kubernetes/commit/83404bf4cccfd113b3e7daf7adb657e70e0c3b1a)) +* improve error message on missing yaml ([37afac9](https://gitlab.com/to-be-continuous/kubernetes/commit/37afac95aacf834e784d8fd312573a7e08706672)) ## [6.1.4](https://gitlab.com/to-be-continuous/kubernetes/compare/6.1.3...6.1.4) (2024-06-02) diff --git a/README.md b/README.md index 16244d35f7e1a788679eb8486552e029ebb16d1e..253ae5e912ceb0d830a700ea014bd1a64067f0e3 100644 --- a/README.md +++ b/README.md @@ -6,17 +6,17 @@ or [Kustomize](https://kubernetes.io/docs/tasks/manage-kubernetes-objects/kustom ## Usage -This template can be used both as a [CI/CD component](https://docs.gitlab.com/ee/ci/components/#use-a-component-in-a-cicd-configuration) +This template can be used both as a [CI/CD component](https://docs.gitlab.com/ee/ci/components/#use-a-component) or using the legacy [`include:project`](https://docs.gitlab.com/ee/ci/yaml/index.html#includeproject) syntax. ### Use as a CI/CD component -Add the following to your `gitlab-ci.yml`: +Add the following to your `.gitlab-ci.yml`: ```yaml include: # 1: include the component - - component: gitlab.com/to-be-continuous/kubernetes/gitlab-ci-k8s@6.1.4 + - component: $CI_SERVER_FQDN/to-be-continuous/kubernetes/gitlab-ci-k8s@6.3.0 # 2: set/override component inputs inputs: # ⚠ this is only an example @@ -29,13 +29,13 @@ include: ### Use as a CI/CD template (legacy) -Add the following to your `gitlab-ci.yml`: +Add the following to your `.gitlab-ci.yml`: ```yaml include: # 1: include the template - project: 'to-be-continuous/kubernetes' - ref: '6.1.4' + ref: '6.3.0' file: '/templates/gitlab-ci-k8s.yml' variables: @@ -279,7 +279,7 @@ by using available environment variables: * `${k8s_namespace}`: the Kubernetes namespace currently used for deployment/cleanup * `${hostname}`: the environment hostname, extracted from the current environment url (after late variable expansion - see below) 2. any [GitLab CI variable](https://docs.gitlab.com/ee/ci/variables/predefined_variables.html) -3. any [custom variable](https://docs.gitlab.com/ee/ci/variables/#add-a-cicd-variable-to-a-project) +3. any [custom variable](https://docs.gitlab.com/ee/ci/variables/#for-a-project) (ex: `${SECRET_TOKEN}` that you have set in your project CI/CD variables) While your scripts may simply use any of those variables, your Kubernetes and Kustomize resources can use **variable substitution** @@ -323,7 +323,7 @@ data: The K8S template supports two ways of providing your environments url: * a **static way**: when the environments url can be determined in advance, probably because you're exposing your routes through a DNS you manage, -* a [**dynamic way**](https://docs.gitlab.com/ee/ci/environments/#set-dynamic-environment-urls-after-a-job-finishes): when the url cannot be known before the +* a [**dynamic way**](https://docs.gitlab.com/ee/ci/environments/#set-a-dynamic-environment-url): when the url cannot be known before the deployment job is executed. The **static way** can be implemented simply by setting the appropriate configuration variable(s) depending on the environment (see environments configuration chapters): @@ -368,7 +368,7 @@ You may also add and propagate your own custom variables, by pushing them to the Here are some advices about your **secrets** (variables marked with a :lock:): -1. Manage them as [project or group CI/CD variables](https://docs.gitlab.com/ee/ci/variables/#add-a-cicd-variable-to-a-project): +1. Manage them as [project or group CI/CD variables](https://docs.gitlab.com/ee/ci/variables/#for-a-project): * [**masked**](https://docs.gitlab.com/ee/ci/variables/#mask-a-cicd-variable) to prevent them from being inadvertently displayed in your job logs, * [**protected**](https://docs.gitlab.com/ee/ci/variables/#protected-cicd-variables) if you want to secure some secrets @@ -538,12 +538,12 @@ With: ```yaml include: # main template - - component: gitlab.com/to-be-continuous/kubernetes/gitlab-ci-k8s@6.1.4 + - component: $CI_SERVER_FQDN/to-be-continuous/kubernetes/gitlab-ci-k8s@6.3.0 inputs: # ⚠ oc-container image (includes required curl) kubectl-image: registry.hub.docker.com/docker.io/appuio/oc:v4.14 # Vault variant - - component: gitlab.com/to-be-continuous/kubernetes/gitlab-ci-k8s-vault@6.1.4 + - component: $CI_SERVER_FQDN/to-be-continuous/kubernetes/gitlab-ci-k8s-vault@6.3.0 inputs: # audience claim for JWT vault-oidc-aud: "https://vault.acme.host" @@ -555,3 +555,64 @@ variables: K8S_DEFAULT_KUBE_CONFIG: "@url@http://vault-secrets-provider/api/secrets/b7ecb6ebabc231/my-app/kubernetes/noprod?field=kube_config" K8S_PROD_KUBE_CONFIG: "@url@http://vault-secrets-provider/api/secrets/b7ecb6ebabc231/my-app/kubernetes/prod?field=kube_config" ``` + +### Google Cloud variant + +This variant uses [Application Default Credentials][gcp-adc] through the `GOOGLE_APPLICATION_CREDENTIALS` variable using Workload Identity federation. + +List of requirements before using this variant: + +1. You must have a Workload Identity Federation Pool and Provider configured, +2. You must have a Service Account with the `roles/iam.workloadIdentityUser` IAM role + granted to the Workload Identity [principal][gcp-iam-principals] matching your Gitlab project or group, +3. Optionally, you can set the `GOOGLE_CLOUD_PROJECT` template variable + to define the default Google Cloud project. +4. You must have create a `kubeconfig.yaml` configuration which [enable application default credentials for kubectl](https://cloud.google.com/kubernetes-engine/docs/how-to/api-server-authentication#environments-without-gcloud) + +The Gitlab documentation has some [details about Workload Identity Federation integration][gcp-gitlab-wif]. + +This [blog post about OIDC impersonation through Workload Identify Federation][gcp-wif-example] might also be of help. + +[gcp-adc]: https://cloud.google.com/docs/authentication/client-libraries +[gcp-provider]: https://registry.terraform.io/providers/hashicorp/google/latest/docs/guides/provider_reference#running-terraform-outside-of-google-cloud +[gcp-iam-principals]: https://cloud.google.com/iam/docs/principal-identifiers +[gcp-gitlab-wif]: https://docs.gitlab.com/ee/ci/cloud_services/google_cloud/ +[gcp-wif-example]: https://blog.salrashid.dev/articles/2021/understanding_workload_identity_federation/#oidc-impersonated + +#### Configuration + +The variant requires the additional configuration parameters: + +| Input / Variable | Description | Default value | +| ----------------- | -------------------------------------- | ----------------- | +| `gcp-oidc-aud` / `GCP_OIDC_AUD` | The `aud` claim for the JWT token | `$CI_SERVER_URL` | +| `gcp-oidc-provider` / `GCP_OIDC_PROVIDER` | Default Workload Identity Provider associated with GitLab to [authenticate with OpenID Connect](https://docs.gitlab.com/ee/ci/cloud_services/google_cloud/) | _none_ | +| `gcp-oidc-account` / `GCP_OIDC_ACCOUNT` | Default Service Account to which impersonate with OpenID Connect authentication | _none_ | +| `gcp-review-oidc-provider` / `GCP_REVIEW_OIDC_PROVIDER` | Workload Identity Provider associated with GitLab to [authenticate with OpenID Connect](https://docs.gitlab.com/ee/ci/cloud_services/google_cloud/) on `review` environment _(only define to override default)_ | _none_ | +| `gcp-review-oidc-account` / `GCP_REVIEW_OIDC_ACCOUNT` | Service Account to which impersonate with OpenID Connect authentication on `review` environment _(only define to override default)_ | _none_ | +| `gcp-integ-oidc-provider` / `GCP_INTEG_OIDC_PROVIDER` | Workload Identity Provider associated with GitLab to [authenticate with OpenID Connect](https://docs.gitlab.com/ee/ci/cloud_services/google_cloud/) on `integration` environment _(only define to override default)_ | _none_ | +| `gcp-integ-oidc-account` / `GCP_INTEG_OIDC_ACCOUNT` | Service Account to which impersonate with OpenID Connect authentication on `integration` environment _(only define to override default)_ | _none_ | +| `gcp-staging-oidc-provider` / `GCP_STAGING_OIDC_PROVIDER` | Workload Identity Provider associated with GitLab to [authenticate with OpenID Connect](https://docs.gitlab.com/ee/ci/cloud_services/google_cloud/) on `staging` environment _(only define to override default)_ | _none_ | +| `gcp-staging-oidc-account` / `GCP_STAGING_OIDC_ACCOUNT` | Service Account to which impersonate with OpenID Connect authentication on `staging` environment _(only define to override default)_ | _none_ | +| `gcp-prod-oidc-provider` / `GCP_PROD_OIDC_PROVIDER` | Workload Identity Provider associated with GitLab to [authenticate with OpenID Connect](https://docs.gitlab.com/ee/ci/cloud_services/google_cloud/) on `production` environment _(only define to override default)_ | _none_ | +| `gcp-prod-oidc-account` / `GCP_PROD_OIDC_ACCOUNT` | Service Account to which impersonate with OpenID Connect authentication on `production` environment _(only define to override default)_ | _none_ | +| `kubectl-image` / `K8S_KUBECTL_IMAGE` | The Docker image used to run Kubernetes `kubectl` commands on [GKE](https://cloud.google.com/kubernetes-engine/docs) | `gcr.io/google.com/cloudsdktool/cloud-sdk:latest` | + +#### Example + +With a common default `GCP_OIDC_PROVIDER` and `GCP_OIDC_ACCOUNT` configuration for non-prod environments, and a specific one for production: + +```yaml +include: + # main template + - component: $CI_SERVER_FQDN/to-be-continuous/kubernetes/gitlab-ci-k8s@6.3.0 + # Google Cloud variant + - component: $CI_SERVER_FQDN/to-be-continuous/kubernetes/gitlab-ci-k8ss-gcp@6.3.0 + inputs: + # common OIDC config for non-prod envs + gcp-oidc-provider: "projects/<gcp_nonprod_proj_id>/locations/global/workloadIdentityPools/<pool_id>/providers/<provider_id>" + gcp-oidc-account: "<name>@$<gcp_nonprod_proj_id>.iam.gserviceaccount.com" + # specific OIDC config for prod + gcp-prod-oidc-provider: "projects/<gcp_prod_proj_id>/locations/global/workloadIdentityPools/<pool_id>/providers/<provider_id>" + gcp-prod-oidc-account: "<name>@$<gcp_prod_proj_id>.iam.gserviceaccount.com" +``` diff --git a/kicker.json b/kicker.json index 7293e69f97d5fbb543333d3c5978e698d3ebc3c7..005e2f05fbd02e2bda2e6c984f214e77bf0614f8 100644 --- a/kicker.json +++ b/kicker.json @@ -345,6 +345,73 @@ "secret": true } ] + }, + { + "id": "gcp-auth-provider", + "name": "Google Cloud", + "description": "This variant uses [Application Default Credentials][gcp-adc] through the `GOOGLE_APPLICATION_CREDENTIALS` variable using Workload Identity federation.", + "template_path": "templates/gitlab-ci-k8s-gcp.yml", + "variables": [ + { + "name": "GCP_OIDC_AUD", + "description": "The `aud` claim for the JWT token _(only required for [OIDC authentication](https://docs.gitlab.com/ee/ci/cloud_services/aws/))_", + "default": "$CI_SERVER_URL", + "advanced": true + }, + { + "name": "GCP_OIDC_ACCOUNT", + "description": "Default Service Account to which impersonate with OpenID Connect authentication" + }, + { + "name": "GCP_OIDC_PROVIDER", + "description": "Default Workload Identity Provider associated with GitLab to [authenticate with OpenID Connect](https://docs.gitlab.com/ee/ci/cloud_services/google_cloud/)" + }, + { + "name": "GCP_REVIEW_OIDC_ACCOUNT", + "description": "Service Account to which impersonate with OpenID Connect authentication on `review` environment", + "advanced": true + }, + { + "name": "GCP_REVIEW_OIDC_PROVIDER", + "description": "Workload Identity Provider associated with GitLab to [authenticate with OpenID Connect](https://docs.gitlab.com/ee/ci/cloud_services/google_cloud/) on `review` environment", + "advanced": true + }, + { + "name": "GCP_INTEG_OIDC_ACCOUNT", + "description": "Service Account to which impersonate with OpenID Connect authentication on `integration` environment", + "advanced": true + }, + { + "name": "GCP_INTEG_OIDC_PROVIDER", + "description": "Workload Identity Provider associated with GitLab to [authenticate with OpenID Connect](https://docs.gitlab.com/ee/ci/cloud_services/google_cloud/) on `integration` environment", + "advanced": true + }, + { + "name": "GCP_STAGING_OIDC_ACCOUNT", + "description": "Service Account to which impersonate with OpenID Connect authentication on `staging` environment", + "advanced": true + }, + { + "name": "GCP_STAGING_OIDC_PROVIDER", + "description": "Workload Identity Provider associated with GitLab to [authenticate with OpenID Connect](https://docs.gitlab.com/ee/ci/cloud_services/google_cloud/) on `staging` environment", + "advanced": true + }, + { + "name": "GCP_PROD_OIDC_ACCOUNT", + "description": "Service Account to which impersonate with OpenID Connect authentication on `production` environment", + "advanced": true + }, + { + "name": "GCP_PROD_OIDC_PROVIDER", + "description": "Workload Identity Provider associated with GitLab to [authenticate with OpenID Connect](https://docs.gitlab.com/ee/ci/cloud_services/google_cloud/) on `production` environment", + "advanced": true + }, + { + "name": "K8S_KUBECTL_IMAGE", + "description": "The Docker image used to run Kubernetes `kubectl` commands on [GKE](https://cloud.google.com/kubernetes-engine/docs)", + "default": "gcr.io/google.com/cloudsdktool/cloud-sdk:latest" + } + ] } ] } diff --git a/templates/gitlab-ci-k8s-gcp.yml b/templates/gitlab-ci-k8s-gcp.yml new file mode 100644 index 0000000000000000000000000000000000000000..5bed69f219b0e192e18721b893345d4ef214e302 --- /dev/null +++ b/templates/gitlab-ci-k8s-gcp.yml @@ -0,0 +1,116 @@ +# ===================================================================================================================== +# === Google Cloud template variant +# ===================================================================================================================== +spec: + inputs: + kubectl-image: + description: The Docker image used to run Kubernetes `kubectl` commands on [GKE](https://cloud.google.com/kubernetes-engine/docs) + default: gcr.io/google.com/cloudsdktool/cloud-sdk:latest + gcp-oidc-aud: + description: The `aud` claim for the JWT token + default: $CI_SERVER_URL + gcp-oidc-account: + description: Default Service Account to which impersonate with OpenID Connect authentication + default: '' + gcp-oidc-provider: + description: Default Workload Identity Provider associated with GitLab to [authenticate with OpenID Connect](https://docs.gitlab.com/ee/ci/cloud_services/google_cloud/) + default: '' + gcp-review-oidc-account: + description: Service Account to which impersonate with OpenID Connect authentication on `review` environment + default: '' + gcp-review-oidc-provider: + description: Workload Identity Provider associated with GitLab to [authenticate with OpenID Connect](https://docs.gitlab.com/ee/ci/cloud_services/google_cloud/) on `review` environment + default: '' + gcp-integ-oidc-account: + description: Service Account to which impersonate with OpenID Connect authentication on `integration` environment + default: '' + gcp-integ-oidc-provider: + description: Workload Identity Provider associated with GitLab to [authenticate with OpenID Connect](https://docs.gitlab.com/ee/ci/cloud_services/google_cloud/) on `integration` environment + default: '' + gcp-staging-oidc-account: + description: Service Account to which impersonate with OpenID Connect authentication on `staging` environment + default: '' + gcp-staging-oidc-provider: + description: Workload Identity Provider associated with GitLab to [authenticate with OpenID Connect](https://docs.gitlab.com/ee/ci/cloud_services/google_cloud/) on `staging` environment + default: '' + gcp-prod-oidc-account: + description: Service Account to which impersonate with OpenID Connect authentication on `production` environment + default: '' + gcp-prod-oidc-provider: + description: Workload Identity Provider associated with GitLab to [authenticate with OpenID Connect](https://docs.gitlab.com/ee/ci/cloud_services/google_cloud/) on `production` environment + default: '' +--- +variables: + # variabilized gcp-auth-provider image + GCP_OIDC_AUD: $[[ inputs.gcp-oidc-aud ]] + GCP_OIDC_ACCOUNT: $[[ inputs.gcp-oidc-account ]] + GCP_OIDC_PROVIDER: $[[ inputs.gcp-oidc-provider ]] + GCP_REVIEW_OIDC_ACCOUNT: $[[ inputs.gcp-review-oidc-account ]] + GCP_REVIEW_OIDC_PROVIDER: $[[ inputs.gcp-review-oidc-provider ]] + GCP_INTEG_OIDC_ACCOUNT: $[[ inputs.gcp-integ-oidc-account ]] + GCP_INTEG_OIDC_PROVIDER: $[[ inputs.gcp-integ-oidc-provider ]] + GCP_STAGING_OIDC_ACCOUNT: $[[ inputs.gcp-staging-oidc-account ]] + GCP_STAGING_OIDC_PROVIDER: $[[ inputs.gcp-staging-oidc-provider ]] + GCP_PROD_OIDC_ACCOUNT: $[[ inputs.gcp-prod-oidc-account ]] + GCP_PROD_OIDC_PROVIDER: $[[ inputs.gcp-prod-oidc-provider ]] + + K8S_KUBECTL_IMAGE: $[[ inputs.kubectl-image ]] + +.gcp-provider-auth: + before_script: + - echo "Installing GCP authentication with env GOOGLE_APPLICATION_CREDENTIALS file" + - echo $GCP_JWT > "$CI_BUILDS_DIR/.auth_token.jwt" + - |- + if [[ "$ENV_TYPE" ]] + then + case "$ENV_TYPE" in + review*) + env_prefix=REVIEW;; + integ*) + env_prefix=INTEG;; + staging*) + env_prefix=STAGING;; + prod*) + env_prefix=PROD;; + *) + ;; + esac + env_oidc_provider=$(eval echo "\$GCP_${env_prefix}_OIDC_PROVIDER") + env_oidc_account=$(eval echo "\$GCP_${env_prefix}_OIDC_ACCOUNT") + fi + oidc_provider="${env_oidc_provider:-$GCP_OIDC_PROVIDER}" + oidc_account="${env_oidc_account:-$GCP_OIDC_ACCOUNT}" + - |- + cat << EOF > "$CI_BUILDS_DIR/google_application_credentials.json" + { + "type": "external_account", + "audience": "//iam.googleapis.com/${oidc_provider}", + "subject_token_type": "urn:ietf:params:oauth:token-type:jwt", + "token_url": "https://sts.googleapis.com/v1/token", + "credential_source": { + "file": "$CI_BUILDS_DIR/.auth_token.jwt" + }, + "service_account_impersonation_url": "https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/${oidc_account}:generateAccessToken" + } + EOF + - export GOOGLE_APPLICATION_CREDENTIALS="$CI_BUILDS_DIR/google_application_credentials.json" + +.k8s-deploy: + id_tokens: + GCP_JWT: + aud: "$GCP_OIDC_AUD" + before_script: + - !reference [.k8s-scripts] + - !reference [.gcp-provider-auth, before_script] + - install_ca_certs "${CUSTOM_CA_CERTS:-$DEFAULT_CA_CERTS}" + - k8s_login + +.k8s-cleanup: + id_tokens: + GCP_JWT: + aud: "$GCP_OIDC_AUD" + before_script: + - !reference [.k8s-scripts] + - !reference [.gcp-provider-auth, before_script] + - install_ca_certs "${CUSTOM_CA_CERTS:-$DEFAULT_CA_CERTS}" + - k8s_login \ No newline at end of file diff --git a/templates/gitlab-ci-k8s-vault.yml b/templates/gitlab-ci-k8s-vault.yml index 871f8eff1ed4255706f2801f9a43c92430024bbb..3c3fc5881702d3072409a892f7d7cb4a5a3de52b 100644 --- a/templates/gitlab-ci-k8s-vault.yml +++ b/templates/gitlab-ci-k8s-vault.yml @@ -22,7 +22,7 @@ variables: .k8s-base: services: - name: "$TBC_TRACKING_IMAGE" - command: ["--service", "--port", "8082", "kubernetes", "6.1.4"] + command: ["--service", "--port", "8082", "kubernetes", "6.3.0"] - name: "$TBC_VAULT_IMAGE" alias: "vault-secrets-provider" variables: diff --git a/templates/gitlab-ci-k8s.yml b/templates/gitlab-ci-k8s.yml index d9b41c55739d2fe4ec3728afdaa3d3d5b9e7bed2..6ac6f9a1fd940a68491eb9159c604b9f3c638a56 100644 --- a/templates/gitlab-ci-k8s.yml +++ b/templates/gitlab-ci-k8s.yml @@ -548,7 +548,7 @@ stages: deploymentfile=$(ls -1 "$K8S_SCRIPTS_DIR/deployment-${environment_type}.yml" 2>/dev/null || ls -1 "$K8S_SCRIPTS_DIR/deployment.yml" 2>/dev/null || echo "") if [[ -z "$deploymentfile" ]] then - log_error "--- deployment file not found" + log_error "--- deployment file not found. Expected '$K8S_SCRIPTS_DIR/deployment-${environment_type}.yml' or '$K8S_SCRIPTS_DIR/deployment.yml'." exit 1 fi @@ -764,7 +764,7 @@ stages: deploymentfile=$(ls -1 "$K8S_SCRIPTS_DIR/deployment-${environment_type}.yml" 2>/dev/null || ls -1 "$K8S_SCRIPTS_DIR/deployment.yml" 2>/dev/null || echo "") if [[ -z "$deploymentfile" ]] then - log_error "--- deployment file not found" + log_error "--- deployment file not found. Expected '$K8S_SCRIPTS_DIR/deployment-${environment_type}.yml' or '$K8S_SCRIPTS_DIR/deployment.yml'." exit 1 fi @@ -790,7 +790,7 @@ stages: entrypoint: [""] services: - name: "$TBC_TRACKING_IMAGE" - command: ["--service", "kubernetes", "6.1.4"] + command: ["--service", "kubernetes", "6.3.0"] before_script: - !reference [.k8s-scripts] - install_ca_certs "${CUSTOM_CA_CERTS:-$DEFAULT_CA_CERTS}"