From 22003d195670f3d923f31014636485bc783068bb Mon Sep 17 00:00:00 2001
From: Bertrand Goareguer <bertrand.goareguer@gmail.com>
Date: Thu, 2 May 2024 08:16:27 +0000
Subject: [PATCH] feat: add support for slim and alpine Python images + change
default base image
BREAKING CHANGE: the default base image has been changed to python:debian-slim
---
README.md | 9 +++--
kicker.json | 7 ++--
templates/gitlab-ci-python.yml | 71 ++++++++++++++++++++++++++--------
3 files changed, 63 insertions(+), 24 deletions(-)
diff --git a/README.md b/README.md
index 7614c45..c44ab6a 100644
--- a/README.md
+++ b/README.md
@@ -17,7 +17,7 @@ include:
- component: gitlab.com/to-be-continuous/python/gitlab-ci-python@6.11.1
# 2: set/override component inputs
inputs:
- image: registry.hub.docker.com/library/python:3.10
+ image: registry.hub.docker.com/library/python:3.12-slim
pytest-enabled: true
```
@@ -34,7 +34,7 @@ include:
variables:
# 2: set/override template variables
- PYTHON_IMAGE: registry.hub.docker.com/library/python:3.10
+ PYTHON_IMAGE: registry.hub.docker.com/library/python:3.12-slim
PYTEST_ENABLED: "true"
```
@@ -44,7 +44,7 @@ The Python template uses some global configuration used throughout all jobs.
| Input / Variable | Description | Default value |
| -------------------- | ------------------------------------------------------------------------------------- | ------------------ |
-| `image` / `PYTHON_IMAGE` | The Docker image used to run Python <br/>:warning: **set the version required by your project** | `registry.hub.docker.com/library/python:3` |
+| `image` / `PYTHON_IMAGE` | The Docker image used to run Python <br/>:warning: **set the version required by your project** | `registry.hub.docker.com/library/python:3-slim` |
| `project-dir` / `PYTHON_PROJECT_DIR` | Python project root directory | `.` |
| `build-system` / `PYTHON_BUILD_SYSTEM`| Python build-system to use to install dependencies, build and package the project (see below) | `auto` (auto-detect) |
| `PIP_INDEX_URL` | Python repository url | _none_ |
@@ -243,6 +243,7 @@ It is bound to the `test` stage, and uses the following variables:
| Input / Variable | Description | Default value |
| ---------------- | ----------------------------------------------------------------------- | ----------------- |
| `trivy-enabled` / `PYTHON_TRIVY_ENABLED` | Set to `true` to enable Trivy job | _none_ (disabled) |
+| `trivy-dist-url` / `PYTHON_TRIVY_DIST_URL` | Url to the `tar.gz` package for `linux_amd64` of Trivy to use (ex: `https://github.com/aquasecurity/trivy/releases/download/v0.51.1/trivy_0.51.1_Linux-64bit.tar.gz`)<br/>_When unset, the latest version will be used_ | _none_ |
| `trivy-args` / `PYTHON_TRIVY_ARGS` | Additional [Trivy CLI options](https://aquasecurity.github.io/trivy/v0.21.1/getting-started/cli/fs/) | `--vuln-type library` |
In addition to a textual report in the console, this job produces the following reports, kept for one day:
@@ -561,7 +562,7 @@ include:
- component: gitlab.com/to-be-continuous/python/gitlab-ci-python@6.11.1
# 2: set/override component inputs
inputs:
- image: registry.hub.docker.com/library/python:3.10
+ image: registry.hub.docker.com/library/python:3.12-slim
pytest-enabled: true
- component: gitlab.com/to-be-continuous/python/gitlab-ci-python-gcp@6.11.1
diff --git a/kicker.json b/kicker.json
index 86ea97e..5040247 100644
--- a/kicker.json
+++ b/kicker.json
@@ -10,7 +10,7 @@
{
"name": "PYTHON_IMAGE",
"description": "The Docker image used to run Python - **set the version required by your project**",
- "default": "registry.hub.docker.com/library/python:3"
+ "default": "registry.hub.docker.com/library/python:3-slim"
},
{
"name": "PYTHON_PROJECT_DIR",
@@ -138,9 +138,8 @@
"enable_with": "PYTHON_TRIVY_ENABLED",
"variables": [
{
- "name": "PYTHON_TRIVY_IMAGE",
- "description": "The Docker image used to run Trivy",
- "default": "registry.hub.docker.com/aquasec/trivy:latest",
+ "name": "PYTHON_TRIVY_DIST_URL",
+ "description": "Url to the `tar.gz` package for `linux_amd64` of Trivy to use\n\n_When unset, the latest version will be used_",
"advanced": true
},
{
diff --git a/templates/gitlab-ci-python.yml b/templates/gitlab-ci-python.yml
index 87c9d88..ffc9250 100644
--- a/templates/gitlab-ci-python.yml
+++ b/templates/gitlab-ci-python.yml
@@ -17,7 +17,7 @@ spec:
inputs:
image:
description: The Docker image used to run Python - **set the version required by your project**
- default: registry.hub.docker.com/library/python:3
+ default: registry.hub.docker.com/library/python:3-slim
project-dir:
description: Python project root directory
default: .
@@ -100,9 +100,12 @@ spec:
description: Enable Trivy
type: boolean
default: false
- trivy-image:
- description: The Docker image used to run Trivy
- default: registry.hub.docker.com/aquasec/trivy:latest
+ trivy-dist-url:
+ description: |-
+ Url to the `tar.gz` package for `linux_amd64` of Trivy to use
+
+ _When unset, the latest version will be used_
+ default: ''
trivy-args:
description: Additional [Trivy CLI options](https://aquasecurity.github.io/trivy/v0.21.1/getting-started/cli/fs/)
default: --vuln-type library
@@ -246,7 +249,7 @@ variables:
# Trivy tool
PYTHON_TRIVY_ENABLED: $[[ inputs.trivy-enabled ]]
- PYTHON_TRIVY_IMAGE: $[[ inputs.trivy-image ]]
+ PYTHON_TRIVY_DIST_URL: $[[ inputs.trivy-dist-url ]]
PYTHON_TRIVY_ARGS: $[[ inputs.trivy-args ]]
PYTHON_SBOM_NAME: $[[ inputs.sbom-name ]]
@@ -508,6 +511,18 @@ variables:
else
log_warn "Failed getting secret \\e[33;1m${name}\\e[0m:\\n$(sed 's/^/... /g' "${errors}")"
fi
+ elif command -v python3 > /dev/null
+ then
+ decoded=$(mktemp)
+ errors=$(mktemp)
+ # shellcheck disable=SC2086
+ if python3 -c "import urllib.request ; urllib.request.urlretrieve(\"$url\",\"${decoded}\")" > "${errors}" 2>&1
+ then
+ export ${name}="$(cat ${decoded})"
+ log_info "Successfully fetched secret \\e[33;1m${name}\\e[0m"
+ else
+ log_warn "Failed getting secret \\e[33;1m${name}\\e[0m:\\n$(sed 's/^/... /g' "${errors}")"
+ fi
else
log_warn "Couldn't get secret \\e[33;1m${name}\\e[0m: no http client found"
fi
@@ -530,7 +545,7 @@ variables:
if ! dpkg --status "$@" > /dev/null
then
apt-get update
- apt-get install --yes --quiet "$@"
+ apt-get install --no-install-recommends --yes --quiet "$@"
fi
elif command -v apk > /dev/null
then
@@ -610,7 +625,7 @@ variables:
}
function maybe_install_poetry() {
- if [[ "$PYTHON_BUILD_SYSTEM" == poetry* ]] && ! command -v poetry > /dev/null
+ if [[ "$PYTHON_BUILD_SYSTEM" =~ ^poetry.* ]] && ! command -v poetry > /dev/null
then
# shellcheck disable=SC2086
pip install ${PIP_OPTS} "$PYTHON_BUILD_SYSTEM"
@@ -1136,13 +1151,27 @@ py-trivy:
# force no dependencies
dependencies: []
script:
- - maybe_install_packages wget apt-transport-https gnupg lsb-release
- mkdir -p -m 777 reports
- install_requirements
- - wget -qO - https://aquasecurity.github.io/trivy-repo/deb/public.key | apt-key add -
- - echo "deb https://aquasecurity.github.io/trivy-repo/deb $(lsb_release -sc) main" | tee -a /etc/apt/sources.list.d/trivy.list
- - apt-get update
- - apt-get install trivy
+ - |
+ if [[ -z "$PYTHON_TRIVY_DIST_URL" ]]
+ then
+ log_info "Trivy version unset: retrieve latest version..."
+ trivy_version=$(python3 -c 'import urllib.request;url="https://github.com/aquasecurity/trivy/releases/latest";opener=urllib.request.build_opener(type("NoRedirection", (urllib.request.HTTPErrorProcessor,), {"http_response": lambda self, req, resp: resp, "https_response": lambda self, req, resp: resp})());req=urllib.request.Request(url, method="HEAD");print(opener.open(req).headers.get("Location").split("/tag/v")[1])')
+ PYTHON_TRIVY_DIST_URL="https://github.com/aquasecurity/trivy/releases/download/v${trivy_version}/trivy_${trivy_version}_Linux-64bit.tar.gz"
+ log_info "... use latest Trivy version: \\e[32m$PYTHON_TRIVY_DIST_URL\\e[0m"
+ fi
+ python_trivy="$XDG_CACHE_HOME/trivy-$(echo "$PYTHON_TRIVY_DIST_URL" | md5sum | cut -d" " -f1)"
+ if [[ -f $python_trivy ]]
+ then
+ log_info "Trivy found in cache (\\e[32m$PYTHON_TRIVY_DIST_URL\\e[0m): reuse"
+ else
+ log_info "Trivy not found in cache (\\e[32m$PYTHON_TRIVY_DIST_URL\\e[0m): download"
+ python3 -c 'import urllib.request;urllib.request.urlretrieve("'$PYTHON_TRIVY_DIST_URL'","trivy.tar.gz")'
+ tar zxf trivy.tar.gz trivy
+ mkdir -p $XDG_CACHE_HOME
+ mv ./trivy $python_trivy
+ fi
- |
if [[ "$PYTHON_BUILD_SYSTEM" == poetry* ]]
then
@@ -1162,15 +1191,15 @@ py-trivy:
log_warn "The ./requirements.txt file does not match the ./reports/requirements.txt file generated via pip freeze. Make sure to include all dependencies with pinned versions in ./requirements.txt and re-commit the file."
fi
fi
- if [ $(trivy fs ${PYTHON_TRIVY_ARGS} --format table --exit-code 0 ./reports/ | grep -c "Number of language-specific files: 0") -eq 1 ]; then
+ if [ $($python_trivy fs ${PYTHON_TRIVY_ARGS} --format table --exit-code 0 ./reports/ | grep -c "Number of language-specific files: 0") -eq 1 ]; then
log_error "Could not find a file listing all dependencies with their versions."
exit 1
fi
if [[ "$DEFECTDOJO_TRIVY_REPORTS" ]]
then
- trivy fs ${PYTHON_TRIVY_ARGS} --exit-code 0 --list-all-pkgs --format json --output reports/py-trivy.trivy.json ./reports/
+ $python_trivy fs ${PYTHON_TRIVY_ARGS} --exit-code 0 --list-all-pkgs --format json --output reports/py-trivy.trivy.json ./reports/
fi
- trivy fs ${PYTHON_TRIVY_ARGS} --format table ./reports/
+ $python_trivy fs ${PYTHON_TRIVY_ARGS} --format table ./reports/
artifacts:
name: "$CI_JOB_NAME artifacts from $CI_PROJECT_NAME on $CI_COMMIT_REF_SLUG"
expire_in: 1 day
@@ -1203,7 +1232,7 @@ py-sbom:
if [[ -z "$PYTHON_SBOM_SYFT_URL" ]]
then
log_info "Syft version unset: retrieve latest version..."
- syft_version=$(curl -Ls -o /dev/null -w %{url_effective} https://github.com/anchore/syft/releases/latest | grep -o '[^/v]*$')
+ syft_version=$(python3 -c 'import urllib.request;print(urllib.request.urlopen("https://github.com/anchore/syft/releases/latest").url)' | grep -o '[^/v]*$')
PYTHON_SBOM_SYFT_URL="https://github.com/anchore/syft/releases/download/v${syft_version}/syft_${syft_version}_linux_amd64.tar.gz"
log_info "... use latest Syft version: \\e[32m$PYTHON_SBOM_SYFT_URL\\e[0m"
fi
@@ -1235,7 +1264,13 @@ py-sbom:
py-release:
extends: .python-base
stage: publish
+ before_script:
+ - !reference [.python-base, before_script]
+ # install git and OpenSSH
+ - maybe_install_packages git openssh-client
script:
+ - apt-get update
+ - apt-get install -y git openssh-client
- git config --global user.email "$GITLAB_USER_EMAIL"
- git config --global user.name "$GITLAB_USER_LOGIN"
- git checkout -B $CI_COMMIT_REF_NAME
@@ -1257,6 +1292,10 @@ py-release:
py-publish:
extends: .python-base
stage: publish
+ before_script:
+ - !reference [.python-base, before_script]
+ # install curl (to decode @url@ variables)
+ - maybe_install_packages curl
script:
- py_publish
artifacts:
--
GitLab