From 96c2920b8da419cedefe67e9cd497df146f94b49 Mon Sep 17 00:00:00 2001 From: Timothy Stone <gitlab@petmystone.com> Date: Sun, 2 Jul 2023 10:33:31 +0000 Subject: [PATCH] feat(jib): add Maven Jib variant to build container images for your Java applications --- README.md | 87 ++++++++++++ kicker.json | 95 +++++++++++++ templates/gitlab-ci-maven-jib.yml | 212 ++++++++++++++++++++++++++++++ 3 files changed, 394 insertions(+) create mode 100644 templates/gitlab-ci-maven-jib.yml diff --git a/README.md b/README.md index 81bbf05..d6b0458 100644 --- a/README.md +++ b/README.md @@ -362,3 +362,90 @@ Note that the password should be an access token with `write_repository` scope a ... </scm> ``` + +## Variants + +### Jib variant + +This variant builds optimized Docker and OCI images for your Java applications (without deep mastery of Docker best-practices) with [Jib](https://github.com/GoogleContainerTools/jib). + +#### Configuration + +This variant uses the [Jib Maven Plugin](https://github.com/GoogleContainerTools/jib/tree/master/jib-maven-plugin) to build a Docker container and publish that container to a registry. + +##### Images and registries config + +| Name | Description | Default value | +| -------------------------------------------- | ------------------------ | ------------------------------------------------- | +| `MAVEN_JIB_SNAPSHOT_IMAGE` | Container snapshot image | `$CI_REGISTRY_IMAGE/snapshot:$CI_COMMIT_REF_SLUG` | +| `MAVEN_JIB_RELEASE_IMAGE` | Container release image | `$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME` | +| :lock: `MAVEN_JIB_REGISTRY_USER` | Default registry username for image registry | `$CI_REGISTRY_USER` _(default GitLab registry user)_ | +| :lock: `MAVEN_JIB_REGISTRY_PASSWORD` | Default registry password for image registry | `$CI_REGISTRY_PASSWORD` _(default GitLab registry password)_ | +| :lock: `MAVEN_JIB_REGISTRY_SNAPSHOT_USER` | Registry username for snapshot image registry.<br/> Only set if different from default. | _none_ | +| :lock: `MAVEN_JIB_REGISTRY_SNAPSHOT_PASSWORD`| Registry password for snapshot image registry.<br/> Only set if different from default. | _none_ | +| :lock: `MAVEN_JIB_REGISTRY_RELEASE_USER` | Registry username for release image registry.<br/> Only set if different from default. | _none_ | +| :lock: `MAVEN_JIB_REGISTRY_RELEASE_PASSWORD` | Registry password for release image registry.<br/> Only set if different from default. | _none_ | + +The template uses GitLab registries and authentication defaults. See the Docker template for configuring alternate [registries and credentials](https://gitlab.com/to-be-continuous/docker#registries-and-credentials). + +##### Security scanning and reporting + +| Name | Description | Default value | +| -------------------------------------- | ------------------------ | ------------------------------------------------- | +| `MAVEN_SBOM_IMAGE` | The image used to perform and complete the Security Bill of Materials | `registry.hub.docker.com/anchore/syft:debug` | +| `MAVEN_SBOM_OPTS` | SBOM options to complete the Security Bill of Materials | `--catalogers rpm-db-cataloger,alpmdb-cataloger,apkdb-cataloger,dpkgdb-cataloger,portage-catalogerE` | +| `MAVEN_TRIVY_SECURITY_LEVEL_THRESHOLD` | Security level which fails the `mvn-trivy` job | `UNKNOWN,LOW,MEDIUM,HIGH,CRITICAL` | +| `MAVEN_TRIVY_IMAGE` | The image to perform container security scanning | `registry.hub.docker.com/aquasec/trivy:latest` | +| `MAVEN_TRIVY_ARGS` | Arguments for the execution of Trivy | `--ignore-unfixed --vuln-type os` | + + +##### Jib build and publish configuration + +Tho `mvn-build` job produces and uploads the container snapshot to the registry provided in `$MAVEN_JIB_SNAPSHOT_IMAGE` via the Jib `build` goal, e.g., `mvn verify com.google.cloud.tools:jib-maven-plugin:build`. + +Publishing the release image follows the two-phase Maven release and deploy model. The `mvn-release` job is responsible for versioning and tagging +the `pom.xml` using the Maven Release Plugin, e.g., `release:prepare`. The `mvn-deploy-release` job deploys, or "releases," the container via [`skopeo copy` arguments](https://github.com/containers/skopeo/blob/main/docs/skopeo-copy.1.md) to the provided registry in `$MAVEN_JIB_RELEASE_IMAGE`. + +| Name | Description | Default value | +| --------------------------------- | ---------------------------------------------------------- | ----------------- | +| `MAVEN_SKOPEO_IMAGE` | The image used to publish docker image with Skopeo | `quay.io/skopeo/stable:latest` | +| `MAVEN_JIB_BUILD_ARGS` | [Jib Maven Plugin arguments](https://github.com/GoogleContainerTools/jib/tree/master/jib-maven-plugin#extended-usage). | `-Djib.to.image=$MAVEN_JIB_SNAPSHOT_IMAGE` | +| `MAVEN_JIB_PUBLISH_ARGS` | Additional [`skopeo copy` arguments](https://github.com/containers/skopeo/blob/main/docs/skopeo-copy.1.md), e.g., `--additional-tag=strings` | _none_ | +| `MAVEN_JIB_PROD_PUBLISH_STRATEGY` | Defines the publish to production strategy for `mvn-release` and `mvn-deploy-release` jobs. One of `none`, `auto`, `manual`. | `manual` | + +#### Usage + +See the Jib [Quickstart](https://github.com/GoogleContainerTools/jib/tree/master/jib-maven-plugin) +for minimal guidance on the use of the plugin for your project. If you're here, you probably have a use case, +i.e., you kicked off a [JHipster](https://jhipster.tech/) project and found Jib pre-configured for containerization. + +The template uses GitLab registries and authentication defaults. See the Docker template for configuring alternate [registries and credentials](https://gitlab.com/to-be-continuous/docker#registries-and-credentials). + +`$MAVEN_JIB_BUILD_ARGS` sets the snapshot container publish registry via a System Property, `-Djib.to.image=$MAVEN_JIB_SNAPSHOT_IMAGE`. This can be declaratively provided in the POM configuration for the Jib plugin and omitted, e.g., `#MAVEN_JIB_BUILD_ARGS` or `$MAVEN_JIB_BUILD_ARGS: ""` in the project `.gitlab-ci.yml`. +This is advanced usage and should be understood in context of how `skopeo copy` works in the production pipeline. + +#### Registry authorization + +The variant tooling, Jib and Skopeo, support [Docker configuration files (default)](https://github.com/GoogleContainerTools/jib/tree/master/jib-maven-plugin#using-docker-configuration-files). +Jib supports additional authentication methods, including [credential helpers](https://github.com/GoogleContainerTools/jib/tree/master/jib-maven-plugin#using-docker-credential-helpers), +[the POM and CLI](https://github.com/GoogleContainerTools/jib/tree/master/jib-maven-plugin#using-specific-credentials), +and [even Maven Settings](https://github.com/GoogleContainerTools/jib/tree/master/jib-maven-plugin#using-specific-credentials), e.g., `.m2/settings.xml`. + +All authentication methods should use masked GitLab environment variables. + +#### Example + +```yaml +include: + # main template + - project: 'to-be-continuous/maven' + ref: '3.5.0' + file: '/templates/gitlab-ci-maven.yml' + # Jib is implemented as an extension to Maven, and uses supporting features of the TBC Maven template + - project: 'to-be-continuous/maven' + ref: '3.5.0' + file: '/templates/gitlab-ci-maven-jib.yml' + +variables: +``` + diff --git a/kicker.json b/kicker.json index b968e39..57b9223 100644 --- a/kicker.json +++ b/kicker.json @@ -197,5 +197,100 @@ } ] } + ], + "variants": [ + { + "id": "jib", + "name": "Jib", + "description": "Build Docker and OCI images for your Java applications with [Jib](https://github.com/GoogleContainerTools/jib)", + "template_path": "templates/gitlab-ci-maven-jib.yml", + "features": [ + { + "id": "mvn-trivy", + "name": "Maven Trivy", + "description": "[Trivy](https://github.com/aquasecurity/trivy) vulnerability analysis", + "disable_with": "MAVEN_TRIVY_DISABLED", + "variables": [ + { + "name": "MAVEN_TRIVY_IMAGE", + "description": "The docker image used to scan images with Trivy", + "default": "registry.hub.docker.com/aquasec/trivy:latest", + "advanced": true + }, + { + "name": "MAVEN_TRIVY_ADDR", + "type": "url", + "description": "The Trivy server address" + }, + { + "name": "MAVEN_TRIVY_SECURITY_LEVEL_THRESHOLD", + "type": "enum", + "values": ["UNKNOWN,LOW,MEDIUM,HIGH,CRITICAL", "LOW,MEDIUM,HIGH,CRITICAL", "MEDIUM,HIGH,CRITICAL", "HIGH,CRITICAL", "CRITICAL"], + "description": "Severities of vulnerabilities to be displayed (comma separated values: `UNKNOWN`, `LOW`, `MEDIUM`, `HIGH`, `CRITICAL`)", + "default": "UNKNOWN,LOW,MEDIUM,HIGH,CRITICAL" + }, + { + "name": "MAVEN_TRIVY_ARGS", + "description": "Additional `trivy client` arguments", + "default": "--ignore-unfixed --vuln-type os", + "advanced": true + } + ] + }, + { + "id": "mvn-sbom", + "name": "Maven Software Bill of Materials", + "description": "This job generates a file listing all dependencies using [syft](https://github.com/anchore/syft)", + "disable_with": "MAVEN_SBOM_DISABLED", + "variables": [ + { + "name": "MAVEN_SBOM_IMAGE", + "default": "registry.hub.docker.com/anchore/syft:debug", + "advanced": true + }, + { + "name": "MAVEN_SBOM_OPTS", + "description": "Options for syft used for SBOM analysis", + "default": "--catalogers rpm-db-cataloger,alpmdb-cataloger,apkdb-cataloger,dpkgdb-cataloger,portage-cataloger", + "advanced": true + } + ] + } + ], + "variables": [ + { + "name": "MAVEN_JIB_SNAPSHOT_IMAGE", + "description": "Maven Jib Snapshot image", + "default": "$CI_REGISTRY_IMAGE/snapshot:$CI_COMMIT_REF_SLUG" + }, + { + "name": "MAVEN_JIB_RELEASE_IMAGE", + "description": "Maven Jib Release image", + "default": "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME" + }, + { + "name": "MAVEN_SKOPEO_IMAGE", + "description": "The image used to publish images with Skopeo", + "default": "quay.io/skopeo/stable:latest", + "advanced": true + }, + { + "name": "MAVEN_JIB_BUILD_ARGS", + "description": "[Jib Maven Plugin arguments](https://github.com/GoogleContainerTools/jib/tree/master/jib-maven-plugin#extended-usage)", + "default": "-Djib.to.image=$MAVEN_JIB_SNAPSHOT_IMAGE" + }, + { + "name": "MAVEN_JIB_PROD_PUBLISH_STRATEGY", + "description": "Defines the publish to production strategy.", + "type": "enum", + "values": ["none", "manual", "auto"], + "default": "manual" + }, + { + "name": "MAVEN_JIB_PUBLISH_ARGS", + "description": "Additional [`skopeo copy` arguments](https://github.com/containers/skopeo/blob/master/docs/skopeo-copy.1.md#options)" + } + ] + } ] } diff --git a/templates/gitlab-ci-maven-jib.yml b/templates/gitlab-ci-maven-jib.yml new file mode 100644 index 0000000..832645f --- /dev/null +++ b/templates/gitlab-ci-maven-jib.yml @@ -0,0 +1,212 @@ +# ===================================================================================================================== +# === JIB template variant +# ===================================================================================================================== +variables: + MAVEN_SBOM_IMAGE: "registry.hub.docker.com/anchore/syft:debug" + MAVEN_SBOM_OPTS: "--catalogers rpm-db-cataloger,alpmdb-cataloger,apkdb-cataloger,dpkgdb-cataloger,portage-cataloger" + MAVEN_TRIVY_SECURITY_LEVEL_THRESHOLD: "UNKNOWN,LOW,MEDIUM,HIGH,CRITICAL" + MAVEN_TRIVY_IMAGE: "registry.hub.docker.com/aquasec/trivy:latest" + MAVEN_TRIVY_ARGS: "--ignore-unfixed --vuln-type os" + MAVEN_JIB_SNAPSHOT_IMAGE: "$CI_REGISTRY_IMAGE/snapshot:$CI_COMMIT_REF_SLUG" + MAVEN_JIB_RELEASE_IMAGE: "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME" + MAVEN_JIB_BUILD_ARGS: "-Djib.to.image=$MAVEN_JIB_SNAPSHOT_IMAGE" + MAVEN_JIB_PROD_PUBLISH_STRATEGY: "manual" + MAVEN_SKOPEO_IMAGE: "quay.io/skopeo/stable:latest" + +.mvn-jib-scripts: &mvn-jib-scripts | + # BEGSCRIPT + set -e + + function configure_registries_auth() { + maven_jib_snapshot_authn_token=$(echo -n "${MAVEN_JIB_REGISTRY_SNAPSHOT_USER:-${MAVEN_JIB_REGISTRY_USER:-$CI_REGISTRY_USER}}:${MAVEN_JIB_REGISTRY_SNAPSHOT_PASSWORD:-${MAVEN_JIB_REGISTRY_PASSWORD:-$CI_REGISTRY_PASSWORD}}" | base64 | tr -d '\n') + maven_jib_snapshot_registry_host=$(echo "$MAVEN_JIB_SNAPSHOT_IMAGE" | cut -d/ -f1) + + maven_jib_release_authn_token=$(echo -n "${MAVEN_JIB_REGISTRY_RELEASE_USER:-${MAVEN_JIB_REGISTRY_USER:-$CI_REGISTRY_USER}}:${MAVEN_JIB_REGISTRY_RELEASE_PASSWORD:-${MAVEN_JIB_REGISTRY_PASSWORD:-$CI_REGISTRY_PASSWORD}}" | base64 | tr -d '\n') + maven_jib_release_registry_host=$(echo "$MAVEN_JIB_RELEASE_IMAGE" | cut -d/ -f1) + + maven_jib_snapshot_config_json=$(echo -n "{\"auths\":{\"$maven_jib_snapshot_registry_host\":{\"auth\":\"$maven_jib_snapshot_authn_token\"},\"HttpHeaders\":{\"User-Agent\":\"$USER_AGENT\"}}}") + maven_jib_release_config_json=$(echo -n "{\"auths\":{\"$maven_jib_release_registry_host\":{\"auth\":\"$maven_jib_release_authn_token\"},\"HttpHeaders\":{\"User-Agent\":\"$USER_AGENT\"}}}") + + BUILDTOOL_HOME=${BUILDTOOL_HOME:-$HOME} + # Create Docker auth config (supported by Jib) + mkdir -p "$BUILDTOOL_HOME/.docker" + echo "${maven_jib_snapshot_config_json}" > $BUILDTOOL_HOME/.docker/config.json + echo "${maven_jib_release_config_json}" > $BUILDTOOL_HOME/.docker/release-config.json + + log_info "Registry authentication configured for \\e[33;1m${maven_jib_snapshot_registry_host}\\e[0m" + } + + configure_registries_auth + + # ENDSCRIPT + +mvn-build: + extends: .mvn-base + script: + # initialize Docker auth config + - *mvn-jib-scripts + # build and push snapshot container + - >- + mvn ${TRACE+-X} $MAVEN_CLI_OPTS $mvn_settings_opt $java_proxy_args verify + com.google.cloud.tools:jib-maven-plugin:build + $MAVEN_JIB_BUILD_ARGS + - output_coverage + # create dotenv file + - jib_digest=$(cat target/jib-image.digest | cut -f2 -d':' ) + - jib_repository=${MAVEN_JIB_SNAPSHOT_IMAGE%:*} + - jib_tag=${MAVEN_JIB_SNAPSHOT_IMAGE##*:} + - | + { + echo "jib_image=$MAVEN_JIB_SNAPSHOT_IMAGE" + echo "jib_image_digest=$jib_repository@$jib_digest" + echo "jib_repository=$jib_repository" + echo "jib_tag=$jib_tag" + echo "jib_digest=$jib_digest" + } > jib.env + artifacts: + reports: + dotenv: + - jib.env + +mvn-sbom: + extends: .mvn-base + stage: package-test + image: + name: $MAVEN_SBOM_IMAGE + entrypoint: [""] + # force no dependency + dependencies: [] + script: + - mkdir -p -m 777 reports + - /syft packages $MAVEN_JIB_SNAPSHOT_IMAGE $MAVEN_SBOM_OPTS -o cyclonedx-json=reports/mvn-sbom-${jib_digest}.cyclonedx.json + - chmod a+r reports/mvn-sbom-${jib_digest}.cyclonedx.json + artifacts: + name: "SBOM for container from $CI_PROJECT_NAME on $CI_COMMIT_REF_SLUG" + expire_in: 1 week + when: always + paths: + - "reports/mvn-jib-sbom-*.cyclonedx.json" + reports: + cyclonedx: + - "reports/mvn-jib-sbom-*.cyclonedx.json" + +mvn-trivy: + extends: .mvn-base + stage: package-test + image: + name: $MAVEN_TRIVY_IMAGE + entrypoint: [""] + dependencies: [] + variables: + TRIVY_CACHE_DIR: ".trivycache/" + script: | + # cache cleanup is needed when scanning images with the same tags, it does not remove the database + trivy image --clear-cache + export TRIVY_USERNAME=${MAVEN_JIB_REGISTRY_SNAPSHOT_USER:-${MAVEN_JIB_REGISTRY_USER:-$CI_REGISTRY_USER}} + export TRIVY_PASSWORD=${MAVEN_JIB_REGISTRY_SNAPSHOT_PASSWORD:-${MAVEN_JIB_REGISTRY_PASSWORD:-$CI_REGISTRY_PASSWORD}} + export basename=$(echo "${MAVEN_JIB_SNAPSHOT_IMAGE}" | sed 's|[/:]|_|g') + mkdir -p ./reports + if [[ -z "${MAVEN_TRIVY_ADDR}" ]]; then + log_warn "\\e[93mYou are using Trivy in standalone mode. To get faster scans, consider setting the MAVEN_TRIVY_ADDR variable to the address of a Trivy server. More info here: https://aquasecurity.github.io/trivy/latest/docs/references/modes/client-server/\\e[0m" + trivy image --download-db-only + export trivy_opts="image" + else + log_info "You are using Trivy in client/server mode with the following server: ${MAVEN_TRIVY_ADDR}" + export trivy_opts="image --server ${MAVEN_TRIVY_ADDR}" + fi + # Add common trivy arguments + export trivy_opts="${trivy_opts} --no-progress --severity ${MAVEN_TRIVY_SECURITY_LEVEL_THRESHOLD} ${MAVEN_TRIVY_ARGS}" + # GitLab format (no fail) + trivy ${trivy_opts} --format template --exit-code 0 --template "@/contrib/gitlab.tpl" --output reports/docker-trivy-${basename}.gitlab.json $MAVEN_JIB_SNAPSHOT_IMAGE + # JSON format (no fail) + if [[ "$DEFECTDOJO_TRIVY_REPORTS" ]] + then + trivy ${trivy_opts} --format json --exit-code 0 --output reports/docker-trivy-${basename}.native.json $MAVEN_JIB_SNAPSHOT_IMAGE + fi + # console output (fail) + trivy ${trivy_opts} --format table --exit-code 1 $MAVEN_JIB_SNAPSHOT_IMAGE + artifacts: + when: always + paths: + - "reports/jib-trivy-*" + reports: + container_scanning: "reports/jib-trivy-*.gitlab.json" + cache: + paths: + - .trivycache/ + rules: + - if: '$MAVEN_TRIVY_DISABLED == "true"' + when: never + - !reference [.test-policy, rules] + +mvn-deploy-release: + extends: .mvn-base + image: + name: "$MAVEN_SKOPEO_IMAGE" + entrypoint: [""] + stage: publish + variables: + GIT_STRATEGY: none + script: + # initialize Docker auth config + - *mvn-jib-scripts + - | + if [[ "${SEMREL_INFO_ON}" && "${MVN_SEMREL_RELEASE_DISABLED}" != "true" ]] + then + if [[ -z "${SEMREL_INFO_NEXT_VERSION}" ]] + then + log_warn "[semantic-release] no new version to release: skip" + exit 0 + else + MAVEN_JIB_RELEASE_IMAGE=$(echo "$MAVEN_JIB_RELEASE_IMAGE" | sed "s/\(:.*\)\{0,1\}$/:$SEMREL_INFO_NEXT_VERSION/") + log_info "[semantic-release] new Image tag is set: $MAVEN_JIB_RELEASE_IMAGE" + fi + fi + + if [[ "$MAVEN_JIB_SNAPSHOT_IMAGE" == "$MAVEN_JIB_RELEASE_IMAGE" ]] + then + log_warn "\\e[93mYou should consider distinguishing snapshot and release images as they do not differ. Skipping publish phase as image has already been created by previous job.\\e[0m" + exit 0 + fi + BUILDTOOL_HOME=${BUILDTOOL_HOME:-$HOME} + skopeo copy --src-authfile $BUILDTOOL_HOME/.docker/config.json --dest-authfile $BUILDTOOL_HOME/.docker/release-config.json ${MAVEN_JIB_PUBLISH_ARGS} docker://$MAVEN_JIB_SNAPSHOT_IMAGE docker://$MAVEN_JIB_RELEASE_IMAGE + log_info "Well done, your image is published and can be downloaded by doing: docker pull $MAVEN_JIB_RELEASE_IMAGE" + - jib_digest=$(skopeo inspect --authfile $BUILDTOOL_HOME/.docker/release-config.json --format='{{ .Digest }}' "docker://$MAVEN_JIB_RELEASE_IMAGE") + - jib_repository=${MAVEN_JIB_RELEASE_IMAGE%:*} + - jib_tag=${MAVEN_JIB_RELEASE_IMAGE##*:} + - | + { + echo "jib_image=$MAVEN_JIB_RELEASE_IMAGE" + echo "jib_image_digest=$jib_repository@$jib_digest" + echo "jib_repository=$jib_repository" + echo "jib_tag=$jib_tag" + echo "jib_digest=$jib_digest" + } > jib.env + artifacts: + reports: + dotenv: + - jib.env + rules: + # exclude if $MAVEN_DEPLOY_ENABLED not set + - if: '$MAVEN_DEPLOY_ENABLED != "true"' + when: never + # on tag: if semrel info not enabled or semrel integration disabled + - if: '$CI_COMMIT_TAG' + # exclude non-production branches + - if: '$CI_COMMIT_REF_NAME !~ $PROD_REF' + when: never + # exclude if snapshot is same as release image and semrel info not enabled or semrel integration disabled + - if: '$MAVEN_JIB_SNAPSHOT_IMAGE == $MAVEN_JIB_RELEASE_IMAGE && ($SEMREL_INFO_ON == null || $SEMREL_INFO_ON == "" || $MVN_SEMREL_RELEASE_DISABLED == "true")' + when: never + - if: '$MAVEN_JIB_PROD_PUBLISH_STRATEGY == "manual"' + when: manual + - if: '$MAVEN_JIB_PROD_PUBLISH_STRATEGY == "auto"' + +# ===================================================================================================================== +# === Disable Maven template jobs not required for Docker Jib pipeline +# ===================================================================================================================== + +# mvn-build supersedes - deploys a snapshot container to the registry +mvn-deploy-snapshot: + rules: + - when: never -- GitLab