diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4b7d625190949a0651058e989b92325a407114c4..fc2b17d49084c9198eeec5f839654d97a80af3d7 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,18 @@
-# [4.10.0](https://git.code.tecnalia.dev/smartdatalab/public/ci-cd-components/golang/compare/4.9.2...4.10.0) (2024-08-29)
+## [4.11.1](https://gitlab.com/to-be-continuous/golang/compare/4.11.0...4.11.1) (2024-12-06)
+
+
+### Bug Fixes
+
+* semgrep subdir ([0e26288](https://gitlab.com/to-be-continuous/golang/commit/0e26288dd6b27ce3e4b92ab5c21e6a73d1152902))
+
+# [4.11.0](https://gitlab.com/to-be-continuous/golang/compare/4.10.0...4.11.0) (2024-08-30)
+
+
+### Features
+
+* standard TBC secrets decoding ([89e0f0f](https://gitlab.com/to-be-continuous/golang/commit/89e0f0fb93c2e1a21abcd09977651b567f181ed1))
+
+# [4.10.0](https://gitlab.com/to-be-continuous/golang/compare/4.9.2...4.10.0) (2024-07-05)
### Features
diff --git a/README.md b/README.md
index e887205c3ac1ee96650b2397e04f2490cb54597c..4efd8e7e44c51f24493d50ee557459a848786296 100644
--- a/README.md
+++ b/README.md
@@ -14,7 +14,7 @@ Add the following to your `.gitlab-ci.yml`:
```yaml
include:
# 1: include the component
- - component: $CI_SERVER_FQDN/to-be-continuous/golang/gitlab-ci-golang@4.10.0
+ - component: $CI_SERVER_FQDN/to-be-continuous/golang/gitlab-ci-golang@4.11.1
# 2: set/override component inputs
inputs:
image: "registry.hub.docker.com/library/golang:buster" # ⚠ this is only an example
@@ -28,7 +28,7 @@ Add the following to your `.gitlab-ci.yml`:
include:
# 1: include the template
- project: 'to-be-continuous/golang'
- ref: '4.10.0'
+ ref: '4.11.1'
file: '/templates/gitlab-ci-golang.yml'
variables:
@@ -154,7 +154,7 @@ In addition to a textual report in the console, the test jobs produce the follow
| Report | Format | Usage |
|-----------------------------------------------------|------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------|
| `$GO_PROJECT_DIR/reports/go-test.native.txt` | native Go test report (text) | N/A |
-| `$GO_PROJECT_DIR/reports/go-test.native.json` | native Go test report (json) | [SonarQube integration](https://docs.sonarqube.org/latest/analysis/test-coverage/test-execution-parameters/#header-8) |
+| `$GO_PROJECT_DIR/reports/go-test.native.json` | native Go test report (json) | [SonarQube integration](https://docs.sonarsource.com/sonarqube-server/latest/analyzing-source-code/test-coverage/test-execution-parameters/#go) |
| `$GO_PROJECT_DIR/reports/go-test.xunit.xml` | [xUnit](https://en.wikipedia.org/wiki/XUnit) test report(s) | [GitLab integration](https://docs.gitlab.com/ee/ci/yaml/artifacts_reports.html#artifactsreportsjunit) |
| `$GO_PROJECT_DIR/reports/go-coverage.native.out` | native Go coverage | N/A |
| `$GO_PROJECT_DIR/reports/go-coverage.cobertura.xml` | [Cobertura XML](https://gcovr.com/en/stable/output/cobertura.html) coverage report | [GitLab integration](https://docs.gitlab.com/ee/ci/yaml/artifacts_reports.html#artifactsreportscoverage_report) |
@@ -177,7 +177,35 @@ In addition to a textual report in the console, this job produces the following
| Report | Format | Usage |
|-------------------------------------------------------|----------------------------------------------------------|-------------------------------------------------------------------------------------------------------------|
| `$GO_PROJECT_DIR/reports/go-ci-lint.codeclimate.json` | [Code Climate](https://docs.codeclimate.com/docs/pylint) | [GitLab integration](https://docs.gitlab.com/ee/ci/yaml/artifacts_reports.html#artifactsreportscodequality) |
-| `$GO_PROJECT_DIR/reports/go-ci-lint.checkstyle.xml` | Checkstyle | [SonarQube integration](https://docs.sonarqube.org/latest/analysis/external-issues/) |
+| `$GO_PROJECT_DIR/reports/go-ci-lint.checkstyle.xml` | Checkstyle | [SonarQube integration](https://docs.sonarsource.com/sonarqube-server/latest/analyzing-source-code/importing-external-issues/external-analyzer-reports/) |
+
+### `go-semgrep` job
+
+This job performs a [Semgrep](https://semgrep.dev/docs/) analysis.
+
+It is bound to the `test` stage, and uses the following variables:
+
+| Input / Variable | Description | Default Value |
+| ---------------- | ----------- | ------------- |
+| `semgrep-disabled` / `GO_SEMGREP_DISABLED` | Set to `true` to disable this job | _none_ |
+| `semgrep-image` / `GO_SEMGREP_IMAGE` | The Docker image used to run [Semgrep](https://semgrep.dev/docs/) | `registry.hub.docker.com/semgrep/semgrep:latest` |
+| `semgrep-args` / `GO_SEMGREP_ARGS` | Semgrep [scan options](https://semgrep.dev/docs/cli-reference#semgrep-scan-command-options) | `--metrics off --disable-version-check` |
+| `semgrep-rules` / `GO_SEMGREP_RULES` | Space-separated list of [Semgrep rules](https://semgrep.dev/docs/running-rules).<br/>Can be both local YAML files or remote rules from the [Segmrep Registry](https://semgrep.dev/explore) (denoted by the `p/` prefix). | `p/golang p/gosec` |
+| `semgrep-download-rules-enabled` / `GO_SEMGREP_DOWNLOAD_RULES_ENABLED` | Download Semgrep remote rules | `true` |
+
+> :information_source: Semgrep may [collect some metrics](https://semgrep.dev/docs/metrics), especially when using rules from the Semgrep Registry.
+> To protect your privacy and let you run Semgrep in air-gap environments, this template disables all Semgrep metrics by default:
+>
+> * rules from the Semgrep registry are pre-downloaded and passed to Semgrep as local rule files (can be disabled by setting `semgrep-download-rules-enabled` / `GO_SEMGREP_DOWNLOAD_RULES_ENABLED` to `false`),
+> * the `--metrics` option is set to `off`,
+> * the `--disable-version-check` option is set.
+
+In addition to a textual report in the console, this job produces the following reports, kept for one week:
+
+| Report | Format | Usage |
+| ------ | ------ | ----- |
+| `$GO_PROJECT_DIR/reports/golang-semgrep.gitlab.json` | [GitLab's SAST format](https://semgrep.dev/docs/cli-reference#semgrep-scan-command-options) | [GitLab integration](https://docs.gitlab.com/ee/ci/yaml/artifacts_reports.html#artifactsreportssast) |
+| `$GO_PROJECT_DIR/reports/golang-semgrep.native.json` | [Semgrep's JSON format](https://semgrep.dev/docs/cli-reference#semgrep-scan-command-options) | [DefectDojo integration](https://docs.defectdojo.com/en/connecting_your_tools/parsers/file/semgrep/)<br/>_This report is generated only if DefectDojo template is detected_ |
### `go-mod-outdated` job
@@ -197,7 +225,7 @@ Checking outdated modules can be a long operation and therefore the job is confi
If you're using the SonarQube template to analyse your Go code, here is a sample `sonar-project.properties` file:
```properties
-# see: https://docs.sonarqube.org/latest/analyzing-source-code/test-coverage/test-execution-parameters/#go
+# see: https://docs.sonarsource.com/sonarqube-server/latest/analyzing-source-code/test-coverage/test-execution-parameters/#go
# set your source directory(ies) here (relative to the sonar-project.properties file)
sonar.sources=.
# exclude unwanted directories and files from being analysed
@@ -218,9 +246,9 @@ sonar.go.golangci-lint.reportPaths=reports/go-ci-lint.checkstyle.xml
More info:
-* [Go language support](https://docs.sonarqube.org/latest/analyzing-source-code/test-coverage/test-execution-parameters/#go)
-* [test coverage & execution parameters](https://docs.sonarqube.org/latest/analysis/coverage/)
-* [third-party issues](https://docs.sonarqube.org/latest/analysis/external-issues/)
+* [Go language support](https://docs.sonarsource.com/sonarqube-server/latest/analyzing-source-code/test-coverage/test-execution-parameters/#go)
+* [test coverage](https://docs.sonarsource.com/sonarqube-server/latest/analyzing-source-code/test-coverage/test-coverage-parameters/) & [test execution](https://docs.sonarsource.com/sonarqube-server/latest/analyzing-source-code/test-coverage/test-execution-parameters/) parameters
+* [external analyzer reports](https://docs.sonarsource.com/sonarqube-server/latest/analyzing-source-code/importing-external-issues/external-analyzer-reports/)
:warning: an [unsolved issue](https://jira.sonarsource.com/browse/SONARSLANG-450) may prevent SonarQube Go plugin from
importing your test reports.
@@ -257,4 +285,4 @@ It is bound to the `test` stage, and uses the following variables:
| --------------------- | -------------------------------------- | ----------------- |
| `vulncheck-disabled` / `GO_VULNCHECK_DISABLED` | Set to `true` to disable this job | _none_
| `vulncheck-args` / `GO_VULNCHECK_ARGS` | `govulncheck` [command line arguments](https://pkg.go.dev/golang.org/x/vuln/cmd/govulncheck#hdr-Flags) | `./...` |
-| `go-govulncheck-job-tags` / `GO_GOVULNCHECK_JOB_TAGS` | Tags to be used for selecting runners for the job | `[]` |
\ No newline at end of file
+| `go-govulncheck-job-tags` / `GO_GOVULNCHECK_JOB_TAGS` | Tags to be used for selecting runners for the job | `[]` |
diff --git a/bumpversion.sh b/bumpversion.sh
index 329e866dac988c049574a0a9f26ba89979c523a8..708faf434d2459d63b2bdaceada5eb32b0fd39eb 100755
--- a/bumpversion.sh
+++ b/bumpversion.sh
@@ -27,7 +27,7 @@ if [[ "$curVer" ]]; then
log_info "Bump version from \\e[33;1m${curVer}\\e[0m to \\e[33;1m${nextVer}\\e[0m (release type: $relType)..."
# replace in README
- sed -e "s/ref: *'$curVer'/ref: '$nextVer'/" -e "s/ref: *\"$curVer\”/ref: \”$nextVer\”/" -e "s/component: *\(.*\)@$curVer/component: \1@$nextVer/" README.md > README.md.next
+ sed -e "s/ref: *'$curVer'/ref: '$nextVer'/" -e "s/ref: *\"$curVer\"/ref: \"$nextVer\"/" -e "s/component: *\(.*\)@$curVer/component: \1@$nextVer/" README.md > README.md.next
mv -f README.md.next README.md
# replace in template and variants
diff --git a/kicker.json b/kicker.json
index e718114af00d06a3f73c3ad78b9b16998e303b1b..26d334434c04959291acda8e70a41016bca9b5b8 100644
--- a/kicker.json
+++ b/kicker.json
@@ -201,6 +201,35 @@
}
]
},
+ {
+ "id": "go-semgrep",
+ "name": "Semgrep",
+ "description": "[Semgrep](https://semgrep.dev/docs/) analysis",
+ "disable_with": "GO_SEMGREP_DISABLED",
+ "variables": [
+ {
+ "name": "GO_SEMGREP_IMAGE",
+ "description": "The Docker image used to run [Semgrep](https://semgrep.dev/docs/)",
+ "default": "registry.hub.docker.com/semgrep/semgrep:latest"
+ },
+ {
+ "name": "GO_SEMGREP_ARGS",
+ "description": "Semgrep [scan options](https://semgrep.dev/docs/cli-reference#semgrep-scan-command-options)",
+ "default": "--metrics off --disable-version-check"
+ },
+ {
+ "name": "GO_SEMGREP_RULES",
+ "description": "Space-separated list of [Semgrep rules](https://semgrep.dev/docs/running-rules).\n\nCan be both local YAML files or remote rules from the [Semgrep Registry](https://semgrep.dev/explore) (denoted by the `p/` prefix)",
+ "default": "p/golang p/gosec"
+ },
+ {
+ "name": "GO_SEMGREP_DOWNLOAD_RULES_ENABLED",
+ "description": "Download Semgrep remote rules",
+ "type": "boolean",
+ "default": "true"
+ }
+ ]
+ },
{
"id": "govulncheck",
"name": "Govulncheck",
diff --git a/templates/gitlab-ci-golang.yml b/templates/gitlab-ci-golang.yml
index fd16d8e4e92ca7c11433be2f2eb0b04260b03305..7a1d01b3e294bef19aacd5df30ca6f1f3c1b1e75 100644
--- a/templates/gitlab-ci-golang.yml
+++ b/templates/gitlab-ci-golang.yml
@@ -81,6 +81,26 @@ spec:
ci-lint-args:
description: '`golangci-lint` [command line arguments](https://github.com/golangci/golangci-lint#command-line-options)'
default: -E gosec,goimports ./...
+ semgrep-image:
+ description: The Docker image used to run [Semgrep](https://semgrep.dev/docs/)
+ default: registry.hub.docker.com/semgrep/semgrep:latest
+ semgrep-disabled:
+ description: Disable Semgrep
+ type: boolean
+ default: false
+ semgrep-args:
+ description: Semgrep [scan options](https://semgrep.dev/docs/cli-reference#semgrep-scan-command-options)
+ default: --metrics off --disable-version-check
+ semgrep-rules:
+ description: |-
+ Space-separeted list of [Semgrep rules](https://semgrep.dev/docs/running-rules).
+
+ Can be both local YAML files or remote rules from the [Semgrep Registry](https://semgrep.dev/explore) (denoted by the `p/` prefix)
+ default: p/golang p/gosec
+ semgrep-download-rules-enabled:
+ description: Download Semgrep remote rules
+ type: boolean
+ default: true
mod-outdated-args:
description: '`god-mod-outdated` [command line arguments](https://github.com/psampaz/go-mod-outdated#usage'
default: -update -direct
@@ -229,6 +249,11 @@ variables:
GO_CI_LINT_DISABLED: $[[ inputs.ci-lint-disabled ]]
GO_SBOM_DISABLED: $[[ inputs.sbom-disabled ]]
GO_VULNCHECK_DISABLED: $[[ inputs.vulncheck-disabled ]]
+ GO_SEMGREP_IMAGE: $[[ inputs.semgrep-image ]]
+ GO_SEMGREP_DISABLED: $[[ inputs.semgrep-disabled ]]
+ GO_SEMGREP_ARGS: $[[ inputs.semgrep-args ]]
+ GO_SEMGREP_RULES: $[[ inputs.semgrep-rules ]]
+ GO_SEMGREP_DOWNLOAD_RULES_ENABLED: $[[ inputs.semgrep-download-rules-enabled ]]
# Image of cyclonedx-gomod used for SBOM analysis
GO_SBOM_IMAGE: $[[ inputs.sbom-image ]]
@@ -368,6 +393,77 @@ stages:
log_info "... done"
}
+ # evaluate and export a secret
+ # - $1: secret variable name
+ function eval_secret() {
+ name=$1
+ value=$(eval echo "\$${name}")
+ case "$value" in
+ @b64@*)
+ decoded=$(mktemp)
+ errors=$(mktemp)
+ if echo "$value" | cut -c6- | base64 -d > "${decoded}" 2> "${errors}"
+ then
+ # shellcheck disable=SC2086
+ export ${name}="$(cat ${decoded})"
+ log_info "Successfully decoded base64 secret \\e[33;1m${name}\\e[0m"
+ else
+ fail "Failed decoding base64 secret \\e[33;1m${name}\\e[0m:\\n$(sed 's/^/... /g' "${errors}")"
+ fi
+ ;;
+ @hex@*)
+ decoded=$(mktemp)
+ errors=$(mktemp)
+ if echo "$value" | cut -c6- | sed 's/\([0-9A-F]\{2\}\)/\\\\x\1/gI' | xargs printf > "${decoded}" 2> "${errors}"
+ then
+ # shellcheck disable=SC2086
+ export ${name}="$(cat ${decoded})"
+ log_info "Successfully decoded hexadecimal secret \\e[33;1m${name}\\e[0m"
+ else
+ fail "Failed decoding hexadecimal secret \\e[33;1m${name}\\e[0m:\\n$(sed 's/^/... /g' "${errors}")"
+ fi
+ ;;
+ @url@*)
+ url=$(echo "$value" | cut -c6-)
+ if command -v curl > /dev/null
+ then
+ decoded=$(mktemp)
+ errors=$(mktemp)
+ if curl -s -S -f --connect-timeout 5 -o "${decoded}" "$url" 2> "${errors}"
+ then
+ # shellcheck disable=SC2086
+ export ${name}="$(cat ${decoded})"
+ log_info "Successfully curl'd secret \\e[33;1m${name}\\e[0m"
+ else
+ log_warn "Failed getting secret \\e[33;1m${name}\\e[0m:\\n$(sed 's/^/... /g' "${errors}")"
+ fi
+ elif command -v wget > /dev/null
+ then
+ decoded=$(mktemp)
+ errors=$(mktemp)
+ if wget -T 5 -O "${decoded}" "$url" 2> "${errors}"
+ then
+ # shellcheck disable=SC2086
+ export ${name}="$(cat ${decoded})"
+ log_info "Successfully wget'd secret \\e[33;1m${name}\\e[0m"
+ else
+ log_warn "Failed getting secret \\e[33;1m${name}\\e[0m:\\n$(sed 's/^/... /g' "${errors}")"
+ fi
+ else
+ fail "Couldn't get secret \\e[33;1m${name}\\e[0m: no http client found"
+ fi
+ ;;
+ esac
+ }
+
+ function eval_all_secrets() {
+ encoded_vars=$(env | grep -v '^scoped__' | awk -F '=' '/^[a-zA-Z0-9_]*=@(b64|hex|url)@/ {print $1}')
+ for var in $encoded_vars
+ do
+ eval_secret "$var"
+ done
+ }
+
function output_coverage() {
coverage_out=reports/go-coverage.native.out
if [[ -f "$coverage_out" ]]
@@ -496,7 +592,33 @@ stages:
fi
}
+ function setup_semgrep_rules() {
+ if [[ "${GO_SEMGREP_DOWNLOAD_RULES_ENABLED}" == "true" ]]
+ then
+ log_info "Download Semgrep rule files..."
+ for rule in $GO_SEMGREP_RULES
+ do
+ if [[ -r $rule ]]
+ then
+ log_info "... rule file $rule found: skip"
+ SEMGREP_RULES="${SEMGREP_RULES} $rule"
+ else
+ log_info "... rule file $rule not found : download (https://semgrep.dev/c/$rule)"
+ dest_file="semgrep-${rule/p\//}.yml"
+ wget "https://semgrep.dev/c/$rule" -O "$dest_file"
+ SEMGREP_RULES="${SEMGREP_RULES} $dest_file"
+ fi
+ done
+ SEMGREP_RULES="${SEMGREP_RULES:1}"
+ export SEMGREP_RULES
+ else
+ # download not enabled: simply use $GO_SEMGREP_RULES
+ export SEMGREP_RULES="${GO_SEMGREP_RULES}"
+ fi
+ }
+
unscope_variables
+ eval_all_secrets
# ENDSCRIPT
@@ -506,7 +628,7 @@ stages:
image: $GO_IMAGE
services:
- name: "$TBC_TRACKING_IMAGE"
- command: ["--service", "golang", "4.10.0"]
+ command: ["--service", "golang", "4.11.1"]
variables:
# The directory where 'go install' will install a command.
GOBIN: "$CI_PROJECT_DIR/$GO_PROJECT_DIR/bin"
@@ -641,6 +763,37 @@ go-ci-lint:
- !reference [.test-policy, rules]
tags: $[[ inputs.go-ci-lint-job-tags ]]
+# SAST: semgrep
+go-semgrep:
+ extends: .go-base
+ image: $GO_SEMGREP_IMAGE
+ # unset cache from parent job
+ cache : {}
+ dependencies: []
+ stage: test
+ before_script:
+ - !reference [.go-scripts]
+ - mkdir -p -m 777 ${GO_PROJECT_DIR}/reports
+ - setup_semgrep_rules
+ script:
+ - >-
+ semgrep ci ${TRACE+--verbose} ${GO_SEMGREP_ARGS} --subdir ${GO_PROJECT_DIR}
+ --gitlab-sast-output=${GO_PROJECT_DIR}/reports/golang-semgrep.gitlab.json
+ ${DEFECTDOJO_SEMGREP_REPORTS:+--json-output=${GO_PROJECT_DIR}/reports/golang-semgrep.native.json}
+ artifacts:
+ name: "$CI_JOB_NAME artifacts from $CI_PROJECT_NAME on $ĈI_COMMIT_REF_SLUG"
+ when: "always"
+ expire_in: 1 week
+ reports:
+ sast: $GO_PROJECT_DIR/reports/golang-semgrep.gitlab.json
+ paths:
+ - $GO_PROJECT_DIR/reports/golang-semgrep.*
+ rules:
+ # exclude if disable
+ - if: '$GO_SEMGREP_DISABLED == "true"'
+ when: never
+ - !reference [.test-policy, rules]
+
go-mod-outdated:
extends: .go-base
stage: test