diff --git a/.gitleaksignore b/.gitleaksignore
index eae290df9d4d1b32df6ca200d40d0a451860fc11..bb79c807a160af9fd98c2a262ab9bddf2df6f1f5 100644
--- a/.gitleaksignore
+++ b/.gitleaksignore
@@ -1 +1,2 @@
-dda82d21c9ba0e572abb74e0adb97268dc46d438:README.md:private-key:320
\ No newline at end of file
+dda82d21c9ba0e572abb74e0adb97268dc46d438:README.md:private-key:320
+ff8b9856a0bb045932f4810410404261cd848ea4:README.md:private-key:320
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 2416236ca43f408a9f54003a29dbd77fb43b904b..29b7c8b897fc45b0871c6beb40d1d58799f9fd73 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,10 +1,115 @@
-## [7.0.2](https://git.code.tecnalia.com/smartdatalab/public/ci-cd-components/python/compare/7.0.1...7.0.2) (2024-07-26)
+## [7.7.1](https://gitlab.com/to-be-continuous/python/compare/7.7.0...7.7.1) (2025-01-12)
### Bug Fixes
-* issue [#73](https://git.code.tecnalia.com/smartdatalab/public/ci-cd-components/python/issues/73) github_get_latest_version ([ce26d5a](https://git.code.tecnalia.com/smartdatalab/public/ci-cd-components/python/commit/ce26d5abba8950f30bad1d992a2481bf252359b7))
-* README for trivy now enabled by default ([f5d5f2e](https://git.code.tecnalia.com/smartdatalab/public/ci-cd-components/python/commit/f5d5f2e9c186b6aeb0c55ef45a65b85615b9ad7b))
+* move back 'reports' dir creation at job level to fix variants missing reports dir ([bf15efe](https://gitlab.com/to-be-continuous/python/commit/bf15efe4b008a5f292e782d0363a52000bf43f37))
+
+# [7.7.0](https://gitlab.com/to-be-continuous/python/compare/7.6.0...7.7.0) (2025-01-12)
+
+
+### Features
+
+* add auto-release as an optional feature for releases ([9db709a](https://gitlab.com/to-be-continuous/python/commit/9db709ad8fe96c7ed524f8083e57b845914e4009))
+
+# [7.6.0](https://gitlab.com/to-be-continuous/python/compare/7.5.2...7.6.0) (2025-01-08)
+
+
+### Features
+
+* add separate 'publish-enabled' to enable publishing package ([6f9ee56](https://gitlab.com/to-be-continuous/python/commit/6f9ee56d00ee5408953fa24323dbba81aa2d4f3a))
+
+## [7.5.2](https://gitlab.com/to-be-continuous/python/compare/7.5.1...7.5.2) (2024-12-22)
+
+
+### Bug Fixes
+
+* **test:** handle decimal coverage ([4fb81f8](https://gitlab.com/to-be-continuous/python/commit/4fb81f8b66bf285f173a2335f8c34523d0f7ca3d))
+
+## [7.5.1](https://gitlab.com/to-be-continuous/python/compare/7.5.0...7.5.1) (2024-11-21)
+
+
+### Bug Fixes
+
+* **CodeArtifact:** fix AWS CodeArtifact variant ([c913e65](https://gitlab.com/to-be-continuous/python/commit/c913e6538d88efaf1d6f0eb7742e7531d66a32c2))
+
+# [7.5.0](https://gitlab.com/to-be-continuous/python/compare/7.4.0...7.5.0) (2024-11-11)
+
+
+### Features
+
+* **Ruff:** add `ruff-format` job for code formatting ([142589f](https://gitlab.com/to-be-continuous/python/commit/142589f2c260336d3a703af3e149c1c666fd5373))
+
+# [7.4.0](https://gitlab.com/to-be-continuous/python/compare/7.3.3...7.4.0) (2024-11-08)
+
+
+### Features
+
+* add AWS CodeArtifact support (variant) ([128fb99](https://gitlab.com/to-be-continuous/python/commit/128fb9950c1354c211abe17d5cba19d75dd66ecc))
+
+## [7.3.3](https://gitlab.com/to-be-continuous/python/compare/7.3.2...7.3.3) (2024-11-06)
+
+
+### Bug Fixes
+
+* correct bandit exclude of .venv and .cache ([ed95527](https://gitlab.com/to-be-continuous/python/commit/ed955279f56f2d66a2a7532b35515f2309f05f5c)), closes [#92](https://gitlab.com/to-be-continuous/python/issues/92)
+
+## [7.3.2](https://gitlab.com/to-be-continuous/python/compare/7.3.1...7.3.2) (2024-11-02)
+
+
+### Bug Fixes
+
+* limit security reports access to developer role or higher ([40c85ef](https://gitlab.com/to-be-continuous/python/commit/40c85eff562a00ceb9b381ef72472ce1910b97ab))
+
+## [7.3.1](https://gitlab.com/to-be-continuous/python/compare/7.3.0...7.3.1) (2024-10-25)
+
+
+### Bug Fixes
+
+* **Trivy:** trivy scan fails when issues are found ([671b781](https://gitlab.com/to-be-continuous/python/commit/671b78142c08cdd5bbf1441a81705b96dbf0740f))
+* use right options for uv with extras deps ([354af5a](https://gitlab.com/to-be-continuous/python/commit/354af5ad8294ad8f3de3f7ad6aeaf8752d5f2625))
+
+# [7.3.0](https://gitlab.com/to-be-continuous/python/compare/7.2.0...7.3.0) (2024-10-15)
+
+
+### Features
+
+* **uv:** add uv support as a new build system ([8aeb20b](https://gitlab.com/to-be-continuous/python/commit/8aeb20b09347ff35398a4a707852a9cc17cc6842)), closes [#80](https://gitlab.com/to-be-continuous/python/issues/80)
+* **uv:** add uv support as a new build system ([d22ffba](https://gitlab.com/to-be-continuous/python/commit/d22ffbacb4228cb4ffdc6396bca9e43ad194bfff))
+
+# [7.2.0](https://gitlab.com/to-be-continuous/python/compare/7.1.1...7.2.0) (2024-10-04)
+
+
+### Bug Fixes
+
+* **release:** support full semantic-versioning specifcation (with prerelease and build metadata) ([08e9d7e](https://gitlab.com/to-be-continuous/python/commit/08e9d7e9f7f1bdd43a2070c9ee5abb16a8b8aaa0))
+* **trivy:** use --pkg-types instead of deprecated --vuln-type option ([5e0a0d2](https://gitlab.com/to-be-continuous/python/commit/5e0a0d2918fd7539bd2e1cb955e99ef5857db1f5))
+
+
+### Features
+
+* **trivy:** enable comprehensive priority ([322eb1b](https://gitlab.com/to-be-continuous/python/commit/322eb1b88c49d9a1662ad6b6199541f1a82860ef))
+
+## [7.1.1](https://gitlab.com/to-be-continuous/python/compare/7.1.0...7.1.1) (2024-10-03)
+
+
+### Bug Fixes
+
+* Poetry Build system test ([9505604](https://gitlab.com/to-be-continuous/python/commit/95056049e7ee8239b6358def7c594e7002036574))
+
+# [7.1.0](https://gitlab.com/to-be-continuous/python/compare/7.0.2...7.1.0) (2024-09-15)
+
+
+### Bug Fixes
+
+* check trivy activity to match new log format ([edd8fcf](https://gitlab.com/to-be-continuous/python/commit/edd8fcf71f1b251c467d6bbce6e8a190d4584dda))
+* pylint --ignore .cache not working now use find to exclude .cache ([e1463bc](https://gitlab.com/to-be-continuous/python/commit/e1463bc750fbd24b12d407267061d8ae8a3718f1))
+
+
+### Features
+
+* isort exclude .cache ([e333183](https://gitlab.com/to-be-continuous/python/commit/e333183ca48aa98baf9d510caf0c8f3f93d04b82))
+* remove unnecesary install when use poetry or pipenv ([f025c6d](https://gitlab.com/to-be-continuous/python/commit/f025c6df22d48bd735458fc478b18d2235a715a2))
## [7.0.2](https://gitlab.com/to-be-continuous/python/compare/7.0.1...7.0.2) (2024-05-20)
diff --git a/README.md b/README.md
index 8c0c22d93f5953ef8b6569e692b6786cfc35a6b4..6549aefb50989cb2d57c442119b2028b1b5a3a84 100644
--- a/README.md
+++ b/README.md
@@ -14,7 +14,7 @@ Add the following to your `.gitlab-ci.yml`:
```yaml
include:
# 1: include the component
- - component: $CI_SERVER_FQDN/to-be-continuous/python/gitlab-ci-python@7.0.2
+ - component: $CI_SERVER_FQDN/to-be-continuous/python/gitlab-ci-python@7.7.1
# 2: set/override component inputs
inputs:
image: registry.hub.docker.com/library/python:3.12-slim
@@ -29,7 +29,7 @@ Add the following to your `.gitlab-ci.yml`:
include:
# 1: include the template
- project: 'to-be-continuous/python'
- ref: '7.0.2'
+ ref: '7.7.1'
file: '/templates/gitlab-ci-python.yml'
variables:
@@ -65,12 +65,13 @@ By default it tries to auto-detect the build system used by the project (based o
and/or `setup.py` and/or `requirements.txt`), but the build system might also be set explicitly using the
`$PYTHON_BUILD_SYSTEM` variable:
-| Value | Build System (scope) |
-| ---------------- | ---------------------------------------------------------- |
-| _none_ (default) or `auto` | The template tries to **auto-detect** the actual build system |
-| `setuptools` | [Setuptools](https://setuptools.pypa.io/) (dependencies, build & packaging) |
-| `poetry` | [Poetry](https://python-poetry.org/) (dependencies, build, test & packaging) |
-| `pipenv` | [Pipenv](https://pipenv.pypa.io/) (dependencies only) |
+| Value | Build System (scope) |
+| ---------------- |--------------------------------------------------------------------------------------------------------|
+| _none_ (default) or `auto` | The template tries to **auto-detect** the actual build system |
+| `setuptools` | [Setuptools](https://setuptools.pypa.io/) (dependencies, build & packaging) |
+| `poetry` | [Poetry](https://python-poetry.org/) (dependencies, build, test & packaging) |
+| `uv` | [uv](https://docs.astral.sh/uv/) (dependencies, build, test & packaging)|
+| `pipenv` | [Pipenv](https://pipenv.pypa.io/) (dependencies only) |
| `reqfile` | [Requirements Files](https://pip.pypa.io/en/stable/user_guide/#requirements-files) (dependencies only) |
:warning: You can explicitly set the build tool version by setting `$PYTHON_BUILD_SYSTEM` variable including a [version identification](https://peps.python.org/pep-0440/). For example `PYTHON_BUILD_SYSTEM="poetry==1.1.15"`
@@ -108,7 +109,7 @@ In addition to a textual report in the console, this job produces the following
| Report | Format | Usage |
| -------------- | ---------------------------------------------------------------------------- | ----------------- |
| `$PYTHON_PROJECT_DIR/reports/py-lint.codeclimate.json` | [Code Climate](https://docs.codeclimate.com/docs/pylint) | [GitLab integration](https://docs.gitlab.com/ee/ci/yaml/artifacts_reports.html#artifactsreportscodequality) |
-| `$PYTHON_PROJECT_DIR/reports/py-lint.parseable.txt` | [parseable](https://pylint.pycqa.org/en/latest/user_guide/usage/output.html) | [SonarQube integration](https://docs.sonarqube.org/latest/analysis/external-issues/) |
+| `$PYTHON_PROJECT_DIR/reports/py-lint.parseable.txt` | [parseable](https://pylint.pycqa.org/en/latest/user_guide/usage/output.html) | [SonarQube integration](https://docs.sonarsource.com/sonarqube-server/latest/analyzing-source-code/importing-external-issues/external-analyzer-reports/) |
### Test jobs
@@ -152,8 +153,8 @@ In addition to a textual report in the console, this job produces the following
| Report | Format | Usage |
| -------------- | ---------------------------------------------------------------------------- | ----------------- |
-| `$PYTHON_PROJECT_DIR/reports/TEST-*.xml` | [xUnit](https://en.wikipedia.org/wiki/XUnit) test report(s) | [GitLab integration](https://docs.gitlab.com/ee/ci/yaml/artifacts_reports.html#artifactsreportsjunit) & [SonarQube integration](https://docs.sonarqube.org/latest/analysis/test-coverage/test-execution-parameters/#header-8) |
-| `$PYTHON_PROJECT_DIR/reports/py-coverage.cobertura.xml` | [Cobertura XML](https://gcovr.com/en/stable/output/cobertura.html) coverage report | [GitLab integration](https://docs.gitlab.com/ee/ci/yaml/artifacts_reports.html#artifactsreportscoverage_report) & [SonarQube integration](https://docs.sonarqube.org/latest/analysis/test-coverage/python-test-coverage/) |
+| `$PYTHON_PROJECT_DIR/reports/TEST-*.xml` | [xUnit](https://en.wikipedia.org/wiki/XUnit) test report(s) | [GitLab integration](https://docs.gitlab.com/ee/ci/yaml/artifacts_reports.html#artifactsreportsjunit) & [SonarQube integration](https://docs.sonarsource.com/sonarqube-server/latest/analyzing-source-code/test-coverage/test-execution-parameters/#python) |
+| `$PYTHON_PROJECT_DIR/reports/py-coverage.cobertura.xml` | [Cobertura XML](https://gcovr.com/en/stable/output/cobertura.html) coverage report | [GitLab integration](https://docs.gitlab.com/ee/ci/yaml/artifacts_reports.html#artifactsreportscoverage_report) & [SonarQube integration](https://docs.sonarsource.com/sonarqube-server/latest/analyzing-source-code/test-coverage/python-test-coverage/) |
#### `py-pytest` job
@@ -186,8 +187,8 @@ In addition to a textual report in the console, this job produces the following
| Report | Format | Usage |
| -------------- | ---------------------------------------------------------------------------- | ----------------- |
-| `$PYTHON_PROJECT_DIR/reports/TEST-*.xml` | [xUnit](https://en.wikipedia.org/wiki/XUnit) test report(s) | [GitLab integration](https://docs.gitlab.com/ee/ci/yaml/artifacts_reports.html#artifactsreportsjunit) & [SonarQube integration](https://docs.sonarqube.org/latest/analysis/test-coverage/test-execution-parameters/#header-8) |
-| `$PYTHON_PROJECT_DIR/reports/py-coverage.cobertura.xml` | [Cobertura XML](https://gcovr.com/en/stable/output/cobertura.html) coverage report | [GitLab integration](https://docs.gitlab.com/ee/ci/yaml/artifacts_reports.html#artifactsreportscoverage_report) & [SonarQube integration](https://docs.sonarqube.org/latest/analysis/test-coverage/python-test-coverage/) |
+| `$PYTHON_PROJECT_DIR/reports/TEST-*.xml` | [xUnit](https://en.wikipedia.org/wiki/XUnit) test report(s) | [GitLab integration](https://docs.gitlab.com/ee/ci/yaml/artifacts_reports.html#artifactsreportsjunit) & [SonarQube integration](https://docs.sonarsource.com/sonarqube-server/latest/analyzing-source-code/test-coverage/test-execution-parameters/#python) |
+| `$PYTHON_PROJECT_DIR/reports/py-coverage.cobertura.xml` | [Cobertura XML](https://gcovr.com/en/stable/output/cobertura.html) coverage report | [GitLab integration](https://docs.gitlab.com/ee/ci/yaml/artifacts_reports.html#artifactsreportscoverage_report) & [SonarQube integration](https://docs.sonarsource.com/sonarqube-server/latest/analyzing-source-code/test-coverage/python-test-coverage/) |
#### `py-nosetests` job
@@ -211,8 +212,8 @@ In addition to a textual report in the console, this job produces the following
| Report | Format | Usage |
| -------------- | ---------------------------------------------------------------------------- | ----------------- |
-| `$PYTHON_PROJECT_DIR/reports/TEST-*.xml` | [xUnit](https://en.wikipedia.org/wiki/XUnit) test report(s) | [GitLab integration](https://docs.gitlab.com/ee/ci/yaml/artifacts_reports.html#artifactsreportsjunit) & [SonarQube integration](https://docs.sonarqube.org/latest/analysis/test-coverage/test-execution-parameters/#header-8) |
-| `$PYTHON_PROJECT_DIR/reports/py-coverage.cobertura.xml` | [Cobertura XML](https://gcovr.com/en/stable/output/cobertura.html) coverage report | [GitLab integration](https://docs.gitlab.com/ee/ci/yaml/artifacts_reports.html#artifactsreportscoverage_report) & [SonarQube integration](https://docs.sonarqube.org/latest/analysis/test-coverage/python-test-coverage/) |
+| `$PYTHON_PROJECT_DIR/reports/TEST-*.xml` | [xUnit](https://en.wikipedia.org/wiki/XUnit) test report(s) | [GitLab integration](https://docs.gitlab.com/ee/ci/yaml/artifacts_reports.html#artifactsreportsjunit) & [SonarQube integration](https://docs.sonarsource.com/sonarqube-server/latest/analyzing-source-code/test-coverage/test-execution-parameters/#python) |
+| `$PYTHON_PROJECT_DIR/reports/py-coverage.cobertura.xml` | [Cobertura XML](https://gcovr.com/en/stable/output/cobertura.html) coverage report | [GitLab integration](https://docs.gitlab.com/ee/ci/yaml/artifacts_reports.html#artifactsreportscoverage_report) & [SonarQube integration](https://docs.sonarsource.com/sonarqube-server/latest/analyzing-source-code/test-coverage/python-test-coverage/) |
#### `py-compile` job
@@ -238,16 +239,16 @@ It is bound to the `test` stage, and uses the following variables:
| `bandit-args` / `BANDIT_ARGS` | Additional [Bandit CLI options](https://github.com/PyCQA/bandit#usage) | `--recursive .` |
| `py-bandit-job-tags` / `PY_BANDIT_JOB_TAGS` | Tags to be used for selecting runners for the job | `[]` |
-In addition to a textual report in the console, this job produces the following reports, kept for one day:
+In addition to a textual report in the console, this job produces the following reports, kept for one day and only available for download by users with the Developer role or higher:
| Report | Format | Usage |
| -------------- | ---------------------------------------------------------------------------- | ----------------- |
-| `$PYTHON_PROJECT_DIR/reports/py-bandit.bandit.csv` | [CSV](https://bandit.readthedocs.io/en/latest/formatters/csv.html) | [SonarQube integration](https://docs.sonarqube.org/latest/analysis/external-issues/)<br/>_This report is generated only if SonarQube template is detected_ |
-| `$PYTHON_PROJECT_DIR/reports/py-bandit.bandit.json` | [JSON](https://bandit.readthedocs.io/en/latest/formatters/json.html) | [DefectDojo integration](https://defectdojo.github.io/django-DefectDojo/integrations/parsers/#bandit)<br/>_This report is generated only if DefectDojo template is detected_ |
+| `$PYTHON_PROJECT_DIR/reports/py-bandit.bandit.csv` | [CSV](https://bandit.readthedocs.io/en/latest/formatters/csv.html) | [SonarQube integration](https://docs.sonarsource.com/sonarqube-server/latest/analyzing-source-code/importing-external-issues/external-analyzer-reports/)<br/>_This report is generated only if SonarQube template is detected_ |
+| `$PYTHON_PROJECT_DIR/reports/py-bandit.bandit.json` | [JSON](https://bandit.readthedocs.io/en/latest/formatters/json.html) | [DefectDojo integration](https://docs.defectdojo.com/en/connecting_your_tools/parsers/file/bandit/)<br/>_This report is generated only if DefectDojo template is detected_ |
### `py-trivy` job (dependency check)
-This job performs a dependency check analysis using [Trivy](https://github.com/aquasecurity/trivy/).
+This job performs a dependency check analysis using [Trivy](https://aquasecurity.github.io/trivy).
:warning: This job is now **enabled by default** since version 7.0.0
@@ -260,11 +261,21 @@ It is bound to the `test` stage, and uses the following variables:
| `trivy-args` / `PYTHON_TRIVY_ARGS` | Additional [Trivy CLI options](https://aquasecurity.github.io/trivy/v0.21.1/getting-started/cli/fs/) | `--vuln-type library` |
| `py-trivy-job-tags` / `PY_TRIVY_JOB_TAGS` | Tags to be used for selecting runners for the job | `[]` |
-In addition to a textual report in the console, this job produces the following reports, kept for one day:
+Other Trivy parameters shall be configured using [Trivy environment variables](https://aquasecurity.github.io/trivy/latest/docs/references/configuration/cli/trivy_filesystem/#options).
+Examples:
+* `TRIVY_SEVERITY`: severities of security issues to be displayed (comma separated values: `UNKNOWN`, `LOW`, `MEDIUM`, `HIGH`, `CRITICAL`)
+* `TRIVY_SERVER`: server address (enables [client/server mode](https://trivy.dev/latest/docs/references/modes/client-server/))
+* `TRIVY_DB_REPOSITORY`: OCI repository to retrieve Trivy Database from
+* ...
+
+:warning: if you're using Trivy in multiple templates with different parameter values (ex: different `TRIVY_SEVERITY` threshold with Python and - says - Docker templates), then it is
+recommanded to pass the configuration as CLI options using the `trivy-args` input / `PYTHON_TRIVY_ARGS` variable.
+
+In addition to a textual report in the console, this job produces the following reports, kept for one day and only available for download by users with the Developer role or higher:
| Report | Format | Usage |
| -------------- | ---------------------------------------------------------------------------- | ----------------- |
-| `$PYTHON_PROJECT_DIR/reports/py-trivy.trivy.json` | [JSON](https://aquasecurity.github.io/trivy/latest/docs/configuration/reporting/#json) | [DefectDojo integration](https://defectdojo.github.io/django-DefectDojo/integrations/parsers/#trivy)<br/>_This report is generated only if DefectDojo template is detected_ |
+| `$PYTHON_PROJECT_DIR/reports/py-trivy.trivy.json` | [JSON](https://aquasecurity.github.io/trivy/latest/docs/configuration/reporting/#json) | [DefectDojo integration](https://docs.defectdojo.com/en/connecting_your_tools/parsers/file/trivy/)<br/>_This report is generated only if DefectDojo template is detected_ |
### `py-sbom` job
@@ -315,14 +326,24 @@ This job **disabled by default** and runs [Ruff](https://docs.astral.sh/ruff/) o
| `ruff-ext-exclude` / `RUFF_EXT_EXCLUDE` | Define [extend-exclude](https://docs.astral.sh/ruff/settings/#extend-exclude) files | _.venv,.cache_ |
| `py-ruff-job-tags` / `PY_RUFF_JOB_TAGS` | Tags to be used for selecting runners for the job | `[]` |
-:warning: Ruff can replace isort, Black, Bandit, Pylint and much more. [More info](https://github.com/astral-sh/ruff/blob/main/docs/faq.md#which-tools-does-ruff-replace).
+:warning: Ruff can replace isort, Bandit, Pylint and much more. [More info](https://github.com/astral-sh/ruff/blob/main/docs/faq.md#which-tools-does-ruff-replace).
In addition to logs in the console, this job produces the following reports, kept for one week:
| Report | Format | Usage |
| -------------- | ---------------------------------------------------------------------------- | ----------------- |
| `$PYTHON_PROJECT_DIR/reports/py-ruff.gitlab.json` | [GitLab](https://docs.astral.sh/ruff/settings/#output-format) | [GitLab integration](https://docs.gitlab.com/ee/ci/yaml/artifacts_reports.html#artifactsreportscodequality) |
-| `$PYTHON_PROJECT_DIR/reports/py-ruff.native.json` | [JSON](https://docs.astral.sh/ruff/settings/#output-format) | [SonarQube integration](https://docs.sonarqube.org/latest/analysis/external-issues/)<br/>_This report is generated only if SonarQube template is detected_ |
+| `$PYTHON_PROJECT_DIR/reports/py-ruff.native.json` | [JSON](https://docs.astral.sh/ruff/settings/#output-format) | [SonarQube integration](https://docs.sonarsource.com/sonarqube-server/latest/analyzing-source-code/importing-external-issues/external-analyzer-reports/)<br/>_This report is generated only if SonarQube template is detected_ |
+
+### `py-ruff-format` job
+
+This job **disabled by default** and runs [Ruff](https://docs.astral.sh/ruff/) on the repo. It is bound to the build stage.
+
+| Input / Variable | Description | Default value |
+| ---------------- | ----------------------------------------------------------------------- | ----------------- |
+| `ruff-format-enabled` / `RUFF_FORMAT_ENABLED` | Set to `true` to enable ruff job | _none_ (disabled) |
+
+:warning: Ruff can replace Black and much more. [More info](https://github.com/astral-sh/ruff/blob/main/docs/faq.md#which-tools-does-ruff-replace).
#### `py-mypy` job
@@ -343,14 +364,14 @@ In addition to a textual report in the console, this job produces the following
| Report | Format | Usage |
| -------------- | ---------------------------------------------------------------------------- | ----------------- |
| `$PYTHON_PROJECT_DIR/reports/py-mypy.codeclimate.json` | [Code Climate](https://github.com/soul-catcher/mypy-gitlab-code-quality) | [GitLab integration](https://docs.gitlab.com/ee/ci/yaml/artifacts_reports.html#artifactsreportscodequality) |
-| `$PYTHON_PROJECT_DIR/reports/py-mypy.console.txt` | [mypy console output](https://mypy.readthedocs.io/) | [SonarQube integration](https://docs.sonarqube.org/latest/analysis/external-issues/) |
+| `$PYTHON_PROJECT_DIR/reports/py-mypy.console.txt` | [mypy console output](https://mypy.readthedocs.io/) | [SonarQube integration](https://docs.sonarsource.com/sonarqube-server/latest/analyzing-source-code/importing-external-issues/external-analyzer-reports/) |
### SonarQube analysis
If you're using the SonarQube template to analyse your Python code, here is a sample `sonar-project.properties` file:
```properties
-# see: https://docs.sonarqube.org/latest/analyzing-source-code/test-coverage/python-test-coverage/
+# see: https://docs.sonarsource.com/sonarqube-server/latest/analyzing-source-code/test-coverage/python-test-coverage/
# set your source directory(ies) here (relative to the sonar-project.properties file)
sonar.sources=.
# exclude unwanted directories and files from being analysed
@@ -376,9 +397,9 @@ sonar.python.mypy.reportPaths=reports/py-mypy.console.txt
More info:
-* [Python language support](https://docs.sonarqube.org/latest/analyzing-source-code/test-coverage/python-test-coverage/)
-* [test coverage & execution parameters](https://docs.sonarqube.org/latest/analysis/coverage/)
-* [third-party issues](https://docs.sonarqube.org/latest/analysis/external-issues/)
+* [Python language support](https://docs.sonarsource.com/sonarqube-server/latest/analyzing-source-code/test-coverage/python-test-coverage/)
+* [test coverage](https://docs.sonarsource.com/sonarqube-server/latest/analyzing-source-code/test-coverage/test-coverage-parameters/) & [test execution](https://docs.sonarsource.com/sonarqube-server/latest/analyzing-source-code/test-coverage/test-execution-parameters/) parameters
+* [external analyzer reports](https://docs.sonarsource.com/sonarqube-server/latest/analyzing-source-code/importing-external-issues/external-analyzer-reports/)
### `py-release` job
@@ -389,9 +410,10 @@ This job is **disabled by default** and allows to perform a complete release of
3. build the [Python packages](https://packaging.python.org/),
4. publish the built packages to a PyPI compatible repository ([GitLab packages](https://docs.gitlab.com/ee/user/packages/pypi_repository/) by default).
-The Python template supports two packaging systems:
+The Python template supports three packaging systems:
* [Poetry](https://python-poetry.org/): uses Poetry-specific [version](https://python-poetry.org/docs/cli/#version), [build](https://python-poetry.org/docs/cli/#build) and [publish](https://python-poetry.org/docs/cli/#publish) commands.
+* [uv](https://docs.astral.sh/uv/): uses [uv](https://docs.astral.sh/uv/) as version management, [build](https://docs.astral.sh/uv/guides/publish/#building-your-package) as package builder and [publish](https://docs.astral.sh/uv/guides/publish/) to publish.
* [Setuptools](https://setuptools.pypa.io/): uses [bump-my-version](https://github.com/callowayproject/bump-my-version) as version management, [build](https://pypa-build.readthedocs.io/) as package builder and [Twine](https://twine.readthedocs.io/) to publish.
The release job is bound to the `publish` stage, appears only on production and integration branches and uses the following variables:
@@ -399,17 +421,37 @@ The release job is bound to the `publish` stage, appears only on production and
| Input / Variable | Description | Default value |
| ----------------------- | ----------------------------------------------------------------------- | ----------------- |
| `release-enabled` / `PYTHON_RELEASE_ENABLED`| Set to `true` to enable the release job | _none_ (disabled) |
+| `auto-release-enabled` / `PYTHON_AUTO_RELEASE_ENABLED`| When set the job start automatically on production branch. When not set (default), the job is manual. Note that this behavior also depends on release-enabled being set. | _none_ (disabled) |
| `release-next` / `PYTHON_RELEASE_NEXT` | The part of the version to increase (one of: `major`, `minor`, `patch`) | `minor` |
| `semrel-release-disabled` / `PYTHON_SEMREL_RELEASE_DISABLED`| Set to `true` to disable [semantic-release integration](#semantic-release-integration) | _none_ (disabled) |
| `GIT_USERNAME` | Git username for Git push operations (see below) | _none_ |
| :lock: `GIT_PASSWORD` | Git password for Git push operations (see below) | _none_ |
| :lock: `GIT_PRIVATE_KEY`| SSH key for Git push operations (see below) | _none_ |
| `release-commit-message` / `PYTHON_RELEASE_COMMIT_MESSAGE`| The Git commit message to use on the release commit. This is templated using the [Python Format String Syntax](http://docs.python.org/2/library/string.html#format-string-syntax). Available in the template context are current_version and new_version. | `chore(python-release): {current_version} → {new_version}` |
+
+When `py-release` job is enabled, `py-publish` job is automatically enabled too.
+
+### `py-publish` job
+
+This job is **disabled by default** and allow to publish the built packages to a PyPI compatible repository ([GitLab packages](https://docs.gitlab.com/ee/user/packages/pypi_repository/) by default.
+
+The Python template supports three packaging systems:
+
+* [Poetry](https://python-poetry.org/): uses Poetry-specific [version](https://python-poetry.org/docs/cli/#version), [build](https://python-poetry.org/docs/cli/#build) and [publish](https://python-poetry.org/docs/cli/#publish) commands.
+* [uv](https://docs.astral.sh/uv/): uses [uv](https://docs.astral.sh/uv/) as version management, [build](https://docs.astral.sh/uv/guides/publish/#building-your-package) as package builder and [publish](https://docs.astral.sh/uv/guides/publish/) to publish.
+* [Setuptools](https://setuptools.pypa.io/): uses [bump-my-version](https://github.com/callowayproject/bump-my-version) as version management, [build](https://pypa-build.readthedocs.io/) as package builder and [Twine](https://twine.readthedocs.io/) to publish.
+
+The publish job is bound to the `publish` stage, is executed on a Git tag matching [semantic versioning pattern](https://semver.org/) and uses the following variables:
+
+| Input / Variable | Description | Default value |
+| ----------------------- | ----------------------------------------------------------------------- | ----------------- |
+| `publish-enabled` / `PYTHON_PUBLISH_ENABLED`| Set to `true` to enable the publish job | _none_ (disabled) |
| `repository-url` / `PYTHON_REPOSITORY_URL`| Target PyPI repository to publish packages | _[GitLab project's PyPI packages repository](https://docs.gitlab.com/ee/user/packages/pypi_repository/)_ |
| `PYTHON_REPOSITORY_USERNAME`| Target PyPI repository username credential | `gitlab-ci-token` |
| :lock: `PYTHON_REPOSITORY_PASSWORD`| Target PyPI repository password credential | `$CI_JOB_TOKEN` |
| `py-release-job-tags` / `PY_RELEASE_JOB_TAGS` | Tags to be used for selecting runners for the job | `[]` |
+
#### Setuptools tip
If you're using a Setuptools configuration, then you will have to write a `.bumpversion.toml` or `pyproject.toml` file.
@@ -512,10 +554,12 @@ In order to be able to communicate with the Vault server, the variant requires t
| Input / Variable | Description | Default value |
| ----------------- | -------------------------------------- | ----------------- |
| `TBC_VAULT_IMAGE` | The [Vault Secrets Provider](https://gitlab.com/to-be-continuous/tools/vault-secrets-provider) image to use (can be overridden) | `registry.gitlab.com/to-be-continuous/tools/vault-secrets-provider:latest` |
-| `vault-base-url` / `VAULT_BASE_URL` | The Vault server base API url | _none_ |
+| `vault-base-url` / `VAULT_BASE_URL` | The Vault server base API url | **must be defined** |
| `vault-oidc-aud` / `VAULT_OIDC_AUD` | The `aud` claim for the JWT | `$CI_SERVER_URL` |
-| :lock: `VAULT_ROLE_ID` | The [AppRole](https://www.vaultproject.io/docs/auth/approle) RoleID | **must be defined** |
-| :lock: `VAULT_SECRET_ID` | The [AppRole](https://www.vaultproject.io/docs/auth/approle) SecretID | **must be defined** |
+| :lock: `VAULT_ROLE_ID` | The [AppRole](https://www.vaultproject.io/docs/auth/approle) RoleID | _none_ |
+| :lock: `VAULT_SECRET_ID` | The [AppRole](https://www.vaultproject.io/docs/auth/approle) SecretID | _none_ |
+
+By default, the variant will authentifacte using a [JWT ID token](https://docs.gitlab.com/ee/ci/secrets/id_token_authentication.html). To use [AppRole](https://www.vaultproject.io/docs/auth/approle) instead the `VAULT_ROLE_ID` and `VAULT_SECRET_ID` should be defined as secret project variables.
#### Usage
@@ -537,9 +581,9 @@ With:
```yaml
include:
# main component
- - component: $CI_SERVER_FQDN/to-be-continuous/python/gitlab-ci-python@7.0.2
+ - component: $CI_SERVER_FQDN/to-be-continuous/python/gitlab-ci-python@7.7.1
# Vault variant
- - component: $CI_SERVER_FQDN/to-be-continuous/python/gitlab-ci-python-vault@7.0.2
+ - component: $CI_SERVER_FQDN/to-be-continuous/python/gitlab-ci-python-vault@7.7.1
inputs:
vault-base-url: "https://vault.acme.host/v1"
# audience claim for JWT
@@ -550,12 +594,11 @@ variables:
GIT_PASSWORD: "@url@http://vault-secrets-provider/api/secrets/b7ecb6ebabc231/git/semantic-release?field=group-access-token"
GIT_PRIVATE_KEY: "@url@http://vault-secrets-provider/api/secrets/b7ecb6ebabc231/git/semantic-release?field=private-key"
PYTHON_REPOSITORY_PASSWORD: "@url@http://vault-secrets-provider/api/secrets/b7ecb6ebabc231/pip-repo/repository?field=password"
- # $VAULT_ROLE_ID and $VAULT_SECRET_ID defined as a secret CI/CD variable
```
### Google Cloud variant
-This variant allows to use Python Google Clients. The variant follow the recommendation [Authenticate for using client libraries](https://cloud.google.com/docs/authentication/client-libraries) with [ADC](https://cloud.google.com/docs/authentication/application-default-credentials)
+This variant allows to use Python Google Clients. The variant follow the recommendation [Authenticate for using client libraries](https://cloud.google.com/docs/authentication/client-libraries) with [ADC](https://cloud.google.com/docs/authentication/application-default-credentials)
[Detailed article on internal OIDC impersonated with Workload Identify Federation](https://blog.salrashid.dev/articles/2021/understanding_workload_identity_federation/#oidc-impersonated)
@@ -579,15 +622,84 @@ The variant requires the additional configuration parameters:
```yaml
include:
- - component: $CI_SERVER_FQDN/to-be-continuous/python/gitlab-ci-python@7.0.2
+ - component: $CI_SERVER_FQDN/to-be-continuous/python/gitlab-ci-python@7.7.1
# 2: set/override component inputs
inputs:
image: registry.hub.docker.com/library/python:3.12-slim
pytest-enabled: true
- - component: $CI_SERVER_FQDN/to-be-continuous/python/gitlab-ci-python-gcp@7.0.2
+ - component: $CI_SERVER_FQDN/to-be-continuous/python/gitlab-ci-python-gcp@7.7.1
inputs:
# common OIDC config for non-prod envs
gcp-oidc-provider: "projects/<gcp_nonprod_proj_id>/locations/global/workloadIdentityPools/<pool_id>/providers/<provider_id>"
gcp-oidc-account: "<name>@$<gcp_nonprod_proj_id>.iam.gserviceaccount.com"
```
+
+### AWS CodeArtifact variant
+
+This variant allows to use PyPi packages from AWS CodeArtifact. The variant follow the recommendation [Authenticate for using client libraries](https://docs.aws.amazon.com/codeartifact/latest/ug/python-configure.html)
+
+It authenticates with AWS CodeArtifact, retrieves and sets the following environment variable:
+
+- `CODEARTIFACT_AUTH_TOKEN` - the AWS CodeArtifact authentication token
+- `CODEARTIFACT_REPOSITORY_ENDPOINT` - the AWS CodeArtifact repository endpoint
+- `CODEARTIFACT_URL` - Formatted URL for the AWS CodeArtifact repository
+
+Most importantly, the variant sets the `pip global.index-url` to the CodeArtifact url.
+
+The variant supports two authentication methods:
+
+1. [federated authentication using OpenID Connect](https://docs.gitlab.com/ee/ci/cloud_services/aws/) (**recommended method**),
+2. or basic authentication with AWS access key ID & secret access key.
+
+:warning: when using this variant, you must have created the CodeArtifact repository.
+
+#### Configuration
+
+The variant *requires* the additional configuration parameters:
+
+| Input / Variable | Description | Default value |
+| --------------------------------------------- | --------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------- |
+| `TBC_AWS_PROVIDER_IMAGE` | The [AWS Auth Provider](https://gitlab.com/to-be-continuous/tools/aws-auth-provider) image to use (can be overridden) | `registry.gitlab.com/to-be-continuous/tools/aws-auth-provider:latest` |
+| `aws-region` / `AWS_REGION` | Default region (where the Codeartifact repository is located) | _none_ |
+| `aws-codeartifact-domain` / `AWS_CODEARTIFACT_DOMAIN` | The CodeArtifact domain name | _none_ |
+| `aws-codeartifact-domain-owner` / `AWS_CODEARTIFACT_DOMAIN_OWNER` | The CodeArtifact domain owner account ID | _none_ |
+| `aws-codeartifact-repository` / `AWS_CODEARTIFACT_REPOSITORY` | The CodeArtifact repository name | _none_ |
+
+##### OIDC authentication config
+
+This is the recommended authentication method. In order to use it, first carefuly follow [GitLab's documentation](https://docs.gitlab.com/ee/ci/cloud_services/aws/),
+then set the required configuration.
+
+| Input / Variable | Description | Default value |
+| ----------------------------------------------------------- | ---------------------------------------------------------------------------------------------- | ---------------- |
+| `aws-oidc-aud` / `AWS_OIDC_AUD` | The `aud` claim for the JWT token | `$CI_SERVER_URL` |
+| `aws-oidc-role-arn` / `AWS_OIDC_ROLE_ARN` | Default IAM Role ARN associated with GitLab | _none_ |
+
+##### Basic authentication config
+
+| Variable | Description | Default value |
+| --------------------------------------- | ---------------------------------------------------------------------------- | ----------------- |
+| :lock: `AWS_ACCESS_KEY_ID` | Default access key ID | _none_ (disabled) |
+| :lock: `AWS_SECRET_ACCESS_KEY` | Default secret access key | _none_ (disabled) |
+
+
+#### Example
+
+```yaml
+include:
+ - component: $CI_SERVER_FQDN/to-be-continuous/python/gitlab-ci-python@7.7.1
+ # 2: set/override component inputs
+ inputs:
+ image: registry.hub.docker.com/library/python:3.12-slim
+ pytest-enabled: true
+
+ - component: $CI_SERVER_FQDN/to-be-continuous/python/gitlab-ci-python-aws-codeartifact@7.7.1
+ inputs:
+ aws-region: "us-east-1"
+ aws-codeartifact-domain: "acme"
+ aws-codeartifact-domain-owner: "123456789012"
+ aws-codeartifact-repository: "my-repo"
+ # common OIDC config for non-prod envs
+ aws-oidc-role-arn: "arn:aws:iam::123456789012:role/gitlab-ci"
+```
\ No newline at end of file
diff --git a/bumpversion.sh b/bumpversion.sh
index 329e866dac988c049574a0a9f26ba89979c523a8..708faf434d2459d63b2bdaceada5eb32b0fd39eb 100755
--- a/bumpversion.sh
+++ b/bumpversion.sh
@@ -27,7 +27,7 @@ if [[ "$curVer" ]]; then
log_info "Bump version from \\e[33;1m${curVer}\\e[0m to \\e[33;1m${nextVer}\\e[0m (release type: $relType)..."
# replace in README
- sed -e "s/ref: *'$curVer'/ref: '$nextVer'/" -e "s/ref: *\"$curVer\”/ref: \”$nextVer\”/" -e "s/component: *\(.*\)@$curVer/component: \1@$nextVer/" README.md > README.md.next
+ sed -e "s/ref: *'$curVer'/ref: '$nextVer'/" -e "s/ref: *\"$curVer\"/ref: \"$nextVer\"/" -e "s/component: *\(.*\)@$curVer/component: \1@$nextVer/" README.md > README.md.next
mv -f README.md.next README.md
# replace in template and variants
diff --git a/kicker.json b/kicker.json
index 21f7e470f6c038d696697c5b1c75838f9f0c97b3..5cbcf8f5dad02b7326364dc34afe6d07e50da0ce 100644
--- a/kicker.json
+++ b/kicker.json
@@ -21,7 +21,7 @@
"name": "PYTHON_BUILD_SYSTEM",
"description": "Python build-system to use to install dependencies, build and package the project",
"type": "enum",
- "values": ["auto", "setuptools", "poetry", "pipenv", "reqfile"],
+ "values": ["auto", "setuptools", "poetry", "pipenv", "reqfile", "uv"],
"default": "auto",
"advanced": true
},
@@ -83,6 +83,12 @@
}
]
},
+ {
+ "id":"publish",
+ "name":"publish",
+ "description":"This job allows publishing the built packages to a PyPI compatible repository ([GitLab packages](https://docs.gitlab.com/ee/user/packages/pypi_repository/) by default.",
+ "enable_with": "PYTHON_PUBLISH_ENABLED"
+ },
{
"id": "pylint",
"name": "pylint",
@@ -192,7 +198,7 @@
{
"id": "trivy",
"name": "Trivy",
- "description": "Detect security vulnerabilities with [Trivy](https://github.com/aquasecurity/trivy/) (dependencies analysis)",
+ "description": "Detect security vulnerabilities with [Trivy](https://aquasecurity.github.io/trivy) (dependencies analysis)",
"disable_with": "PYTHON_TRIVY_DISABLED",
"variables": [
{
@@ -202,8 +208,8 @@
},
{
"name": "PYTHON_TRIVY_ARGS",
- "description": "Additional [Trivy CLI options](https://aquasecurity.github.io/trivy/v0.21.1/getting-started/cli/fs/)",
- "default": "--vuln-type library",
+ "description": "Additional [Trivy CLI options](https://aquasecurity.github.io/trivy/latest/docs/references/configuration/cli/trivy_filesystem/)",
+ "default": "--ignore-unfixed --pkg-types library --detection-priority comprehensive",
"advanced": true
},
{
@@ -253,6 +259,12 @@
"description": "Manually trigger a release of your code (uses [bumpversion](https://pypi.org/project/bumpversion/))",
"enable_with": "PYTHON_RELEASE_ENABLED",
"variables": [
+ {
+ "name": "PYTHON_AUTO_RELEASE_ENABLED",
+ "description": "When set the job start automatically. When not set (default), the job is manual. Note that this behavior also depends on release-enabled being set.",
+ "type": "boolean",
+ "advanced": true
+ },
{
"name": "PYTHON_RELEASE_NEXT",
"type": "enum",
@@ -375,6 +387,14 @@
}
]
},
+ {
+ "id": "ruff-format",
+ "name": "Ruff Format",
+ "description": "An extremely fast Python linter and code formatter, written in Rust. [Ruff](https://docs.astral.sh/ruff/)",
+ "enable_with": "RUFF_FORMAT_ENABLED",
+ "variables": [
+ ]
+ },
{
"id": "mypy",
"name": "mypy",
@@ -459,6 +479,61 @@
"description": "Default Workload Identity Provider associated with GitLab to [authenticate with OpenID Connect](https://docs.gitlab.com/ee/ci/cloud_services/google_cloud/)"
}
]
+ },
+ {
+ "id": "aws-codeartifact",
+ "name": "AWS CodeArtifact",
+ "description": "Retrieves AWS CodeArtifact credentials",
+ "template_path": "templates/gitlab-ci-python-aws-codeartifact.yml",
+ "variables": [
+ {
+ "name": "TBC_AWS_PROVIDER_IMAGE",
+ "description": "The [AWS Auth Provider](https://gitlab.com/to-be-continuous/tools/aws-auth-provider) image to use",
+ "default": "registry.gitlab.com/to-be-continuous/tools/aws-auth-provider:latest",
+ "advanced": true
+ },
+ {
+ "name": "AWS_REGION",
+ "description": "Default region (where the codeartifact repository is located)"
+ },
+ {
+ "name": "AWS_OIDC_AUD",
+ "description": "The `aud` claim for the JWT token _(only required for [OIDC authentication](https://docs.gitlab.com/ee/ci/cloud_services/aws/))_",
+ "default": "$CI_SERVER_URL",
+ "advanced": true
+ },
+ {
+ "name": "AWS_OIDC_ROLE_ARN",
+ "description": "Default IAM Role ARN associated with GitLab _(only required for [OIDC authentication](https://docs.gitlab.com/ee/ci/cloud_services/aws/))_"
+ },
+ {
+ "name": "AWS_ACCESS_KEY_ID",
+ "description": "Default access key ID (only required for basic authentication)",
+ "secret": true,
+ "advanced": true
+ },
+ {
+ "name": "AWS_SECRET_ACCESS_KEY",
+ "description": "Default secret access key (only required for basic authentication)",
+ "secret": true,
+ "advanced": true
+ },
+ {
+ "name": "AWS_CODEARTIFACT_DOMAIN",
+ "description": "The AWS CodeArtifact domain",
+ "mandatory": true
+ },
+ {
+ "name": "AWS_CODEARTIFACT_DOMAIN_OWNER",
+ "description": "The AWS CodeArtifact domain owner",
+ "mandatory": true
+ },
+ {
+ "name": "AWS_CODEARTIFACT_REPOSITORY",
+ "description": "The AWS CodeArtifact repository",
+ "mandatory": true
+ }
+ ]
}
]
-}
\ No newline at end of file
+}
diff --git a/templates/gitlab-ci-python-aws-codeartifact.yml b/templates/gitlab-ci-python-aws-codeartifact.yml
new file mode 100644
index 0000000000000000000000000000000000000000..60baea0c05de835f801a4090f0b0db3977dc2641
--- /dev/null
+++ b/templates/gitlab-ci-python-aws-codeartifact.yml
@@ -0,0 +1,61 @@
+# =====================================================================================================================
+# === AWS CodeArtifact Auth template variant
+# =====================================================================================================================
+spec:
+ inputs:
+ aws-codeartifact-domain:
+ description: AWS CodeArtifact domain name
+ default: ''
+ aws-codeartifact-domain-owner:
+ description: AWS CodeArtifact domain owner account ID
+ default: ''
+ aws-codeartifact-repository:
+ description: AWS CodeArtifact repository name
+ default: ''
+ aws-region:
+ description: Default region (where the Codeartifact registry is located)
+ default: ''
+ aws-oidc-aud:
+ description: The `aud` claim for the JWT token _(only required for [OIDC authentication](https://docs.gitlab.com/ee/ci/cloud_services/aws/))_
+ default: $CI_SERVER_URL
+ aws-oidc-role-arn:
+ description: Default IAM Role ARN associated with GitLab _(only required for [OIDC
+ authentication](https://docs.gitlab.com/ee/ci/cloud_services/aws/))_
+ default: ''
+---
+variables:
+ TBC_AWS_PROVIDER_IMAGE: registry.gitlab.com/to-be-continuous/tools/aws-auth-provider:latest
+ AWS_OIDC_AUD: $[[ inputs.aws-oidc-aud ]]
+ AWS_REGION: $[[ inputs.aws-region ]]
+ AWS_OIDC_ROLE_ARN: $[[ inputs.aws-oidc-role-arn ]]
+ AWS_CODEARTIFACT_DOMAIN: $[[ inputs.aws-codeartifact-domain ]]
+ AWS_CODEARTIFACT_DOMAIN_OWNER: $[[ inputs.aws-codeartifact-domain-owner ]]
+ AWS_CODEARTIFACT_REPOSITORY: $[[ inputs.aws-codeartifact-repository ]]
+
+
+.codeartifact-pip-config:
+ before_script:
+ - CODEARTIFACT_URL=https://aws:${PYTHON_REPOSITORY_PASSWORD}@${PYTHON_REPOSITORY_URL#https://}simple
+ - pip config set global.index-url $CODEARTIFACT_URL
+
+.python-base:
+ services:
+ - name: "$TBC_TRACKING_IMAGE"
+ command: ["--service", "python", "7.3.0"]
+ - name: "$TBC_AWS_PROVIDER_IMAGE"
+ alias: "aws-auth-provider"
+ id_tokens:
+ # required for OIDC auth
+ AWS_JWT:
+ aud: "$AWS_OIDC_AUD"
+ variables:
+ PYTHON_REPOSITORY_USERNAME: aws
+ PYTHON_REPOSITORY_PASSWORD: "@url@http://aws-auth-provider/codeartifact/auth/token"
+ PYTHON_REPOSITORY_URL: "@url@http://aws-auth-provider/codeartifact/repository/endpoint?format=pypi"
+ AWS_JWT: "$AWS_JWT"
+ before_script:
+ - !reference [.python-scripts]
+ - install_ca_certs "${CUSTOM_CA_CERTS:-$DEFAULT_CA_CERTS}"
+ - cd ${PYTHON_PROJECT_DIR}
+ - guess_build_system
+ - !reference [.codeartifact-pip-config, before_script]
diff --git a/templates/gitlab-ci-python-gcp.yml b/templates/gitlab-ci-python-gcp.yml
index f39d794c55f45d2c7edd5dba586666cc853483db..b93ad6a4db137b134a74d07fb3d3daa5da704d1a 100644
--- a/templates/gitlab-ci-python-gcp.yml
+++ b/templates/gitlab-ci-python-gcp.yml
@@ -44,7 +44,7 @@ variables:
image: $PYTHON_IMAGE
services:
- name: "$TBC_TRACKING_IMAGE"
- command: ["--service", "python", "7.0.2"]
+ command: ["--service", "python", "7.7.1"]
variables:
GCP_JWT: $GCP_JWT
before_script:
diff --git a/templates/gitlab-ci-python-vault.yml b/templates/gitlab-ci-python-vault.yml
index 41370aace3fec112bfec7b8bc800570041a5f1de..349cd49e0b53812b4d4e94f585e2cc03942e38d0 100644
--- a/templates/gitlab-ci-python-vault.yml
+++ b/templates/gitlab-ci-python-vault.yml
@@ -22,7 +22,7 @@ variables:
.python-base:
services:
- name: "$TBC_TRACKING_IMAGE"
- command: ["--service", "python", "7.0.2"]
+ command: ["--service", "python", "7.7.1"]
- name: "$TBC_VAULT_IMAGE"
alias: "vault-secrets-provider"
variables:
diff --git a/templates/gitlab-ci-python.yml b/templates/gitlab-ci-python.yml
index 1376a989395d2cee17ada305fd3c7178ea8735f3..2ccfa2773350027f5bb06ab0cc54ce54f3b632ff 100644
--- a/templates/gitlab-ci-python.yml
+++ b/templates/gitlab-ci-python.yml
@@ -29,6 +29,7 @@ spec:
- poetry
- pipenv
- reqfile
+ - uv
default: auto
reqs-file:
description: |-
@@ -107,8 +108,8 @@ spec:
_When unset, the latest version will be used_
default: ''
trivy-args:
- description: Additional [Trivy CLI options](https://aquasecurity.github.io/trivy/v0.21.1/getting-started/cli/fs/)
- default: --vuln-type library
+ description: Additional [Trivy CLI options](https://aquasecurity.github.io/trivy/latest/docs/references/configuration/cli/trivy_filesystem/)
+ default: --ignore-unfixed --pkg-types library --detection-priority comprehensive
sbom-disabled:
description: Disable Software Bill of Materials
type: boolean
@@ -129,6 +130,14 @@ spec:
description: Enable Release
type: boolean
default: false
+ auto-release-enabled:
+ description: When set the job start automatically on production branch. When not set (default), the job is manual. Note that this behavior also depends on release-enabled being set.
+ type: boolean
+ default: false
+ publish-enabled:
+ description: Enable Publish Package
+ type: boolean
+ default: false
release-next:
description: 'The part of the version to increase (one of: `major`, `minor`, `patch`)'
options:
@@ -165,9 +174,10 @@ spec:
ruff-args:
description: Additional [Ruff Linter CLI options](https://docs.astral.sh/ruff/configuration/#full-command-line-interface)
default: ""
- ruff-ext-exclude:
- description: Define [extend-exclude](https://docs.astral.sh/ruff/settings/#extend-exclude) files
- default: ""
+ ruff-format-enabled:
+ description: Enable Ruff
+ type: boolean
+ default: false
mypy-enabled:
description: Enable mypy
type: boolean
@@ -301,7 +311,7 @@ variables:
# default integration ref name (pattern)
INTEG_REF: '/^develop$/'
# default release tag name (pattern)
- RELEASE_REF: '/^v?[0-9]+\.[0-9]+\.[0-9]+$/'
+ RELEASE_REF: '/^v?[0-9]+\.[0-9]+\.[0-9]+(-[a-zA-Z0-9-\.]+)?(\+[a-zA-Z0-9-\.]+)?$/'
# compileall
PYTHON_COMPILE_ARGS: $[[ inputs.compile-args ]]
@@ -346,12 +356,14 @@ variables:
BANDIT_ENABLED: $[[ inputs.bandit-enabled ]]
PYTHON_SBOM_DISABLED: $[[ inputs.sbom-disabled ]]
PYTHON_RELEASE_ENABLED: $[[ inputs.release-enabled ]]
+ PYTHON_PUBLISH_ENABLED: $[[ inputs.publish-enabled ]]
+ PYTHON_AUTO_RELEASE_ENABLED: $[[ inputs.auto-release-enabled ]]
PYTHON_BLACK_ENABLED: $[[ inputs.black-enabled ]]
PYTHON_ISORT_ENABLED: $[[ inputs.isort-enabled ]]
RUFF_ENABLED: $[[ inputs.ruff-enabled ]]
RUFF_ARGS: $[[ inputs.ruff-args ]]
- RUFF_EXT_EXCLUDE: $[[ inputs.ruff-ext-exclude ]]
+ RUFF_FORMAT_ENABLED: $[[ inputs.ruff-format-enabled ]]
MYPY_ENABLED: $[[ inputs.mypy-enabled ]]
MYPY_ARGS: $[[ inputs.mypy-args ]]
MYPY_FILES: $[[ inputs.mypy-files ]]
@@ -582,9 +594,9 @@ variables:
decoded=$(mktemp)
errors=$(mktemp)
# shellcheck disable=SC2086
- if python3 -c "import urllib.request ; urllib.request.urlretrieve(\"$url\",\"${decoded}\")" > "${errors}" 2>&1
+ if python3 -c "import urllib.request ; urllib.request.urlretrieve(\"$url\",\"${decoded}\")" > "${errors}" 2>&1
then
- export ${name}="$(cat ${decoded})"
+ export ${name}="$(cat ${decoded})"
log_info "Successfully fetched secret \\e[33;1m${name}\\e[0m"
else
log_warn "Failed getting secret \\e[33;1m${name}\\e[0m:\\n$(sed 's/^/... /g' "${errors}")"
@@ -630,7 +642,7 @@ variables:
case "${PYTHON_BUILD_SYSTEM:-auto}" in
auto)
;;
- poetry*|setuptools*|pipenv*)
+ poetry*|setuptools*|pipenv*|uv*)
log_info "--- Build system explicitly declared: ${PYTHON_BUILD_SYSTEM}"
return
;;
@@ -649,6 +661,17 @@ variables:
export PYTHON_BUILD_SYSTEM="reqfile"
return
fi
+
+ if [[ -f "uv.lock" ]]
+ then
+ if [[ -f "pyproject.toml" ]]
+ then
+ log_info "--- Build system auto-detected: uv (uv.lock and pyproject.toml)"
+ export PYTHON_BUILD_SYSTEM="uv"
+ return
+ fi
+ log_error "--- Build system auto-detected: uv (uv.lock) but no pyproject.toml found: please read template doc"
+ fi
if [[ -f "pyproject.toml" ]]
then
@@ -691,7 +714,14 @@ variables:
}
function maybe_install_poetry() {
- if [[ "$PYTHON_BUILD_SYSTEM" =~ ^poetry.* ]] && ! command -v poetry > /dev/null
+ if [[ "$PYTHON_BUILD_SYSTEM" =~ ^poetry ]] && ! command -v poetry > /dev/null
+ then
+ # shellcheck disable=SC2086
+ pip install ${PIP_OPTS} "$PYTHON_BUILD_SYSTEM"
+ fi
+ }
+ function maybe_install_uv() {
+ if [[ "$PYTHON_BUILD_SYSTEM" =~ ^uv ]] && ! command -v uv > /dev/null
then
# shellcheck disable=SC2086
pip install ${PIP_OPTS} "$PYTHON_BUILD_SYSTEM"
@@ -741,14 +771,25 @@ variables:
log_warn "--- requirements build system defined, but no ${PYTHON_REQS_FILE} file found"
fi
;;
+ uv*)
+ if [[ ! -f "uv.lock" ]]; then
+ log_warn "Using uv but \\e[33;1muv.lock\\e[0m file not found: you shall commit it with your project files"
+ fi
+ maybe_install_uv
+ uv sync --frozen ${PYTHON_EXTRA_DEPS:+--extra "$PYTHON_EXTRA_DEPS"}
+ ;;
esac
}
function _run() {
- if [[ "$PYTHON_BUILD_SYSTEM" == poetry* ]]
+ if [[ "$PYTHON_BUILD_SYSTEM" =~ ^poetry ]]
then
maybe_install_poetry
poetry run "$@"
+ elif [[ "$PYTHON_BUILD_SYSTEM" =~ ^uv ]]
+ then
+ maybe_install_uv
+ uv run "$@"
else
"$@"
fi
@@ -759,15 +800,26 @@ variables:
}
function _pip() {
- # shellcheck disable=SC2086
- _run pip ${PIP_OPTS} "$@"
+ if [[ "$PYTHON_BUILD_SYSTEM" =~ ^uv ]]
+ then
+ maybe_install_uv
+ # shellcheck disable=SC2086
+ uv pip ${PIP_OPTS} "$@"
+ else
+ # shellcheck disable=SC2086
+ _run pip ${PIP_OPTS} "$@"
+ fi
}
function py_package() {
- if [[ "$PYTHON_BUILD_SYSTEM" == poetry* ]]
+ if [[ "$PYTHON_BUILD_SYSTEM" =~ ^poetry ]]
then
maybe_install_poetry
poetry build
+ elif [[ "$PYTHON_BUILD_SYSTEM" =~ ^uv ]]
+ then
+ maybe_install_uv
+ uv build
else
# shellcheck disable=SC2086
pip install ${PIP_OPTS} build
@@ -815,7 +867,7 @@ variables:
fi
# 2: bump-my-version (+ Git commit & tag)
- if [[ "$PYTHON_BUILD_SYSTEM" == poetry* ]]
+ if [[ "$PYTHON_BUILD_SYSTEM" =~ ^poetry ]]
then
maybe_install_poetry
if [[ -z "$py_next_version" ]]
@@ -833,6 +885,33 @@ variables:
py_commit_message=$(python -c "print('$PYTHON_RELEASE_COMMIT_MESSAGE'.format(current_version='$py_cur_version', new_version='$py_next_version'))")
git commit -m "$py_commit_message"
git tag "$py_next_version"
+ elif [[ "$PYTHON_BUILD_SYSTEM" =~ ^uv ]]
+ then
+ maybe_install_uv
+ if [[ -z "$py_next_version" ]]
+ then
+ # quick version waiting for uv to manage bump
+ # related uv MR https://github.com/astral-sh/uv/pull/7248#issuecomment-2395465334
+ mkdir -p -m 777 tbc_tmp
+ uvx --from toml-cli toml get --toml-path pyproject.toml project.version > tbc_tmp/version.txt
+ py_cur_version=$(cat tbc_tmp/version.txt)
+
+ py_release_part="$PYTHON_RELEASE_NEXT"
+ log_info "[bump-my-version] increase \\e[1;94m${py_release_part}\\e[0m (from current \\e[1;94m${py_cur_version}\\e[0m)"
+ uvx bump-my-version bump ${TRACE+--verbose} --current-version "$py_cur_version" "$py_release_part" tbc_tmp/version.txt
+ py_next_version=$(cat tbc_tmp/version.txt)
+ rm -fr tbc_tmp/version.txt
+ fi
+
+ log_info "[uv] change version \\e[1;94m${py_cur_version}\\e[0m → \\e[1;94m${py_next_version}\\e[0m"
+ uvx --from toml-cli toml set --toml-path pyproject.toml project.version "$py_next_version"
+
+ # Git commit and tag
+ git add pyproject.toml
+ # emulate bump-my-version to generate commit message
+ py_commit_message=$(python -c "print('$PYTHON_RELEASE_COMMIT_MESSAGE'.format(current_version='$py_cur_version', new_version='$py_next_version'))")
+ git commit -m "$py_commit_message"
+ git tag --force "$py_next_version"
else
# Setuptools / bump-my-version
# shellcheck disable=SC2086
@@ -873,10 +952,10 @@ variables:
}
function py_publish() {
- if [[ "$PYTHON_BUILD_SYSTEM" == poetry* ]]
+ if [[ "$PYTHON_BUILD_SYSTEM" =~ ^poetry ]]
then
maybe_install_poetry
-
+
if [[ "$PYTHON_PACKAGE_ENABLED" != "true" ]]
then
log_info "--- build packages (poetry)..."
@@ -886,10 +965,22 @@ variables:
log_info "--- publish packages (poetry) to $PYTHON_REPOSITORY_URL with user $PYTHON_REPOSITORY_USERNAME..."
poetry config repositories.user_defined "$PYTHON_REPOSITORY_URL"
poetry publish ${TRACE+--verbose} --username "$PYTHON_REPOSITORY_USERNAME" --password "$PYTHON_REPOSITORY_PASSWORD" --repository user_defined
+ elif [[ "$PYTHON_BUILD_SYSTEM" =~ ^uv ]]
+ then
+ maybe_install_uv
+
+ if [[ "$PYTHON_PACKAGE_ENABLED" != "true" ]]
+ then
+ log_info "--- build packages (uv)..."
+ uv build ${TRACE+--verbose}
+ fi
+
+ log_info "--- publish packages (uv) to $PYTHON_REPOSITORY_URL with user $PYTHON_REPOSITORY_USERNAME..."
+ uv publish ${TRACE+--verbose} --username "$PYTHON_REPOSITORY_USERNAME" --password "$PYTHON_REPOSITORY_PASSWORD" --publish-url "$PYTHON_REPOSITORY_URL"
else
# shellcheck disable=SC2086
pip install ${PIP_OPTS} build twine
-
+
if [[ "$PYTHON_PACKAGE_ENABLED" != "true" ]]
then
log_info "--- build packages (build)..."
@@ -935,19 +1026,20 @@ stages:
- production
###############################################################################################
-# Generic python job #
+# Generic python jobs #
###############################################################################################
.python-base:
image: $PYTHON_IMAGE
services:
- name: "$TBC_TRACKING_IMAGE"
- command: ["--service", "python", "7.0.2"]
+ command: ["--service", "python", "7.7.1"]
variables:
# set local cache dir; most Python tools honour XDG specs
XDG_CACHE_HOME: "$CI_PROJECT_DIR/.cache"
PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
POETRY_CACHE_DIR: "$CI_PROJECT_DIR/.cache/poetry"
PIPENV_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pipenv"
+ UV_CACHE_DIR: "$CI_PROJECT_DIR/.cache/uv"
POETRY_VIRTUALENVS_IN_PROJECT: "false"
cache:
key: "$CI_COMMIT_REF_SLUG-python"
@@ -960,6 +1052,24 @@ stages:
- cd ${PYTHON_PROJECT_DIR}
- guess_build_system
+.python-test:
+ extends: .python-base
+ stage: build
+ coverage: /^TOTAL.+?(\d+(?:\.\d+)?\%)$/
+ artifacts:
+ name: "$CI_JOB_NAME artifacts from $CI_PROJECT_NAME on $CI_COMMIT_REF_SLUG"
+ expire_in: 1 day
+ when: always
+ reports:
+ junit:
+ - "$PYTHON_PROJECT_DIR/reports/TEST-*.xml"
+ coverage_report:
+ coverage_format: cobertura
+ path: "$PYTHON_PROJECT_DIR/reports/py-coverage.cobertura.xml"
+ paths:
+ - "$PYTHON_PROJECT_DIR/reports/TEST-*.xml"
+ - "$PYTHON_PROJECT_DIR/reports/py-coverage.*"
+
###############################################################################################
# build stage #
###############################################################################################
@@ -985,7 +1095,7 @@ py-lint:
- install_requirements
- _pip install pylint_gitlab # codeclimate reports
# run pylint and generate reports all at once
- - _run pylint --ignore=.cache --output-format=colorized,pylint_gitlab.GitlabCodeClimateReporter:reports/py-lint.codeclimate.json,parseable:reports/py-lint.parseable.txt ${PYLINT_ARGS} ${PYLINT_FILES:-$(find -type f -name "*.py")}
+ - _run pylint --output-format=colorized,pylint_gitlab.GitlabCodeClimateReporter:reports/py-lint.codeclimate.json,parseable:reports/py-lint.parseable.txt ${PYLINT_ARGS} ${PYLINT_FILES:-$(find -type f -name "*.py" -not -path "./.cache/*")}
artifacts:
name: "$CI_JOB_NAME artifacts from $CI_PROJECT_NAME on $CI_COMMIT_REF_SLUG"
expire_in: 1 day
@@ -1020,7 +1130,7 @@ py-black:
script:
- install_requirements
- _pip install black
- - _run black . --check
+ - _run black . --check --extend-exclude '(\/\.cache\/|\/\.venv\/)'
rules:
# exclude if $PYTHON_BLACK_ENABLED not set
- if: '$PYTHON_BLACK_ENABLED != "true"'
@@ -1034,7 +1144,7 @@ py-isort:
script:
- install_requirements
- _pip install isort
- - _run isort . --check-only
+ - _run isort . --check-only --extend-skip .cache --extend-skip .venv
rules:
# exclude if $PYTHON_ISORT_ENABLED not set
- if: '$PYTHON_ISORT_ENABLED != "true"'
@@ -1047,9 +1157,9 @@ py-ruff:
stage: build
script:
- mkdir -p -m 777 reports
- - |
- if [[ ${BANDIT_ENABLED} == "true" || ${PYLINT_ENABLED} == "true" || ${PYTHON_ISORT_ENABLED} == "true" || ${PYTHON_BLACK_ENABLED} == "true" ]]; then
- log_warn "Ruff can replace isort, Black, Bandit, Pylint"
+ - |
+ if [[ ${BANDIT_ENABLED} == "true" || ${PYLINT_ENABLED} == "true" || ${PYTHON_ISORT_ENABLED} == "true" ]]; then
+ log_warn "Ruff can replace isort, Bandit, Pylint"
fi
# Ruff is self dependent tool (written in Rust), it can be installed without project dependencies (_pip and _run don't look required here)
- pip install ${PIP_OPTS} ruff
@@ -1057,10 +1167,10 @@ py-ruff:
- |
if [[ "$SONAR_HOST_URL" ]]
then
- ruff check . ${RUFF_ARGS} ${RUFF_EXCLUDE:---extend-exclude .venv,.cache} --exit-zero --output-format json --output-file reports/py-ruff.native.json
+ ruff check . ${RUFF_ARGS} --extend-exclude .venv,.cache --exit-zero --output-format json --output-file reports/py-ruff.native.json
fi
# then GitLab and grouped/console formats
- - ruff check . ${RUFF_ARGS} ${RUFF_EXCLUDE:---extend-exclude .venv,.cache} --output-format gitlab --output-file reports/py-ruff.gitlab.json || ruff check . ${RUFF_ARGS} ${RUFF_EXCLUDE:---extend-exclude .venv,.cache} --output-format grouped
+ - ruff check . ${RUFF_ARGS} --extend-exclude .venv,.cache --output-format gitlab --output-file reports/py-ruff.gitlab.json || ruff check . ${RUFF_ARGS} --extend-exclude .venv,.cache --output-format grouped
artifacts:
name: "$CI_JOB_NAME artifacts from $CI_PROJECT_NAME on $CI_COMMIT_REF_SLUG"
expire_in: 1 day
@@ -1076,6 +1186,23 @@ py-ruff:
- !reference [.test-policy, rules]
tags: $[[ inputs.py-ruff-job-tags ]]
+py-ruff-format:
+ extends: .python-base
+ stage: build
+ script:
+ - |
+ if [[ ${PYTHON_BLACK_ENABLED} == "true" ]]; then
+ log_warn "Ruff can replace Black"
+ fi
+ # Ruff is self dependent tool (written in Rust), it can be installed without project dependencies (_pip and _run don't look required here)
+ - pip install ${PIP_OPTS} ruff
+ - ruff format --check . --exclude .venv,.cache
+ rules:
+ # exclude if $RUFF_FORMAT_ENABLED not set
+ - if: '$RUFF_FORMAT_ENABLED != "true"'
+ when: never
+ - !reference [.test-policy, rules]
+
py-mypy:
extends: .python-base
stage: build
@@ -1085,7 +1212,7 @@ py-mypy:
- mkdir -p -m 777 reports
- install_requirements
- _pip install mypy mypy-to-codeclimate
- - _run mypy ${MYPY_ARGS} ${MYPY_FILES:-$(find -type f -name "*.py" -not -path "./.cache/*")} | tee reports/py-mypy.console.txt || true
+ - _run mypy ${MYPY_ARGS} ${MYPY_FILES:-$(find -type f -name "*.py" -not -path "./.cache/*" -not -path "./.venv/*")} | tee reports/py-mypy.console.txt || true
# mypy-to-codeclimate will fail if any error was found
- _run mypy-to-codeclimate reports/py-mypy.console.txt reports/py-mypy.codeclimate.json
artifacts:
@@ -1107,8 +1234,7 @@ py-mypy:
# test stage #
###############################################################################################
py-unittest:
- extends: .python-base
- stage: build
+ extends: .python-test
script:
- mkdir -p -m 777 reports
- install_requirements
@@ -1119,20 +1245,6 @@ py-unittest:
- _run coverage run -m xmlrunner discover -o "reports/" $UNITTEST_ARGS
- _run coverage report -m
- _run coverage xml -o "reports/py-coverage.cobertura.xml"
- coverage: /^TOTAL.+?(\d+\%)$/
- artifacts:
- name: "$CI_JOB_NAME artifacts from $CI_PROJECT_NAME on $CI_COMMIT_REF_SLUG"
- expire_in: 1 day
- when: always
- reports:
- junit:
- - "$PYTHON_PROJECT_DIR/reports/TEST-*.xml"
- coverage_report:
- coverage_format: cobertura
- path: "$PYTHON_PROJECT_DIR/reports/py-coverage.cobertura.xml"
- paths:
- - "$PYTHON_PROJECT_DIR/reports/TEST-*.xml"
- - "$PYTHON_PROJECT_DIR/reports/py-coverage.*"
rules:
# skip if $UNITTEST_ENABLED not set
- if: '$UNITTEST_ENABLED != "true"'
@@ -1141,27 +1253,12 @@ py-unittest:
tags: $[[ inputs.py-unittest-job-tags ]]
py-pytest:
- extends: .python-base
- stage: build
+ extends: .python-test
script:
- mkdir -p -m 777 reports
- install_requirements
- _pip install pytest pytest-cov coverage
- _python -m pytest --junit-xml=reports/TEST-pytests.xml --cov --cov-report term --cov-report xml:reports/py-coverage.cobertura.xml ${PYTEST_ARGS}
- coverage: /^TOTAL.+?(\d+\%)$/
- artifacts:
- name: "$CI_JOB_NAME artifacts from $CI_PROJECT_NAME on $CI_COMMIT_REF_SLUG"
- expire_in: 1 day
- when: always
- reports:
- junit:
- - "$PYTHON_PROJECT_DIR/reports/TEST-*.xml"
- coverage_report:
- coverage_format: cobertura
- path: "$PYTHON_PROJECT_DIR/reports/py-coverage.cobertura.xml"
- paths:
- - "$PYTHON_PROJECT_DIR/reports/TEST-*.xml"
- - "$PYTHON_PROJECT_DIR/reports/py-coverage.*"
rules:
# skip if $PYTEST_ENABLED not set
- if: '$PYTEST_ENABLED != "true"'
@@ -1170,26 +1267,11 @@ py-pytest:
tags: $[[ inputs.py-pytest-job-tags ]]
py-nosetests:
- extends: .python-base
- stage: build
+ extends: .python-test
script:
- mkdir -p -m 777 reports
- install_requirements
- _run nosetests --with-xunit --xunit-file=reports/TEST-nosetests.xml --with-coverage --cover-erase --cover-xml --cover-xml-file=reports/py-coverage.cobertura.xml ${NOSETESTS_ARGS}
- coverage: /^TOTAL.+?(\d+\%)$/
- artifacts:
- name: "$CI_JOB_NAME artifacts from $CI_PROJECT_NAME on $CI_COMMIT_REF_SLUG"
- expire_in: 1 day
- when: always
- reports:
- junit:
- - "$PYTHON_PROJECT_DIR/reports/TEST-*.xml"
- coverage_report:
- coverage_format: cobertura
- path: "$PYTHON_PROJECT_DIR/reports/py-coverage.cobertura.xml"
- paths:
- - "$PYTHON_PROJECT_DIR/reports/TEST-*.xml"
- - "$PYTHON_PROJECT_DIR/reports/py-coverage.*"
rules:
# skip if $NOSETESTS_ENABLED not set
- if: '$NOSETESTS_ENABLED != "true"'
@@ -1211,19 +1293,20 @@ py-bandit:
- |
if [[ "$SONAR_HOST_URL" ]]
then
- _run bandit ${TRACE+--verbose} --exit-zero --exclude ./.cache --format csv --output reports/py-bandit.bandit.csv ${BANDIT_ARGS}
+ _run bandit ${TRACE+--verbose} --exit-zero --exclude ./.cache,./.venv --format csv --output reports/py-bandit.bandit.csv ${BANDIT_ARGS}
fi
# JSON (for DefectDojo)
- |
if [[ "$DEFECTDOJO_BANDIT_REPORTS" ]]
then
- _run bandit ${TRACE+--verbose} --exit-zero --exclude ./.cache --format json --output reports/py-bandit.bandit.json ${BANDIT_ARGS}
+ _run bandit ${TRACE+--verbose} --exit-zero --exclude ./.cache,./.venv --format json --output reports/py-bandit.bandit.json ${BANDIT_ARGS}
fi
- - _run bandit ${TRACE+--verbose} --exclude ./.cache ${BANDIT_ARGS}
+ - _run bandit ${TRACE+--verbose} --exclude ./.cache,./.venv ${BANDIT_ARGS}
artifacts:
when: always
name: "$CI_JOB_NAME artifacts from $CI_PROJECT_NAME on $CI_COMMIT_REF_SLUG"
expire_in: 1 day
+ access: developer
paths:
- "$PYTHON_PROJECT_DIR/reports/py-bandit.*"
rules:
@@ -1241,7 +1324,6 @@ py-trivy:
dependencies: []
script:
- mkdir -p -m 777 reports
- - install_requirements
- |
if [[ -z "$PYTHON_TRIVY_DIST_URL" ]]
then
@@ -1260,17 +1342,24 @@ py-trivy:
tar zxf trivy.tar.gz trivy
mkdir -p $XDG_CACHE_HOME
mv ./trivy $python_trivy
- fi
- - |
- if [[ "$PYTHON_BUILD_SYSTEM" == poetry* ]]
- then
- # When using Poetry, `pip freeze` outputs a requirements.txt with @file URLs for each wheel
- # These @file URLs in requirements.txt are not supported by Trivy
- # So instead of simply using pip freeze, we use `poetry export`
- poetry export -f requirements.txt --without-hashes --output reports/requirements.txt
- else
- _pip freeze | tee ./reports/requirements.txt
fi
+ - |
+ case "$PYTHON_BUILD_SYSTEM" in
+ poetry*|pipenv*)
+ log_info "$PYTHON_BUILD_SYSTEM build system (\\e[32muse lock file\\e[0m)"
+ cp poetry.lock Pipfile.lock ./reports 2>/dev/null || true
+ ;;
+ uv*)
+ log_info "$PYTHON_BUILD_SYSTEM build system used (\\e[32mmust generate pinned requirements.txt from uv.lock\\e[0m)"
+ maybe_install_uv
+ uv export > ./reports/requirements.txt
+ ;;
+ *)
+ log_info "$PYTHON_BUILD_SYSTEM build system used (\\e[32mmust generate pinned requirements.txt\\e[0m)"
+ install_requirements
+ _pip freeze | tee ./reports/requirements.txt
+ ;;
+ esac
if [[ -f "./requirements.txt" ]]
then
sort -u ./requirements.txt | grep -v "^[ ]*$" > ./requirements.txt.sorted
@@ -1280,18 +1369,22 @@ py-trivy:
log_warn "The ./requirements.txt file does not match the ./reports/requirements.txt file generated via pip freeze. Make sure to include all dependencies with pinned versions in ./requirements.txt and re-commit the file."
fi
fi
- if [ $($python_trivy fs ${PYTHON_TRIVY_ARGS} --format table --exit-code 0 ./reports/ | grep -c "Number of language-specific files: 0") -eq 1 ]; then
+
+ # Generate the native JSON report that can later be converted to other formats
+ $python_trivy fs ${PYTHON_TRIVY_ARGS} --format json --list-all-pkgs --output reports/py-trivy.trivy.json --exit-code 1 ./reports/ > ./reports/trivy.log 2>&1 || exit_code=$?
+ cat ./reports/trivy.log
+ if [ $(grep -ic "Number of language-specific files[^0-9]*0$" ./reports/trivy.log) -eq 1 ]; then
log_error "Could not find a file listing all dependencies with their versions."
exit 1
fi
- if [[ "$DEFECTDOJO_TRIVY_REPORTS" ]]
- then
- $python_trivy fs ${PYTHON_TRIVY_ARGS} --exit-code 0 --list-all-pkgs --format json --output reports/py-trivy.trivy.json ./reports/
- fi
- $python_trivy fs ${PYTHON_TRIVY_ARGS} --format table ./reports/
+ rm ./reports/trivy.log
+ # console output
+ $python_trivy convert --format table reports/py-trivy.trivy.json
+ exit $exit_code
artifacts:
name: "$CI_JOB_NAME artifacts from $CI_PROJECT_NAME on $CI_COMMIT_REF_SLUG"
expire_in: 1 day
+ access: developer
when: always
paths:
- "$PYTHON_PROJECT_DIR/reports/py-trivy.*"
@@ -1311,10 +1404,19 @@ py-sbom:
needs: []
script:
- mkdir -p -m 777 reports
- - install_requirements
- |
case "$PYTHON_BUILD_SYSTEM" in
- setuptools*|reqfile)
+ poetry*|pipenv*)
+ log_info "$PYTHON_BUILD_SYSTEM build system (\\e[32muse lock file\\e[0m)"
+ ;;
+ uv*)
+ log_info "$PYTHON_BUILD_SYSTEM build system used (\\e[32mmust generate pinned requirements.txt from uv.lock\\e[0m)"
+ maybe_install_uv
+ uv export > ./reports/requirements.txt
+ ;;
+ *)
+ log_info "$PYTHON_BUILD_SYSTEM build system used (\\e[32mmust generate pinned requirements.txt\\e[0m)"
+ install_requirements
_pip freeze > "${PYTHON_REQS_FILE}"
;;
esac
@@ -1342,7 +1444,7 @@ py-sbom:
paths:
- "$PYTHON_PROJECT_DIR/reports/py-sbom.cyclonedx.json"
reports:
- cyclonedx:
+ cyclonedx:
- "$PYTHON_PROJECT_DIR/reports/py-sbom.cyclonedx.json"
rules:
# exclude if disabled
@@ -1372,6 +1474,8 @@ py-release:
# exclude if $PYTHON_RELEASE_ENABLED not set
- if: '$PYTHON_RELEASE_ENABLED != "true"'
when: never
+ # on production branch: auto if $PYTHON_AUTO_RELEASE_ENABLED set and implicitly $PYTHON_RELEASE_ENABLED set
+ - if: '$PYTHON_AUTO_RELEASE_ENABLED == "true" && $CI_COMMIT_REF_NAME =~ $PROD_REF'
# on production or integration branch: manual, non blocking
- if: '$CI_COMMIT_REF_NAME =~ $PROD_REF || $CI_COMMIT_REF_NAME =~ $INTEG_REF'
when: manual
@@ -1393,7 +1497,7 @@ py-publish:
- $PYTHON_PROJECT_DIR/dist/*
rules:
# exclude if $PYTHON_RELEASE_ENABLED not set
- - if: '$PYTHON_RELEASE_ENABLED != "true"'
+ - if: '$PYTHON_RELEASE_ENABLED != "true" && $PYTHON_PUBLISH_ENABLED != "true"'
when: never
# on tag with release pattern: auto
- if: '$CI_COMMIT_TAG =~ $RELEASE_REF'