Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • smartdatalab/public/ci-cd-components/python
1 result
Show changes
Commits on Source (20)
include: include:
- project: 'to-be-continuous/tools/gitlab-ci' - component: git.code.tecnalia.com/smartdatalab/public/ci-cd-components/gitlab-ci/extract@master
ref: 'master' inputs:
file: '/templates/extract.yml' extract-script-job-tags: ["docker"]
- project: 'to-be-continuous/tools/gitlab-ci' - component: git.code.tecnalia.com/smartdatalab/public/ci-cd-components/gitlab-ci/validation@master
ref: 'master' inputs:
file: '/templates/validation.yml' check-links-job-tags: ["docker"]
- project: 'to-be-continuous/kicker' tbc-check-job-tags: ["docker"]
ref: 'master' tbc-check-image: registry.gitlab.com/gbenguria/tbc-check:main
file: '/templates/validation.yml' gitlab-ci-lint-job-tags: ["docker"]
- project: 'to-be-continuous/bash' - component: git.code.tecnalia.com/smartdatalab/public/ci-cd-components/kicker/validation@master
ref: '3.4' inputs:
file: '/templates/gitlab-ci-bash.yml' kicker-validation-job-tags: ["docker"]
- project: 'to-be-continuous/semantic-release' schema-base-url: "https://git.code.tecnalia.com/api/v4/projects/smartdatalab%2Fpublic%2Fci-cd-components%2Fkicker/repository/files"
ref: '3.8' - component: git.code.tecnalia.com/smartdatalab/public/ci-cd-components/bash/gitlab-ci-bash@master
file: '/templates/gitlab-ci-semrel.yml' inputs:
bash-shellcheck-job-tags: ["docker"]
- component: git.code.tecnalia.com/smartdatalab/public/ci-cd-components/semantic-release/gitlab-ci-semrel@master
inputs:
semantic-release-job-tags: ["docker"]
stages: stages:
- build - build
......
# [6.9.0](https://git.code.tecnalia.com/smartdatalab/public/ci-cd-components/python/compare/6.8.1...6.9.0) (2024-05-15)
### Bug Fixes
* **cache:** always keep cache ([8cf171e](https://git.code.tecnalia.com/smartdatalab/public/ci-cd-components/python/commit/8cf171edde7577b4179530192169730eff122ff2))
* **workflow:** disable MR pipeline from prod & integ branches ([8703dea](https://git.code.tecnalia.com/smartdatalab/public/ci-cd-components/python/commit/8703dea915ca48038588adcc2a0672269dd24bb4))
### Features
* add ⚡ Ruff linter job ([ef364ef](https://git.code.tecnalia.com/smartdatalab/public/ci-cd-components/python/commit/ef364ef65dad8034b7c7af60bb25195cf20ba466))
* add mypy job ([cc22e8c](https://git.code.tecnalia.com/smartdatalab/public/ci-cd-components/python/commit/cc22e8c0eda56d2df00e22b8034da40fd2cfc937))
* **ruff:** generate JSON report when SonarQube is detected ([81e711d](https://git.code.tecnalia.com/smartdatalab/public/ci-cd-components/python/commit/81e711d2cd6e23014c0eaef2ec098fe69b711885))
* some log improvement ([aef5a05](https://git.code.tecnalia.com/smartdatalab/public/ci-cd-components/python/commit/aef5a056d09b22e2c6ce2fa8038632c912e11637))
## [6.11.1](https://gitlab.com/to-be-continuous/python/compare/6.11.0...6.11.1) (2024-05-05)
### Bug Fixes
* **workflow:** disable MR pipeline from prod & integ branches ([8703dea](https://gitlab.com/to-be-continuous/python/commit/8703dea915ca48038588adcc2a0672269dd24bb4))
# [6.11.0](https://gitlab.com/to-be-continuous/python/compare/6.10.0...6.11.0) (2024-04-28)
### Bug Fixes
* **cache:** always keep cache ([8cf171e](https://gitlab.com/to-be-continuous/python/commit/8cf171edde7577b4179530192169730eff122ff2))
### Features
* add mypy job ([cc22e8c](https://gitlab.com/to-be-continuous/python/commit/cc22e8c0eda56d2df00e22b8034da40fd2cfc937))
# [6.10.0](https://gitlab.com/to-be-continuous/python/compare/6.9.0...6.10.0) (2024-04-14)
### Features
* **ruff:** generate JSON report when SonarQube is detected ([81e711d](https://gitlab.com/to-be-continuous/python/commit/81e711d2cd6e23014c0eaef2ec098fe69b711885))
# [6.9.0](https://gitlab.com/to-be-continuous/python/compare/6.8.1...6.9.0) (2024-04-13)
### Features
* add ⚡ Ruff linter job ([ef364ef](https://gitlab.com/to-be-continuous/python/commit/ef364ef65dad8034b7c7af60bb25195cf20ba466))
* some log improvement ([aef5a05](https://gitlab.com/to-be-continuous/python/commit/aef5a056d09b22e2c6ce2fa8038632c912e11637))
## [6.8.1](https://gitlab.com/to-be-continuous/python/compare/6.8.0...6.8.1) (2024-04-03) ## [6.8.1](https://gitlab.com/to-be-continuous/python/compare/6.8.0...6.8.1) (2024-04-03)
......
...@@ -14,7 +14,7 @@ Add the following to your `gitlab-ci.yml`: ...@@ -14,7 +14,7 @@ Add the following to your `gitlab-ci.yml`:
```yaml ```yaml
include: include:
# 1: include the component # 1: include the component
- component: gitlab.com/to-be-continuous/python/gitlab-ci-python@6.8.1 - component: gitlab.com/to-be-continuous/python/gitlab-ci-python@6.11.1
# 2: set/override component inputs # 2: set/override component inputs
inputs: inputs:
image: registry.hub.docker.com/library/python:3.10 image: registry.hub.docker.com/library/python:3.10
...@@ -29,7 +29,7 @@ Add the following to your `gitlab-ci.yml`: ...@@ -29,7 +29,7 @@ Add the following to your `gitlab-ci.yml`:
include: include:
# 1: include the template # 1: include the template
- project: 'to-be-continuous/python' - project: 'to-be-continuous/python'
ref: '6.8.1' ref: '6.11.1'
file: '/templates/gitlab-ci-python.yml' file: '/templates/gitlab-ci-python.yml'
variables: variables:
...@@ -82,6 +82,10 @@ This job allows building your Python project [distribution packages](https://pac ...@@ -82,6 +82,10 @@ This job allows building your Python project [distribution packages](https://pac
It is bound to the `build` stage, it is **disabled by default** and can be enabled by setting `$PYTHON_PACKAGE_ENABLED` to `true`. It is bound to the `build` stage, it is **disabled by default** and can be enabled by setting `$PYTHON_PACKAGE_ENABLED` to `true`.
| Input / Variable | Description | Default value |
| ------------------------ | ---------------------------------- | ----------------- |
| `py-package-job-tags` / `PY_PACKAGE_JOB_TAGS` | Tags to be used for selecting runners for the job | `[]` |
### Lint jobs ### Lint jobs
#### `py-lint` job #### `py-lint` job
...@@ -93,6 +97,7 @@ It is bound to the `build` stage, and uses the following variables: ...@@ -93,6 +97,7 @@ It is bound to the `build` stage, and uses the following variables:
| Input / Variable | Description | Default value | | Input / Variable | Description | Default value |
| ------------------------ | ---------------------------------- | ----------------- | | ------------------------ | ---------------------------------- | ----------------- |
| `pylint-enabled` / `PYLINT_ENABLED` | Set to `true` to enable the `pylint` job | _none_ (disabled) |
| `pylint-args` / `PYLINT_ARGS` | Additional [pylint CLI options](http://pylint.pycqa.org/en/latest/user_guide/run.html#command-line-options) | _none_ | | `pylint-args` / `PYLINT_ARGS` | Additional [pylint CLI options](http://pylint.pycqa.org/en/latest/user_guide/run.html#command-line-options) | _none_ |
| `pylint-files` / `PYLINT_FILES` | Files or directories to analyse | _none_ (by default analyses all found python source files) | | `pylint-files` / `PYLINT_FILES` | Files or directories to analyse | _none_ (by default analyses all found python source files) |
...@@ -123,6 +128,7 @@ It is bound to the `build` stage, and uses the following variables: ...@@ -123,6 +128,7 @@ It is bound to the `build` stage, and uses the following variables:
| Input / Variable | Description | Default value | | Input / Variable | Description | Default value |
| ------------------------ | -------------------------------------------------------------------- | ----------------------- | | ------------------------ | -------------------------------------------------------------------- | ----------------------- |
| `unittest-enabled` / `UNITTEST_ENABLED` | Set to `true` to enable the `unittest` job | _none_ (disabled) |
| `unittest-args` / `UNITTEST_ARGS` | Additional xmlrunner/unittest CLI options | _none_ | | `unittest-args` / `UNITTEST_ARGS` | Additional xmlrunner/unittest CLI options | _none_ |
:information_source: use a `.coveragerc` file at the root of your Python project to control the coverage settings. :information_source: use a `.coveragerc` file at the root of your Python project to control the coverage settings.
...@@ -155,7 +161,9 @@ It is bound to the `build` stage, and uses the following variables: ...@@ -155,7 +161,9 @@ It is bound to the `build` stage, and uses the following variables:
| Input / Variable | Description | Default value | | Input / Variable | Description | Default value |
| ------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------- | | ------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------- |
| `pytest-enabled` / `PYTEST_ENABLED` | Set to `true` to enable the `pytest` job | _none_ (disabled) |
| `pytest-args` / `PYTEST_ARGS` | Additional [pytest](https://docs.pytest.org/en/stable/usage.html) or [pytest-cov](https://github.com/pytest-dev/pytest-cov#usage) CLI options | _none_ | | `pytest-args` / `PYTEST_ARGS` | Additional [pytest](https://docs.pytest.org/en/stable/usage.html) or [pytest-cov](https://github.com/pytest-dev/pytest-cov#usage) CLI options | _none_ |
| `py-pytest-job-tags` / `PY_PYTEST_JOB_TAGS` | Tags to be used for selecting runners for the job | `[]` |
:information_source: use a `.coveragerc` file at the root of your Python project to control the coverage settings. :information_source: use a `.coveragerc` file at the root of your Python project to control the coverage settings.
...@@ -187,6 +195,7 @@ It is bound to the `build` stage, and uses the following variables: ...@@ -187,6 +195,7 @@ It is bound to the `build` stage, and uses the following variables:
| Input / Variable | Description | Default value | | Input / Variable | Description | Default value |
| ------------------------ | --------------------------------------------------------------------------------------- | ----------------------- | | ------------------------ | --------------------------------------------------------------------------------------- | ----------------------- |
| `nosetests-enabled` / `NOSETESTS_ENABLED` | Set to `true` to enable the `nose` job | _none_ (disabled) |
| `nosetests-args` / `NOSETESTS_ARGS` | Additional [nose CLI options](https://nose.readthedocs.io/en/latest/usage.html#options) | _none_ | | `nosetests-args` / `NOSETESTS_ARGS` | Additional [nose CLI options](https://nose.readthedocs.io/en/latest/usage.html#options) | _none_ |
By default coverage will be run on all the project directories. You can restrict it to your packages by setting the `$NOSE_COVER_PACKAGE` variable. By default coverage will be run on all the project directories. You can restrict it to your packages by setting the `$NOSE_COVER_PACKAGE` variable.
...@@ -282,6 +291,45 @@ This job **disabled by default** and runs [isort](https://pycqa.github.io/isort/ ...@@ -282,6 +291,45 @@ This job **disabled by default** and runs [isort](https://pycqa.github.io/isort/
| ---------------- | ----------------------------------------------------------------------- | ----------------- | | ---------------- | ----------------------------------------------------------------------- | ----------------- |
| `isort-enabled` / `PYTHON_ISORT_ENABLED` | Set to `true` to enable isort job | _none_ (disabled) | | `isort-enabled` / `PYTHON_ISORT_ENABLED` | Set to `true` to enable isort job | _none_ (disabled) |
### `py-ruff` job
This job **disabled by default** and runs [Ruff](https://docs.astral.sh/ruff/) on the repo. It is bound to the build stage.
| Input / Variable | Description | Default value |
| ---------------- | ----------------------------------------------------------------------- | ----------------- |
| `ruff-enabled` / `RUFF_ENABLED` | Set to `true` to enable ruff job | _none_ (disabled) |
| `ruff-args` / `RUFF_ARGS` | Additional [Ruff Linter CLI options](https://docs.astral.sh/ruff/configuration/#full-command-line-interface) | _none_ |
| `ruff-ext-exclude` / `RUFF_EXT_EXCLUDE` | Define [extend-exclude](https://docs.astral.sh/ruff/settings/#extend-exclude) files | _.venv,.cache_ |
| `py-ruff-job-tags` / `PY_RUFF_JOB_TAGS` | Tags to be used for selecting runners for the job | `[]` |
:warning: Ruff can replace isort, Black, Bandit, Pylint and much more. [More info](https://github.com/astral-sh/ruff/blob/main/docs/faq.md#which-tools-does-ruff-replace).
In addition to logs in the console, this job produces the following reports, kept for one week:
| Report | Format | Usage |
| -------------- | ---------------------------------------------------------------------------- | ----------------- |
| `$PYTHON_PROJECT_DIR/reports/py-ruff.gitlab.json` | [GitLab](https://docs.astral.sh/ruff/settings/#output-format) | [GitLab integration](https://docs.gitlab.com/ee/ci/yaml/artifacts_reports.html#artifactsreportscodequality) |
| `$PYTHON_PROJECT_DIR/reports/py-ruff.native.json` | [JSON](https://docs.astral.sh/ruff/settings/#output-format) | [SonarQube integration](https://docs.sonarqube.org/latest/analysis/external-issues/)<br/>_This report is generated only if SonarQube template is detected_ |
#### `py-mypy` job
This job is **disabled by default** and performs code analysis based on [mypy](https://mypy.readthedocs.io/en/stable/).
It is activated by setting `$MYPY_ENABLED` to `true`.
It is bound to the `build` stage, and uses the following variables:
| Input / Variable | Description | Default value |
| ------------------------ | ---------------------------------- | ----------------- |
| `mypy-enabled` / `MYPY_ENABLED` | Set to `true` to enable the `mypy` job | _none_ (disabled) |
| `mypy-args` / `MYPY_ARGS` | Additional [mypy CLI options](https://mypy.readthedocs.io/en/stable/command_line.html) | _none_ |
| `mypy-files` / `MYPY_FILES` | Files or directories to analyse | _none_ (by default analyses all found python source files) |
In addition to a textual report in the console, this job produces the following reports, kept for one day:
| Report | Format | Usage |
| -------------- | ---------------------------------------------------------------------------- | ----------------- |
| `$PYTHON_PROJECT_DIR/reports/py-mypy.codeclimate.json` | [Code Climate](https://github.com/soul-catcher/mypy-gitlab-code-quality) | [GitLab integration](https://docs.gitlab.com/ee/ci/yaml/artifacts_reports.html#artifactsreportscodequality) |
| `$PYTHON_PROJECT_DIR/reports/py-mypy.console.txt` | [mypy console output](https://mypy.readthedocs.io/) | [SonarQube integration](https://docs.sonarqube.org/latest/analysis/external-issues/) |
### SonarQube analysis ### SonarQube analysis
...@@ -306,6 +354,10 @@ sonar.python.coverage.reportPaths=reports/py-coverage.cobertura.xml ...@@ -306,6 +354,10 @@ sonar.python.coverage.reportPaths=reports/py-coverage.cobertura.xml
sonar.python.pylint.reportPaths=reports/py-lint.parseable.txt sonar.python.pylint.reportPaths=reports/py-lint.parseable.txt
# Bandit: CSV format (if enabled) # Bandit: CSV format (if enabled)
sonar.python.bandit.reportPaths=reports/py-bandit.bandit.csv sonar.python.bandit.reportPaths=reports/py-bandit.bandit.csv
# Ruff: JSON format (if enabled)
sonar.python.ruff.reportPaths=reports/py-ruff.native.json
# mypy: JSON format (if enabled)
sonar.python.mypy.reportPaths=reports/py-mypy.console.txt
``` ```
More info: More info:
...@@ -342,6 +394,7 @@ The release job is bound to the `publish` stage, appears only on production and ...@@ -342,6 +394,7 @@ The release job is bound to the `publish` stage, appears only on production and
| `repository-url` / `PYTHON_REPOSITORY_URL`| Target PyPI repository to publish packages | _[GitLab project's PyPI packages repository](https://docs.gitlab.com/ee/user/packages/pypi_repository/)_ | | `repository-url` / `PYTHON_REPOSITORY_URL`| Target PyPI repository to publish packages | _[GitLab project's PyPI packages repository](https://docs.gitlab.com/ee/user/packages/pypi_repository/)_ |
| `PYTHON_REPOSITORY_USERNAME`| Target PyPI repository username credential | `gitlab-ci-token` | | `PYTHON_REPOSITORY_USERNAME`| Target PyPI repository username credential | `gitlab-ci-token` |
| :lock: `PYTHON_REPOSITORY_PASSWORD`| Target PyPI repository password credential | `$CI_JOB_TOKEN` | | :lock: `PYTHON_REPOSITORY_PASSWORD`| Target PyPI repository password credential | `$CI_JOB_TOKEN` |
| `py-release-job-tags` / `PY_RELEASE_JOB_TAGS` | Tags to be used for selecting runners for the job | `[]` |
#### Setuptools tip #### Setuptools tip
...@@ -470,9 +523,9 @@ With: ...@@ -470,9 +523,9 @@ With:
```yaml ```yaml
include: include:
# main component # main component
- component: gitlab.com/to-be-continuous/python/gitlab-ci-python@6.8.1 - component: gitlab.com/to-be-continuous/python/gitlab-ci-python@6.11.1
# Vault variant # Vault variant
- component: gitlab.com/to-be-continuous/python/gitlab-ci-python-vault@6.8.1 - component: gitlab.com/to-be-continuous/python/gitlab-ci-python-vault@6.11.1
inputs: inputs:
vault-base-url: "https://vault.acme.host/v1" vault-base-url: "https://vault.acme.host/v1"
# audience claim for JWT # audience claim for JWT
...@@ -504,7 +557,7 @@ The variant requires the additional configuration parameters: ...@@ -504,7 +557,7 @@ The variant requires the additional configuration parameters:
| Input / Variable | Description | Default value | | Input / Variable | Description | Default value |
| ----------------- | -------------------------------------- | ----------------- | | ----------------- | -------------------------------------- | ----------------- |
| `gcp-oidc-aud` / `GCP_OIDC_AUD` | The `aud` claim for the JWT token | `$CI_SERVER_URL` | | `gcp-oidc-aud` / `GCP_OIDC_AUD` | The `aud` claim for the JWT token _(only required for [OIDC authentication](https://docs.gitlab.com/ee/ci/cloud_services/google_cloud/))_ | `$CI_SERVER_URL` |
| `gcp-oidc-provider` / `GCP_OIDC_PROVIDER` | Default Workload Identity Provider associated with GitLab to [authenticate with OpenID Connect](https://docs.gitlab.com/ee/ci/cloud_services/google_cloud/) | _none_ | | `gcp-oidc-provider` / `GCP_OIDC_PROVIDER` | Default Workload Identity Provider associated with GitLab to [authenticate with OpenID Connect](https://docs.gitlab.com/ee/ci/cloud_services/google_cloud/) | _none_ |
| `gcp-oidc-account` / `GCP_OIDC_ACCOUNT` | Default Service Account to which impersonate with OpenID Connect authentication | _none_ | | `gcp-oidc-account` / `GCP_OIDC_ACCOUNT` | Default Service Account to which impersonate with OpenID Connect authentication | _none_ |
...@@ -512,13 +565,13 @@ The variant requires the additional configuration parameters: ...@@ -512,13 +565,13 @@ The variant requires the additional configuration parameters:
```yaml ```yaml
include: include:
- component: gitlab.com/to-be-continuous/python/gitlab-ci-python@6.8.1 - component: gitlab.com/to-be-continuous/python/gitlab-ci-python@6.11.1
# 2: set/override component inputs # 2: set/override component inputs
inputs: inputs:
image: registry.hub.docker.com/library/python:3.10 image: registry.hub.docker.com/library/python:3.10
pytest-enabled: true pytest-enabled: true
- component: gitlab.com/to-be-continuous/python/gitlab-ci-python-gcp@6.8.1 - component: gitlab.com/to-be-continuous/python/gitlab-ci-python-gcp@6.11.1
inputs: inputs:
# common OIDC config for non-prod envs # common OIDC config for non-prod envs
gcp-oidc-provider: "projects/<gcp_nonprod_proj_id>/locations/global/workloadIdentityPools/<pool_id>/providers/<provider_id>" gcp-oidc-provider: "projects/<gcp_nonprod_proj_id>/locations/global/workloadIdentityPools/<pool_id>/providers/<provider_id>"
......
...@@ -33,7 +33,7 @@ if [[ "$curVer" ]]; then ...@@ -33,7 +33,7 @@ if [[ "$curVer" ]]; then
# replace in template and variants # replace in template and variants
for tmpl in templates/*.yml for tmpl in templates/*.yml
do do
sed -e "s/command: *\[\"--service\", \"\(.*\)\", \"$curVer\"\]/command: [\"--service\", \"\1\", \"$nextVer\"]/" "$tmpl" > "$tmpl.next" sed -e "s/command: *\[ *\"--service\", *\"\(.*\)\", *\"$curVer\" *\]/command: [\"--service\", \"\1\", \"$nextVer\"]/" "$tmpl" > "$tmpl.next"
mv -f "$tmpl.next" "$tmpl" mv -f "$tmpl.next" "$tmpl"
done done
else else
......
...@@ -4,6 +4,7 @@ ...@@ -4,6 +4,7 @@
"template_path": "templates/gitlab-ci-python.yml", "template_path": "templates/gitlab-ci-python.yml",
"kind": "build", "kind": "build",
"prefix": "python", "prefix": "python",
"job_prefix": "py",
"is_component": true, "is_component": true,
"variables": [ "variables": [
{ {
...@@ -57,11 +58,20 @@ ...@@ -57,11 +58,20 @@
"id":"package", "id":"package",
"name":"package", "name":"package",
"description":"This job allows building your Python project [distribution packages](https://packaging.python.org/en/latest/glossary/#term-Distribution-Package).", "description":"This job allows building your Python project [distribution packages](https://packaging.python.org/en/latest/glossary/#term-Distribution-Package).",
"enable_with": "PYTHON_PACKAGE_ENABLED" "enable_with": "PYTHON_PACKAGE_ENABLED",
"variables": [
{
"name": "PY_PACKAGE_JOB_TAGS",
"description": "Tags to be used for selecting runners for the job",
"type": "array",
"default": [],
"advanced": true
}
]
}, },
{ {
"id": "py-lint", "id": "pylint",
"name": "py-lint", "name": "pylint",
"description": "Code analysis based on [pylint](http://pylint.pycqa.org/en/latest/) Python lib.", "description": "Code analysis based on [pylint](http://pylint.pycqa.org/en/latest/) Python lib.",
"enable_with": "PYLINT_ENABLED", "enable_with": "PYLINT_ENABLED",
"variables": [ "variables": [
...@@ -100,12 +110,19 @@ ...@@ -100,12 +110,19 @@
"name": "PYTEST_ARGS", "name": "PYTEST_ARGS",
"description": "Additional [pytest](https://docs.pytest.org/en/stable/usage.html) or [pytest-cov](https://github.com/pytest-dev/pytest-cov#usage) CLI options", "description": "Additional [pytest](https://docs.pytest.org/en/stable/usage.html) or [pytest-cov](https://github.com/pytest-dev/pytest-cov#usage) CLI options",
"advanced": true "advanced": true
},
{
"name": "PY_PYTEST_JOB_TAGS",
"description": "Tags to be used for selecting runners for the job",
"type": "array",
"default": [],
"advanced": true
} }
] ]
}, },
{ {
"id": "py-nosetests", "id": "nose",
"name": "py-nosetests", "name": "nose",
"description": "Unit tests based on [nose](https://nose.readthedocs.io/) framework", "description": "Unit tests based on [nose](https://nose.readthedocs.io/) framework",
"enable_with": "NOSETESTS_ENABLED", "enable_with": "NOSETESTS_ENABLED",
"variables": [ "variables": [
...@@ -238,6 +255,13 @@ ...@@ -238,6 +255,13 @@
"description": "Target PyPI repository password credential", "description": "Target PyPI repository password credential",
"secret": true, "secret": true,
"default": "$CI_JOB_TOKEN" "default": "$CI_JOB_TOKEN"
},
{
"name": "PY_RELEASE_JOB_TAGS",
"description": "Tags to be used for selecting runners for the job",
"type": "array",
"default": [],
"advanced": true
} }
] ]
}, },
...@@ -252,6 +276,49 @@ ...@@ -252,6 +276,49 @@
"name": "isort", "name": "isort",
"description": "Check imports order with [isort](https://pycqa.github.io/isort)", "description": "Check imports order with [isort](https://pycqa.github.io/isort)",
"enable_with": "PYTHON_ISORT_ENABLED" "enable_with": "PYTHON_ISORT_ENABLED"
},
{
"id": "ruff",
"name": "Ruff",
"description": "An extremely fast Python linter and code formatter, written in Rust. [Ruff](https://docs.astral.sh/ruff/)",
"enable_with": "RUFF_ENABLED",
"variables": [
{
"name": "RUFF_ARGS",
"description": "Additional [Ruff Linter CLI options](https://docs.astral.sh/ruff/configuration/#full-command-line-interface)",
"advanced": true
},
{
"name": "RUFF_EXT_EXCLUDE",
"description": "Define [extend-exclude](https://docs.astral.sh/ruff/settings/#extend-exclude) files",
"advanced": true
},
{
"name": "PY_RUFF_JOB_TAGS",
"description": "Tags to be used for selecting runners for the job",
"type": "array",
"default": [],
"advanced": true
}
]
},
{
"id": "mypy",
"name": "mypy",
"description": "Code analysis based on [mypy](https://mypy.readthedocs.io/).",
"enable_with": "MYPY_ENABLED",
"variables": [
{
"name": "MYPY_ARGS",
"description": "Additional [mypy CLI options](https://mypy.readthedocs.io/en/stable/command_line.html)",
"advanced": true
},
{
"name": "MYPY_FILES",
"description": "Files or directories to analyse",
"advanced": true
}
]
} }
], ],
"variants": [ "variants": [
...@@ -299,7 +366,7 @@ ...@@ -299,7 +366,7 @@
"variables": [ "variables": [
{ {
"name": "GCP_OIDC_AUD", "name": "GCP_OIDC_AUD",
"description": "The `aud` claim for the JWT token _(only required for [OIDC authentication](https://docs.gitlab.com/ee/ci/cloud_services/aws/))_", "description": "The `aud` claim for the JWT token _(only required for [OIDC authentication](https://docs.gitlab.com/ee/ci/cloud_services/google_cloud/))_",
"default": "$CI_SERVER_URL", "default": "$CI_SERVER_URL",
"advanced": true "advanced": true
}, },
......
...@@ -21,7 +21,8 @@ variables: ...@@ -21,7 +21,8 @@ variables:
.gcp-provider-auth: .gcp-provider-auth:
before_script: before_script:
- echo "Installing GCP authentication with env GOOGLE_APPLICATION_CREDENTIALS file" - set -e
- echo -e "[\\e[1;94mINFO\\e[0m] Installing GCP authentication with env GOOGLE_APPLICATION_CREDENTIALS file"
- echo $GCP_JWT > "$CI_BUILDS_DIR/.auth_token.jwt" - echo $GCP_JWT > "$CI_BUILDS_DIR/.auth_token.jwt"
- |- - |-
cat << EOF > "$CI_BUILDS_DIR/google_application_credentials.json" cat << EOF > "$CI_BUILDS_DIR/google_application_credentials.json"
...@@ -43,7 +44,7 @@ variables: ...@@ -43,7 +44,7 @@ variables:
image: $PYTHON_IMAGE image: $PYTHON_IMAGE
services: services:
- name: "$TBC_TRACKING_IMAGE" - name: "$TBC_TRACKING_IMAGE"
command: ["--service", "python", "6.8.1"] command: ["--service", "python", "6.11.1"]
variables: variables:
GCP_JWT: $GCP_JWT GCP_JWT: $GCP_JWT
before_script: before_script:
......
...@@ -22,7 +22,7 @@ variables: ...@@ -22,7 +22,7 @@ variables:
.python-base: .python-base:
services: services:
- name: "$TBC_TRACKING_IMAGE" - name: "$TBC_TRACKING_IMAGE"
command: ["--service", "python", "6.8.1"] command: ["--service", "python", "6.11.1"]
- name: "$TBC_VAULT_IMAGE" - name: "$TBC_VAULT_IMAGE"
alias: "vault-secrets-provider" alias: "vault-secrets-provider"
variables: variables:
......
...@@ -83,7 +83,7 @@ spec: ...@@ -83,7 +83,7 @@ spec:
description: Additional [pytest](https://docs.pytest.org/en/stable/usage.html) or [pytest-cov](https://github.com/pytest-dev/pytest-cov#usage) CLI options description: Additional [pytest](https://docs.pytest.org/en/stable/usage.html) or [pytest-cov](https://github.com/pytest-dev/pytest-cov#usage) CLI options
default: '' default: ''
nosetests-enabled: nosetests-enabled:
description: Enable nosetest description: Enable nose
type: boolean type: boolean
default: false default: false
nosetests-args: nosetests-args:
...@@ -155,12 +155,52 @@ spec: ...@@ -155,12 +155,52 @@ spec:
description: Enable isort description: Enable isort
type: boolean type: boolean
default: false default: false
ruff-enabled:
description: Enable Ruff
type: boolean
default: false
ruff-args:
description: Additional [Ruff Linter CLI options](https://docs.astral.sh/ruff/configuration/#full-command-line-interface)
default: ""
ruff-ext-exclude:
description: Define [extend-exclude](https://docs.astral.sh/ruff/settings/#extend-exclude) files
default: ""
mypy-enabled:
description: Enable mypy
type: boolean
default: false
mypy-args:
description: Additional [mypy CLI options](https://mypy.readthedocs.io/en/stable/command_line.html)
default: ""
mypy-files:
description: Files or directories to analyse
default: ''
py-package-job-tags:
description: tags to filter applicable runners for py-package job
type: array
default: []
py-pytest-job-tags:
description: tags to filter applicable runners for py-pytest job
type: array
default: []
py-ruff-job-tags:
description: tags to filter applicable runners for py-ruff job
type: array
default: []
py-release-job-tags:
description: tags to filter applicable runners for py-release job
type: array
default: []
--- ---
# default workflow rules: Merge Request pipelines # default workflow rules: Merge Request pipelines
workflow: workflow:
rules: rules:
# prevent branch pipeline when an MR is open (prefer MR pipeline) # prevent MR pipeline originating from production or integration branch(es)
- if: '$CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS' - if: '$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME =~ $PROD_REF || $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME =~ $INTEG_REF'
when: never
# on non-prod, non-integration branches: prefer MR pipeline over branch pipeline
- if: '$CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS && $CI_COMMIT_REF_NAME !~ $PROD_REF && $CI_COMMIT_REF_NAME !~ $INTEG_REF'
when: never when: never
- if: '$CI_COMMIT_MESSAGE =~ "/\[(ci skip|skip ci) on ([^],]*,)*tag(,[^],]*)*\]/" && $CI_COMMIT_TAG' - if: '$CI_COMMIT_MESSAGE =~ "/\[(ci skip|skip ci) on ([^],]*,)*tag(,[^],]*)*\]/" && $CI_COMMIT_TAG'
when: never when: never
...@@ -201,14 +241,6 @@ variables: ...@@ -201,14 +241,6 @@ variables:
# variabilized tracking image # variabilized tracking image
TBC_TRACKING_IMAGE: "registry.gitlab.com/to-be-continuous/tools/tracking:master" TBC_TRACKING_IMAGE: "registry.gitlab.com/to-be-continuous/tools/tracking:master"
# Change pip's cache directory to be inside the project directory since we can
# only cache local items.
PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
# Poetry support: force virtualenv not in project dir & use local cache dir
POETRY_CACHE_DIR: "$CI_PROJECT_DIR/.cache/poetry"
POETRY_VIRTUALENVS_IN_PROJECT: "false"
PIPENV_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pipenv"
# PYTHON_IMAGE: "registry.hub.docker.com/library/python:3" # PYTHON_IMAGE: "registry.hub.docker.com/library/python:3"
PYTHON_IMAGE: $[[ inputs.image ]] PYTHON_IMAGE: $[[ inputs.image ]]
# Default Python project root directory # Default Python project root directory
...@@ -270,6 +302,12 @@ variables: ...@@ -270,6 +302,12 @@ variables:
PYTHON_BLACK_ENABLED: $[[ inputs.black-enabled ]] PYTHON_BLACK_ENABLED: $[[ inputs.black-enabled ]]
PYTHON_ISORT_ENABLED: $[[ inputs.isort-enabled ]] PYTHON_ISORT_ENABLED: $[[ inputs.isort-enabled ]]
RUFF_ENABLED: $[[ inputs.ruff-enabled ]]
RUFF_ARGS: $[[ inputs.ruff-args ]]
RUFF_EXT_EXCLUDE: $[[ inputs.ruff-ext-exclude ]]
MYPY_ENABLED: $[[ inputs.mypy-enabled ]]
MYPY_ARGS: $[[ inputs.mypy-args ]]
MYPY_FILES: $[[ inputs.mypy-files ]]
.python-scripts: &python-scripts | .python-scripts: &python-scripts |
...@@ -759,7 +797,7 @@ variables: ...@@ -759,7 +797,7 @@ variables:
poetry build ${TRACE+--verbose} poetry build ${TRACE+--verbose}
fi fi
log_info "--- publish packages (poetry)..." log_info "--- publish packages (poetry) to $PYTHON_REPOSITORY_URL with user $PYTHON_REPOSITORY_USERNAME..."
poetry config repositories.user_defined "$PYTHON_REPOSITORY_URL" poetry config repositories.user_defined "$PYTHON_REPOSITORY_URL"
poetry publish ${TRACE+--verbose} --username "$PYTHON_REPOSITORY_USERNAME" --password "$PYTHON_REPOSITORY_PASSWORD" --repository user_defined poetry publish ${TRACE+--verbose} --username "$PYTHON_REPOSITORY_USERNAME" --password "$PYTHON_REPOSITORY_PASSWORD" --repository user_defined
else else
...@@ -773,7 +811,7 @@ variables: ...@@ -773,7 +811,7 @@ variables:
python -m build python -m build
fi fi
log_info "--- publish packages (twine)..." log_info "--- publish packages (twine) to $PYTHON_REPOSITORY_URL with user $PYTHON_REPOSITORY_USERNAME..."
twine upload ${TRACE+--verbose} --username "$PYTHON_REPOSITORY_USERNAME" --password "$PYTHON_REPOSITORY_PASSWORD" --repository-url "$PYTHON_REPOSITORY_URL" dist/* twine upload ${TRACE+--verbose} --username "$PYTHON_REPOSITORY_USERNAME" --password "$PYTHON_REPOSITORY_PASSWORD" --repository-url "$PYTHON_REPOSITORY_URL" dist/*
fi fi
} }
...@@ -805,15 +843,19 @@ stages: ...@@ -805,15 +843,19 @@ stages:
image: $PYTHON_IMAGE image: $PYTHON_IMAGE
services: services:
- name: "$TBC_TRACKING_IMAGE" - name: "$TBC_TRACKING_IMAGE"
command: ["--service", "python", "6.8.1"] command: ["--service", "python", "6.11.1"]
# Cache downloaded dependencies and plugins between builds. variables:
# To keep cache across branches add 'key: "$CI_JOB_NAME"' # set local cache dir; most Python tools honour XDG specs
XDG_CACHE_HOME: "$CI_PROJECT_DIR/.cache"
PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
POETRY_CACHE_DIR: "$CI_PROJECT_DIR/.cache/poetry"
PIPENV_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pipenv"
POETRY_VIRTUALENVS_IN_PROJECT: "false"
cache: cache:
key: "$CI_COMMIT_REF_SLUG-python" key: "$CI_COMMIT_REF_SLUG-python"
when: always
paths: paths:
- ${PIP_CACHE_DIR} - .cache
- ${POETRY_CACHE_DIR}
- ${PIPENV_CACHE_DIR}
before_script: before_script:
- !reference [.python-scripts] - !reference [.python-scripts]
- install_ca_certs "${CUSTOM_CA_CERTS:-$DEFAULT_CA_CERTS}" - install_ca_certs "${CUSTOM_CA_CERTS:-$DEFAULT_CA_CERTS}"
...@@ -835,6 +877,7 @@ py-package: ...@@ -835,6 +877,7 @@ py-package:
expire_in: 1 day expire_in: 1 day
rules: rules:
- if: '$PYTHON_PACKAGE_ENABLED == "true"' - if: '$PYTHON_PACKAGE_ENABLED == "true"'
tags: $[[ inputs.py-package-job-tags ]]
py-lint: py-lint:
extends: .python-base extends: .python-base
...@@ -897,6 +940,66 @@ py-isort: ...@@ -897,6 +940,66 @@ py-isort:
when: never when: never
- !reference [.test-policy, rules] - !reference [.test-policy, rules]
py-ruff:
extends: .python-base
stage: build
script:
- mkdir -p -m 777 reports
- |
if [[ ${BANDIT_ENABLED} == "true" || ${PYLINT_ENABLED} == "true" || ${PYTHON_ISORT_ENABLED} == "true" || ${PYTHON_BLACK_ENABLED} == "true" ]]; then
log_warn "Ruff can replace isort, Black, Bandit, Pylint"
fi
# Ruff is self dependent tool (written in Rust), it can be installed without project dependencies (_pip and _run don't look required here)
- pip install ${PIP_OPTS} ruff
# JSON output (for SonarQube)
- |
if [[ "$SONAR_HOST_URL" ]]
then
ruff check . ${RUFF_ARGS} ${RUFF_EXCLUDE:---extend-exclude .venv,.cache} --exit-zero --output-format json --output-file reports/py-ruff.native.json
fi
# then GitLab and grouped/console formats
- ruff check . ${RUFF_ARGS} ${RUFF_EXCLUDE:---extend-exclude .venv,.cache} --output-format gitlab --output-file reports/py-ruff.gitlab.json || ruff check . ${RUFF_ARGS} ${RUFF_EXCLUDE:---extend-exclude .venv,.cache} --output-format grouped
artifacts:
name: "$CI_JOB_NAME artifacts from $CI_PROJECT_NAME on $CI_COMMIT_REF_SLUG"
expire_in: 1 day
when: always
reports:
codequality: $PYTHON_PROJECT_DIR/reports/py-ruff.gitlab.json
paths:
- "$PYTHON_PROJECT_DIR/reports/py-ruff.*"
rules:
# exclude if $RUFF_ENABLED not set
- if: '$RUFF_ENABLED != "true"'
when: never
- !reference [.test-policy, rules]
tags: $[[ inputs.py-ruff-job-tags ]]
py-mypy:
extends: .python-base
stage: build
variables:
MYPY_CACHE_DIR: "$CI_PROJECT_DIR/.cache/mypy"
script:
- mkdir -p -m 777 reports
- install_requirements
- _pip install mypy mypy-to-codeclimate
- _run mypy ${MYPY_ARGS} ${MYPY_FILES:-$(find -type f -name "*.py" -not -path "./.cache/*")} | tee reports/py-mypy.console.txt || true
# mypy-to-codeclimate will fail if any error was found
- _run mypy-to-codeclimate reports/py-mypy.console.txt reports/py-mypy.codeclimate.json
artifacts:
name: "$CI_JOB_NAME artifacts from $CI_PROJECT_NAME on $CI_COMMIT_REF_SLUG"
expire_in: 1 day
when: always
reports:
codequality: $PYTHON_PROJECT_DIR/reports/py-mypy.codeclimate.json
paths:
- "$PYTHON_PROJECT_DIR/reports/py-mypy.*"
rules:
# exclude if $MYPY_ENABLED not set
- if: '$MYPY_ENABLED != "true"'
when: never
- !reference [.test-policy, rules]
############################################################################################### ###############################################################################################
# test stage # # test stage #
############################################################################################### ###############################################################################################
...@@ -960,6 +1063,7 @@ py-pytest: ...@@ -960,6 +1063,7 @@ py-pytest:
- if: '$PYTEST_ENABLED != "true"' - if: '$PYTEST_ENABLED != "true"'
when: never when: never
- !reference [.test-policy, rules] - !reference [.test-policy, rules]
tags: $[[ inputs.py-pytest-job-tags ]]
py-nosetests: py-nosetests:
extends: .python-base extends: .python-base
...@@ -1147,6 +1251,7 @@ py-release: ...@@ -1147,6 +1251,7 @@ py-release:
- if: '$CI_COMMIT_REF_NAME =~ $PROD_REF || $CI_COMMIT_REF_NAME =~ $INTEG_REF' - if: '$CI_COMMIT_REF_NAME =~ $PROD_REF || $CI_COMMIT_REF_NAME =~ $INTEG_REF'
when: manual when: manual
allow_failure: true allow_failure: true
tags: $[[ inputs.py-release-job-tags ]]
# (auto from release tag): publishes the Python package(s) to a PyPi registry # (auto from release tag): publishes the Python package(s) to a PyPi registry
py-publish: py-publish:
......