Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • smartdatalab/public/ci-cd-components/python
1 result
Show changes
Commits on Source (20)
Hide whitespace changes
Inline Side-by-side
.gitlab-ci.yml
View file @ af8cca5f
include:
- project: 'to-be-continuous/tools/gitlab-ci'
ref: 'master'
file: '/templates/extract.yml'
- project: 'to-be-continuous/tools/gitlab-ci'
ref: 'master'
file: '/templates/validation.yml'
- project: 'to-be-continuous/kicker'
ref: 'master'
file: '/templates/validation.yml'
- project: 'to-be-continuous/bash'
ref: '3.4'
file: '/templates/gitlab-ci-bash.yml'
- project: 'to-be-continuous/semantic-release'
ref: '3.8'
file: '/templates/gitlab-ci-semrel.yml'
- component: git.code.tecnalia.com/smartdatalab/public/ci-cd-components/gitlab-ci/extract@master
inputs:
extract-script-job-tags: ["docker"]
- component: git.code.tecnalia.com/smartdatalab/public/ci-cd-components/gitlab-ci/validation@master
inputs:
check-links-job-tags: ["docker"]
tbc-check-job-tags: ["docker"]
tbc-check-image: registry.gitlab.com/gbenguria/tbc-check:main
gitlab-ci-lint-job-tags: ["docker"]
- component: git.code.tecnalia.com/smartdatalab/public/ci-cd-components/kicker/validation@master
inputs:
kicker-validation-job-tags: ["docker"]
schema-base-url: "https://git.code.tecnalia.com/api/v4/projects/smartdatalab%2Fpublic%2Fci-cd-components%2Fkicker/repository/files"
- component: git.code.tecnalia.com/smartdatalab/public/ci-cd-components/bash/gitlab-ci-bash@master
inputs:
bash-shellcheck-job-tags: ["docker"]
- component: git.code.tecnalia.com/smartdatalab/public/ci-cd-components/semantic-release/gitlab-ci-semrel@master
inputs:
semantic-release-job-tags: ["docker"]
stages:
- build
......
CHANGELOG.md
View file @ af8cca5f
# [6.9.0](https://git.code.tecnalia.com/smartdatalab/public/ci-cd-components/python/compare/6.8.1...6.9.0) (2024-05-15)
### Bug Fixes
* **cache:** always keep cache ([8cf171e](https://git.code.tecnalia.com/smartdatalab/public/ci-cd-components/python/commit/8cf171edde7577b4179530192169730eff122ff2))
* **workflow:** disable MR pipeline from prod & integ branches ([8703dea](https://git.code.tecnalia.com/smartdatalab/public/ci-cd-components/python/commit/8703dea915ca48038588adcc2a0672269dd24bb4))
### Features
* add ⚡ Ruff linter job ([ef364ef](https://git.code.tecnalia.com/smartdatalab/public/ci-cd-components/python/commit/ef364ef65dad8034b7c7af60bb25195cf20ba466))
* add mypy job ([cc22e8c](https://git.code.tecnalia.com/smartdatalab/public/ci-cd-components/python/commit/cc22e8c0eda56d2df00e22b8034da40fd2cfc937))
* **ruff:** generate JSON report when SonarQube is detected ([81e711d](https://git.code.tecnalia.com/smartdatalab/public/ci-cd-components/python/commit/81e711d2cd6e23014c0eaef2ec098fe69b711885))
* some log improvement ([aef5a05](https://git.code.tecnalia.com/smartdatalab/public/ci-cd-components/python/commit/aef5a056d09b22e2c6ce2fa8038632c912e11637))
## [6.11.1](https://gitlab.com/to-be-continuous/python/compare/6.11.0...6.11.1) (2024-05-05)
### Bug Fixes
* **workflow:** disable MR pipeline from prod & integ branches ([8703dea](https://gitlab.com/to-be-continuous/python/commit/8703dea915ca48038588adcc2a0672269dd24bb4))
# [6.11.0](https://gitlab.com/to-be-continuous/python/compare/6.10.0...6.11.0) (2024-04-28)
### Bug Fixes
* **cache:** always keep cache ([8cf171e](https://gitlab.com/to-be-continuous/python/commit/8cf171edde7577b4179530192169730eff122ff2))
### Features
* add mypy job ([cc22e8c](https://gitlab.com/to-be-continuous/python/commit/cc22e8c0eda56d2df00e22b8034da40fd2cfc937))
# [6.10.0](https://gitlab.com/to-be-continuous/python/compare/6.9.0...6.10.0) (2024-04-14)
### Features
* **ruff:** generate JSON report when SonarQube is detected ([81e711d](https://gitlab.com/to-be-continuous/python/commit/81e711d2cd6e23014c0eaef2ec098fe69b711885))
# [6.9.0](https://gitlab.com/to-be-continuous/python/compare/6.8.1...6.9.0) (2024-04-13)
### Features
* add ⚡ Ruff linter job ([ef364ef](https://gitlab.com/to-be-continuous/python/commit/ef364ef65dad8034b7c7af60bb25195cf20ba466))
* some log improvement ([aef5a05](https://gitlab.com/to-be-continuous/python/commit/aef5a056d09b22e2c6ce2fa8038632c912e11637))
## [6.8.1](https://gitlab.com/to-be-continuous/python/compare/6.8.0...6.8.1) (2024-04-03)
......
README.md
View file @ af8cca5f
......@@ -14,7 +14,7 @@ Add the following to your `gitlab-ci.yml`:
```yaml
include:
# 1: include the component
- component: gitlab.com/to-be-continuous/python/gitlab-ci-python@6.8.1
- component: gitlab.com/to-be-continuous/python/gitlab-ci-python@6.11.1
# 2: set/override component inputs
inputs:
image: registry.hub.docker.com/library/python:3.10
......@@ -29,7 +29,7 @@ Add the following to your `gitlab-ci.yml`:
include:
# 1: include the template
- project: 'to-be-continuous/python'
ref: '6.8.1'
ref: '6.11.1'
file: '/templates/gitlab-ci-python.yml'
variables:
......@@ -82,6 +82,10 @@ This job allows building your Python project [distribution packages](https://pac
It is bound to the `build` stage, it is **disabled by default** and can be enabled by setting `$PYTHON_PACKAGE_ENABLED` to `true`.
| Input / Variable | Description | Default value |
| ------------------------ | ---------------------------------- | ----------------- |
| `py-package-job-tags` / `PY_PACKAGE_JOB_TAGS` | Tags to be used for selecting runners for the job | `[]` |
### Lint jobs
#### `py-lint` job
......@@ -93,6 +97,7 @@ It is bound to the `build` stage, and uses the following variables:
| Input / Variable | Description | Default value |
| ------------------------ | ---------------------------------- | ----------------- |
| `pylint-enabled` / `PYLINT_ENABLED` | Set to `true` to enable the `pylint` job | _none_ (disabled) |
| `pylint-args` / `PYLINT_ARGS` | Additional [pylint CLI options](http://pylint.pycqa.org/en/latest/user_guide/run.html#command-line-options) | _none_ |
| `pylint-files` / `PYLINT_FILES` | Files or directories to analyse | _none_ (by default analyses all found python source files) |
......@@ -123,6 +128,7 @@ It is bound to the `build` stage, and uses the following variables:
| Input / Variable | Description | Default value |
| ------------------------ | -------------------------------------------------------------------- | ----------------------- |
| `unittest-enabled` / `UNITTEST_ENABLED` | Set to `true` to enable the `unittest` job | _none_ (disabled) |
| `unittest-args` / `UNITTEST_ARGS` | Additional xmlrunner/unittest CLI options | _none_ |
:information_source: use a `.coveragerc` file at the root of your Python project to control the coverage settings.
......@@ -155,7 +161,9 @@ It is bound to the `build` stage, and uses the following variables:
| Input / Variable | Description | Default value |
| ------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------- |
| `pytest-enabled` / `PYTEST_ENABLED` | Set to `true` to enable the `pytest` job | _none_ (disabled) |
| `pytest-args` / `PYTEST_ARGS` | Additional [pytest](https://docs.pytest.org/en/stable/usage.html) or [pytest-cov](https://github.com/pytest-dev/pytest-cov#usage) CLI options | _none_ |
| `py-pytest-job-tags` / `PY_PYTEST_JOB_TAGS` | Tags to be used for selecting runners for the job | `[]` |
:information_source: use a `.coveragerc` file at the root of your Python project to control the coverage settings.
......@@ -187,6 +195,7 @@ It is bound to the `build` stage, and uses the following variables:
| Input / Variable | Description | Default value |
| ------------------------ | --------------------------------------------------------------------------------------- | ----------------------- |
| `nosetests-enabled` / `NOSETESTS_ENABLED` | Set to `true` to enable the `nose` job | _none_ (disabled) |
| `nosetests-args` / `NOSETESTS_ARGS` | Additional [nose CLI options](https://nose.readthedocs.io/en/latest/usage.html#options) | _none_ |
By default coverage will be run on all the project directories. You can restrict it to your packages by setting the `$NOSE_COVER_PACKAGE` variable.
......@@ -282,6 +291,45 @@ This job **disabled by default** and runs [isort](https://pycqa.github.io/isort/
| ---------------- | ----------------------------------------------------------------------- | ----------------- |
| `isort-enabled` / `PYTHON_ISORT_ENABLED` | Set to `true` to enable isort job | _none_ (disabled) |
### `py-ruff` job
This job **disabled by default** and runs [Ruff](https://docs.astral.sh/ruff/) on the repo. It is bound to the build stage.
| Input / Variable | Description | Default value |
| ---------------- | ----------------------------------------------------------------------- | ----------------- |
| `ruff-enabled` / `RUFF_ENABLED` | Set to `true` to enable ruff job | _none_ (disabled) |
| `ruff-args` / `RUFF_ARGS` | Additional [Ruff Linter CLI options](https://docs.astral.sh/ruff/configuration/#full-command-line-interface) | _none_ |
| `ruff-ext-exclude` / `RUFF_EXT_EXCLUDE` | Define [extend-exclude](https://docs.astral.sh/ruff/settings/#extend-exclude) files | _.venv,.cache_ |
| `py-ruff-job-tags` / `PY_RUFF_JOB_TAGS` | Tags to be used for selecting runners for the job | `[]` |
:warning: Ruff can replace isort, Black, Bandit, Pylint and much more. [More info](https://github.com/astral-sh/ruff/blob/main/docs/faq.md#which-tools-does-ruff-replace).
In addition to logs in the console, this job produces the following reports, kept for one week:
| Report | Format | Usage |
| -------------- | ---------------------------------------------------------------------------- | ----------------- |
| `$PYTHON_PROJECT_DIR/reports/py-ruff.gitlab.json` | [GitLab](https://docs.astral.sh/ruff/settings/#output-format) | [GitLab integration](https://docs.gitlab.com/ee/ci/yaml/artifacts_reports.html#artifactsreportscodequality) |
| `$PYTHON_PROJECT_DIR/reports/py-ruff.native.json` | [JSON](https://docs.astral.sh/ruff/settings/#output-format) | [SonarQube integration](https://docs.sonarqube.org/latest/analysis/external-issues/)<br/>_This report is generated only if SonarQube template is detected_ |
#### `py-mypy` job
This job is **disabled by default** and performs code analysis based on [mypy](https://mypy.readthedocs.io/en/stable/).
It is activated by setting `$MYPY_ENABLED` to `true`.
It is bound to the `build` stage, and uses the following variables:
| Input / Variable | Description | Default value |
| ------------------------ | ---------------------------------- | ----------------- |
| `mypy-enabled` / `MYPY_ENABLED` | Set to `true` to enable the `mypy` job | _none_ (disabled) |
| `mypy-args` / `MYPY_ARGS` | Additional [mypy CLI options](https://mypy.readthedocs.io/en/stable/command_line.html) | _none_ |
| `mypy-files` / `MYPY_FILES` | Files or directories to analyse | _none_ (by default analyses all found python source files) |
In addition to a textual report in the console, this job produces the following reports, kept for one day:
| Report | Format | Usage |
| -------------- | ---------------------------------------------------------------------------- | ----------------- |
| `$PYTHON_PROJECT_DIR/reports/py-mypy.codeclimate.json` | [Code Climate](https://github.com/soul-catcher/mypy-gitlab-code-quality) | [GitLab integration](https://docs.gitlab.com/ee/ci/yaml/artifacts_reports.html#artifactsreportscodequality) |
| `$PYTHON_PROJECT_DIR/reports/py-mypy.console.txt` | [mypy console output](https://mypy.readthedocs.io/) | [SonarQube integration](https://docs.sonarqube.org/latest/analysis/external-issues/) |
### SonarQube analysis
......@@ -306,6 +354,10 @@ sonar.python.coverage.reportPaths=reports/py-coverage.cobertura.xml
sonar.python.pylint.reportPaths=reports/py-lint.parseable.txt
# Bandit: CSV format (if enabled)
sonar.python.bandit.reportPaths=reports/py-bandit.bandit.csv
# Ruff: JSON format (if enabled)
sonar.python.ruff.reportPaths=reports/py-ruff.native.json
# mypy: JSON format (if enabled)
sonar.python.mypy.reportPaths=reports/py-mypy.console.txt
```
More info:
......@@ -342,6 +394,7 @@ The release job is bound to the `publish` stage, appears only on production and
| `repository-url` / `PYTHON_REPOSITORY_URL`| Target PyPI repository to publish packages | _[GitLab project's PyPI packages repository](https://docs.gitlab.com/ee/user/packages/pypi_repository/)_ |
| `PYTHON_REPOSITORY_USERNAME`| Target PyPI repository username credential | `gitlab-ci-token` |
| :lock: `PYTHON_REPOSITORY_PASSWORD`| Target PyPI repository password credential | `$CI_JOB_TOKEN` |
| `py-release-job-tags` / `PY_RELEASE_JOB_TAGS` | Tags to be used for selecting runners for the job | `[]` |
#### Setuptools tip
......@@ -470,9 +523,9 @@ With:
```yaml
include:
# main component
- component: gitlab.com/to-be-continuous/python/gitlab-ci-python@6.8.1
- component: gitlab.com/to-be-continuous/python/gitlab-ci-python@6.11.1
# Vault variant
- component: gitlab.com/to-be-continuous/python/gitlab-ci-python-vault@6.8.1
- component: gitlab.com/to-be-continuous/python/gitlab-ci-python-vault@6.11.1
inputs:
vault-base-url: "https://vault.acme.host/v1"
# audience claim for JWT
......@@ -504,7 +557,7 @@ The variant requires the additional configuration parameters:
| Input / Variable | Description | Default value |
| ----------------- | -------------------------------------- | ----------------- |
| `gcp-oidc-aud` / `GCP_OIDC_AUD` | The `aud` claim for the JWT token | `$CI_SERVER_URL` |
| `gcp-oidc-aud` / `GCP_OIDC_AUD` | The `aud` claim for the JWT token _(only required for [OIDC authentication](https://docs.gitlab.com/ee/ci/cloud_services/google_cloud/))_ | `$CI_SERVER_URL` |
| `gcp-oidc-provider` / `GCP_OIDC_PROVIDER` | Default Workload Identity Provider associated with GitLab to [authenticate with OpenID Connect](https://docs.gitlab.com/ee/ci/cloud_services/google_cloud/) | _none_ |
| `gcp-oidc-account` / `GCP_OIDC_ACCOUNT` | Default Service Account to which impersonate with OpenID Connect authentication | _none_ |
......@@ -512,13 +565,13 @@ The variant requires the additional configuration parameters:
```yaml
include:
- component: gitlab.com/to-be-continuous/python/gitlab-ci-python@6.8.1
- component: gitlab.com/to-be-continuous/python/gitlab-ci-python@6.11.1
# 2: set/override component inputs
inputs:
image: registry.hub.docker.com/library/python:3.10
pytest-enabled: true
- component: gitlab.com/to-be-continuous/python/gitlab-ci-python-gcp@6.8.1
- component: gitlab.com/to-be-continuous/python/gitlab-ci-python-gcp@6.11.1
inputs:
# common OIDC config for non-prod envs
gcp-oidc-provider: "projects/<gcp_nonprod_proj_id>/locations/global/workloadIdentityPools/<pool_id>/providers/<provider_id>"
......
bumpversion.sh
View file @ af8cca5f
......@@ -33,7 +33,7 @@ if [[ "$curVer" ]]; then
# replace in template and variants
for tmpl in templates/*.yml
do
sed -e "s/command: *\[\"--service\", \"\(.*\)\", \"$curVer\"\]/command: [\"--service\", \"\1\", \"$nextVer\"]/" "$tmpl" > "$tmpl.next"
sed -e "s/command: *\[ *\"--service\", *\"\(.*\)\", *\"$curVer\" *\]/command: [\"--service\", \"\1\", \"$nextVer\"]/" "$tmpl" > "$tmpl.next"
mv -f "$tmpl.next" "$tmpl"
done
else
......
kicker.json
View file @ af8cca5f
......@@ -4,6 +4,7 @@
"template_path": "templates/gitlab-ci-python.yml",
"kind": "build",
"prefix": "python",
"job_prefix": "py",
"is_component": true,
"variables": [
{
......@@ -57,11 +58,20 @@
"id":"package",
"name":"package",
"description":"This job allows building your Python project [distribution packages](https://packaging.python.org/en/latest/glossary/#term-Distribution-Package).",
"enable_with": "PYTHON_PACKAGE_ENABLED"
"enable_with": "PYTHON_PACKAGE_ENABLED",
"variables": [
{
"name": "PY_PACKAGE_JOB_TAGS",
"description": "Tags to be used for selecting runners for the job",
"type": "array",
"default": [],
"advanced": true
}
]
},
{
"id": "py-lint",
"name": "py-lint",
"id": "pylint",
"name": "pylint",
"description": "Code analysis based on [pylint](http://pylint.pycqa.org/en/latest/) Python lib.",
"enable_with": "PYLINT_ENABLED",
"variables": [
......@@ -100,12 +110,19 @@
"name": "PYTEST_ARGS",
"description": "Additional [pytest](https://docs.pytest.org/en/stable/usage.html) or [pytest-cov](https://github.com/pytest-dev/pytest-cov#usage) CLI options",
"advanced": true
},
{
"name": "PY_PYTEST_JOB_TAGS",
"description": "Tags to be used for selecting runners for the job",
"type": "array",
"default": [],
"advanced": true
}
]
},
{
"id": "py-nosetests",
"name": "py-nosetests",
"id": "nose",
"name": "nose",
"description": "Unit tests based on [nose](https://nose.readthedocs.io/) framework",
"enable_with": "NOSETESTS_ENABLED",
"variables": [
......@@ -238,6 +255,13 @@
"description": "Target PyPI repository password credential",
"secret": true,
"default": "$CI_JOB_TOKEN"
},
{
"name": "PY_RELEASE_JOB_TAGS",
"description": "Tags to be used for selecting runners for the job",
"type": "array",
"default": [],
"advanced": true
}
]
},
......@@ -252,6 +276,49 @@
"name": "isort",
"description": "Check imports order with [isort](https://pycqa.github.io/isort)",
"enable_with": "PYTHON_ISORT_ENABLED"
},
{
"id": "ruff",
"name": "Ruff",
"description": "An extremely fast Python linter and code formatter, written in Rust. [Ruff](https://docs.astral.sh/ruff/)",
"enable_with": "RUFF_ENABLED",
"variables": [
{
"name": "RUFF_ARGS",
"description": "Additional [Ruff Linter CLI options](https://docs.astral.sh/ruff/configuration/#full-command-line-interface)",
"advanced": true
},
{
"name": "RUFF_EXT_EXCLUDE",
"description": "Define [extend-exclude](https://docs.astral.sh/ruff/settings/#extend-exclude) files",
"advanced": true
},
{
"name": "PY_RUFF_JOB_TAGS",
"description": "Tags to be used for selecting runners for the job",
"type": "array",
"default": [],
"advanced": true
}
]
},
{
"id": "mypy",
"name": "mypy",
"description": "Code analysis based on [mypy](https://mypy.readthedocs.io/).",
"enable_with": "MYPY_ENABLED",
"variables": [
{
"name": "MYPY_ARGS",
"description": "Additional [mypy CLI options](https://mypy.readthedocs.io/en/stable/command_line.html)",
"advanced": true
},
{
"name": "MYPY_FILES",
"description": "Files or directories to analyse",
"advanced": true
}
]
}
],
"variants": [
......@@ -299,7 +366,7 @@
"variables": [
{
"name": "GCP_OIDC_AUD",
"description": "The `aud` claim for the JWT token _(only required for [OIDC authentication](https://docs.gitlab.com/ee/ci/cloud_services/aws/))_",
"description": "The `aud` claim for the JWT token _(only required for [OIDC authentication](https://docs.gitlab.com/ee/ci/cloud_services/google_cloud/))_",
"default": "$CI_SERVER_URL",
"advanced": true
},
......
templates/gitlab-ci-python-gcp.yml
View file @ af8cca5f
......@@ -21,7 +21,8 @@ variables:
.gcp-provider-auth:
before_script:
- echo "Installing GCP authentication with env GOOGLE_APPLICATION_CREDENTIALS file"
- set -e
- echo -e "[\\e[1;94mINFO\\e[0m] Installing GCP authentication with env GOOGLE_APPLICATION_CREDENTIALS file"
- echo $GCP_JWT > "$CI_BUILDS_DIR/.auth_token.jwt"
- |-
cat << EOF > "$CI_BUILDS_DIR/google_application_credentials.json"
......@@ -43,7 +44,7 @@ variables:
image: $PYTHON_IMAGE
services:
- name: "$TBC_TRACKING_IMAGE"
command: ["--service", "python", "6.8.1"]
command: ["--service", "python", "6.11.1"]
variables:
GCP_JWT: $GCP_JWT
before_script:
......
templates/gitlab-ci-python-vault.yml
View file @ af8cca5f
......@@ -22,7 +22,7 @@ variables:
.python-base:
services:
- name: "$TBC_TRACKING_IMAGE"
command: ["--service", "python", "6.8.1"]
command: ["--service", "python", "6.11.1"]
- name: "$TBC_VAULT_IMAGE"
alias: "vault-secrets-provider"
variables:
......
templates/gitlab-ci-python.yml
View file @ af8cca5f
......@@ -83,7 +83,7 @@ spec:
description: Additional [pytest](https://docs.pytest.org/en/stable/usage.html) or [pytest-cov](https://github.com/pytest-dev/pytest-cov#usage) CLI options
default: ''
nosetests-enabled:
description: Enable nosetest
description: Enable nose
type: boolean
default: false
nosetests-args:
......@@ -155,12 +155,52 @@ spec:
description: Enable isort
type: boolean
default: false
ruff-enabled:
description: Enable Ruff
type: boolean
default: false
ruff-args:
description: Additional [Ruff Linter CLI options](https://docs.astral.sh/ruff/configuration/#full-command-line-interface)
default: ""
ruff-ext-exclude:
description: Define [extend-exclude](https://docs.astral.sh/ruff/settings/#extend-exclude) files
default: ""
mypy-enabled:
description: Enable mypy
type: boolean
default: false
mypy-args:
description: Additional [mypy CLI options](https://mypy.readthedocs.io/en/stable/command_line.html)
default: ""
mypy-files:
description: Files or directories to analyse
default: ''
py-package-job-tags:
description: tags to filter applicable runners for py-package job
type: array
default: []
py-pytest-job-tags:
description: tags to filter applicable runners for py-pytest job
type: array
default: []
py-ruff-job-tags:
description: tags to filter applicable runners for py-ruff job
type: array
default: []
py-release-job-tags:
description: tags to filter applicable runners for py-release job
type: array
default: []
---
# default workflow rules: Merge Request pipelines
workflow:
rules:
# prevent branch pipeline when an MR is open (prefer MR pipeline)
- if: '$CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS'
# prevent MR pipeline originating from production or integration branch(es)
- if: '$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME =~ $PROD_REF || $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME =~ $INTEG_REF'
when: never
# on non-prod, non-integration branches: prefer MR pipeline over branch pipeline
- if: '$CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS && $CI_COMMIT_REF_NAME !~ $PROD_REF && $CI_COMMIT_REF_NAME !~ $INTEG_REF'
when: never
- if: '$CI_COMMIT_MESSAGE =~ "/\[(ci skip|skip ci) on ([^],]*,)*tag(,[^],]*)*\]/" && $CI_COMMIT_TAG'
when: never
......@@ -201,14 +241,6 @@ variables:
# variabilized tracking image
TBC_TRACKING_IMAGE: "registry.gitlab.com/to-be-continuous/tools/tracking:master"
# Change pip's cache directory to be inside the project directory since we can
# only cache local items.
PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
# Poetry support: force virtualenv not in project dir & use local cache dir
POETRY_CACHE_DIR: "$CI_PROJECT_DIR/.cache/poetry"
POETRY_VIRTUALENVS_IN_PROJECT: "false"
PIPENV_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pipenv"
# PYTHON_IMAGE: "registry.hub.docker.com/library/python:3"
PYTHON_IMAGE: $[[ inputs.image ]]
# Default Python project root directory
......@@ -270,6 +302,12 @@ variables:
PYTHON_BLACK_ENABLED: $[[ inputs.black-enabled ]]
PYTHON_ISORT_ENABLED: $[[ inputs.isort-enabled ]]
RUFF_ENABLED: $[[ inputs.ruff-enabled ]]
RUFF_ARGS: $[[ inputs.ruff-args ]]
RUFF_EXT_EXCLUDE: $[[ inputs.ruff-ext-exclude ]]
MYPY_ENABLED: $[[ inputs.mypy-enabled ]]
MYPY_ARGS: $[[ inputs.mypy-args ]]
MYPY_FILES: $[[ inputs.mypy-files ]]
.python-scripts: &python-scripts |
......@@ -759,7 +797,7 @@ variables:
poetry build ${TRACE+--verbose}
fi
log_info "--- publish packages (poetry)..."
log_info "--- publish packages (poetry) to $PYTHON_REPOSITORY_URL with user $PYTHON_REPOSITORY_USERNAME..."
poetry config repositories.user_defined "$PYTHON_REPOSITORY_URL"
poetry publish ${TRACE+--verbose} --username "$PYTHON_REPOSITORY_USERNAME" --password "$PYTHON_REPOSITORY_PASSWORD" --repository user_defined
else
......@@ -773,7 +811,7 @@ variables:
python -m build
fi
log_info "--- publish packages (twine)..."
log_info "--- publish packages (twine) to $PYTHON_REPOSITORY_URL with user $PYTHON_REPOSITORY_USERNAME..."
twine upload ${TRACE+--verbose} --username "$PYTHON_REPOSITORY_USERNAME" --password "$PYTHON_REPOSITORY_PASSWORD" --repository-url "$PYTHON_REPOSITORY_URL" dist/*
fi
}
......@@ -805,15 +843,19 @@ stages:
image: $PYTHON_IMAGE
services:
- name: "$TBC_TRACKING_IMAGE"
command: ["--service", "python", "6.8.1"]
# Cache downloaded dependencies and plugins between builds.
# To keep cache across branches add 'key: "$CI_JOB_NAME"'
command: ["--service", "python", "6.11.1"]
variables:
# set local cache dir; most Python tools honour XDG specs
XDG_CACHE_HOME: "$CI_PROJECT_DIR/.cache"
PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
POETRY_CACHE_DIR: "$CI_PROJECT_DIR/.cache/poetry"
PIPENV_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pipenv"
POETRY_VIRTUALENVS_IN_PROJECT: "false"
cache:
key: "$CI_COMMIT_REF_SLUG-python"
when: always
paths:
- ${PIP_CACHE_DIR}
- ${POETRY_CACHE_DIR}
- ${PIPENV_CACHE_DIR}
- .cache
before_script:
- !reference [.python-scripts]
- install_ca_certs "${CUSTOM_CA_CERTS:-$DEFAULT_CA_CERTS}"
......@@ -835,6 +877,7 @@ py-package:
expire_in: 1 day
rules:
- if: '$PYTHON_PACKAGE_ENABLED == "true"'
tags: $[[ inputs.py-package-job-tags ]]
py-lint:
extends: .python-base
......@@ -897,6 +940,66 @@ py-isort:
when: never
- !reference [.test-policy, rules]
py-ruff:
extends: .python-base
stage: build
script:
- mkdir -p -m 777 reports
- |
if [[ ${BANDIT_ENABLED} == "true" || ${PYLINT_ENABLED} == "true" || ${PYTHON_ISORT_ENABLED} == "true" || ${PYTHON_BLACK_ENABLED} == "true" ]]; then
log_warn "Ruff can replace isort, Black, Bandit, Pylint"
fi
# Ruff is self dependent tool (written in Rust), it can be installed without project dependencies (_pip and _run don't look required here)
- pip install ${PIP_OPTS} ruff
# JSON output (for SonarQube)
- |
if [[ "$SONAR_HOST_URL" ]]
then
ruff check . ${RUFF_ARGS} ${RUFF_EXCLUDE:---extend-exclude .venv,.cache} --exit-zero --output-format json --output-file reports/py-ruff.native.json
fi
# then GitLab and grouped/console formats
- ruff check . ${RUFF_ARGS} ${RUFF_EXCLUDE:---extend-exclude .venv,.cache} --output-format gitlab --output-file reports/py-ruff.gitlab.json || ruff check . ${RUFF_ARGS} ${RUFF_EXCLUDE:---extend-exclude .venv,.cache} --output-format grouped
artifacts:
name: "$CI_JOB_NAME artifacts from $CI_PROJECT_NAME on $CI_COMMIT_REF_SLUG"
expire_in: 1 day
when: always
reports:
codequality: $PYTHON_PROJECT_DIR/reports/py-ruff.gitlab.json
paths:
- "$PYTHON_PROJECT_DIR/reports/py-ruff.*"
rules:
# exclude if $RUFF_ENABLED not set
- if: '$RUFF_ENABLED != "true"'
when: never
- !reference [.test-policy, rules]
tags: $[[ inputs.py-ruff-job-tags ]]
py-mypy:
extends: .python-base
stage: build
variables:
MYPY_CACHE_DIR: "$CI_PROJECT_DIR/.cache/mypy"
script:
- mkdir -p -m 777 reports
- install_requirements
- _pip install mypy mypy-to-codeclimate
- _run mypy ${MYPY_ARGS} ${MYPY_FILES:-$(find -type f -name "*.py" -not -path "./.cache/*")} | tee reports/py-mypy.console.txt || true
# mypy-to-codeclimate will fail if any error was found
- _run mypy-to-codeclimate reports/py-mypy.console.txt reports/py-mypy.codeclimate.json
artifacts:
name: "$CI_JOB_NAME artifacts from $CI_PROJECT_NAME on $CI_COMMIT_REF_SLUG"
expire_in: 1 day
when: always
reports:
codequality: $PYTHON_PROJECT_DIR/reports/py-mypy.codeclimate.json
paths:
- "$PYTHON_PROJECT_DIR/reports/py-mypy.*"
rules:
# exclude if $MYPY_ENABLED not set
- if: '$MYPY_ENABLED != "true"'
when: never
- !reference [.test-policy, rules]
###############################################################################################
# test stage #
###############################################################################################
......@@ -960,6 +1063,7 @@ py-pytest:
- if: '$PYTEST_ENABLED != "true"'
when: never
- !reference [.test-policy, rules]
tags: $[[ inputs.py-pytest-job-tags ]]
py-nosetests:
extends: .python-base
......@@ -1147,6 +1251,7 @@ py-release:
- if: '$CI_COMMIT_REF_NAME =~ $PROD_REF || $CI_COMMIT_REF_NAME =~ $INTEG_REF'
when: manual
allow_failure: true
tags: $[[ inputs.py-release-job-tags ]]
# (auto from release tag): publishes the Python package(s) to a PyPi registry
py-publish:
......