Compare revisions

Pytgaen · Pytgaen · Pierre Smeyers · semantic-release-bot · Pierre Smeyers · Pierre Smeyers
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
 include:
-  - project: 'to-be-continuous/tools/gitlab-ci'
+  - component: git.code.tecnalia.com/smartdatalab/public/ci-cd-components/gitlab-ci/extract@master
-    ref: 'master'
+    inputs:
-    file: '/templates/extract.yml'
+      extract-script-job-tags: ["docker"]
-  - project: 'to-be-continuous/tools/gitlab-ci'
+  - component: git.code.tecnalia.com/smartdatalab/public/ci-cd-components/gitlab-ci/validation@master
-    ref: 'master'
+    inputs:
-    file: '/templates/validation.yml'
+      check-links-job-tags: ["docker"]
-  - project: 'to-be-continuous/kicker'
+      tbc-check-job-tags: ["docker"]
-    ref: 'master'
+      tbc-check-image: registry.gitlab.com/gbenguria/tbc-check:main
-    file: '/templates/validation.yml'
+      gitlab-ci-lint-job-tags: ["docker"]
-  - project: 'to-be-continuous/bash'
+  - component: git.code.tecnalia.com/smartdatalab/public/ci-cd-components/kicker/validation@master
-    ref: '3.4'
+    inputs:
-    file: '/templates/gitlab-ci-bash.yml'
+      kicker-validation-job-tags: ["docker"]
-  - project: 'to-be-continuous/semantic-release'
+      schema-base-url: "https://git.code.tecnalia.com/api/v4/projects/smartdatalab%2Fpublic%2Fci-cd-components%2Fkicker/repository/files"
-    ref: '3.8'
+  - component: git.code.tecnalia.com/smartdatalab/public/ci-cd-components/bash/gitlab-ci-bash@master
-    file: '/templates/gitlab-ci-semrel.yml'
+    inputs:
+      bash-shellcheck-job-tags: ["docker"]
+  - component: git.code.tecnalia.com/smartdatalab/public/ci-cd-components/semantic-release/gitlab-ci-semrel@master
+    inputs:
+      semantic-release-job-tags: ["docker"]
 stages:
  - build

--- a/CHANGELOG.md
+++ b/CHANGELOG.md
+# [6.9.0](https://git.code.tecnalia.com/smartdatalab/public/ci-cd-components/python/compare/6.8.1...6.9.0) (2024-05-15)
+### Bug Fixes
+* **cache:** always keep cache ([8cf171e](https://git.code.tecnalia.com/smartdatalab/public/ci-cd-components/python/commit/8cf171edde7577b4179530192169730eff122ff2))
+* **workflow:** disable MR pipeline from prod & integ branches ([8703dea](https://git.code.tecnalia.com/smartdatalab/public/ci-cd-components/python/commit/8703dea915ca48038588adcc2a0672269dd24bb4))
+### Features
+* add ⚡ Ruff linter job ([ef364ef](https://git.code.tecnalia.com/smartdatalab/public/ci-cd-components/python/commit/ef364ef65dad8034b7c7af60bb25195cf20ba466))
+* add mypy job ([cc22e8c](https://git.code.tecnalia.com/smartdatalab/public/ci-cd-components/python/commit/cc22e8c0eda56d2df00e22b8034da40fd2cfc937))
+* **ruff:** generate JSON report when SonarQube is detected ([81e711d](https://git.code.tecnalia.com/smartdatalab/public/ci-cd-components/python/commit/81e711d2cd6e23014c0eaef2ec098fe69b711885))
+* some log improvement ([aef5a05](https://git.code.tecnalia.com/smartdatalab/public/ci-cd-components/python/commit/aef5a056d09b22e2c6ce2fa8038632c912e11637))
+## [6.11.1](https://gitlab.com/to-be-continuous/python/compare/6.11.0...6.11.1) (2024-05-05)
+### Bug Fixes
+* **workflow:** disable MR pipeline from prod & integ branches ([8703dea](https://gitlab.com/to-be-continuous/python/commit/8703dea915ca48038588adcc2a0672269dd24bb4))
+# [6.11.0](https://gitlab.com/to-be-continuous/python/compare/6.10.0...6.11.0) (2024-04-28)
+### Bug Fixes
+* **cache:** always keep cache ([8cf171e](https://gitlab.com/to-be-continuous/python/commit/8cf171edde7577b4179530192169730eff122ff2))
+### Features
+* add mypy job ([cc22e8c](https://gitlab.com/to-be-continuous/python/commit/cc22e8c0eda56d2df00e22b8034da40fd2cfc937))
+# [6.10.0](https://gitlab.com/to-be-continuous/python/compare/6.9.0...6.10.0) (2024-04-14)
+### Features
+* **ruff:** generate JSON report when SonarQube is detected ([81e711d](https://gitlab.com/to-be-continuous/python/commit/81e711d2cd6e23014c0eaef2ec098fe69b711885))
+# [6.9.0](https://gitlab.com/to-be-continuous/python/compare/6.8.1...6.9.0) (2024-04-13)
+### Features
+* add ⚡ Ruff linter job ([ef364ef](https://gitlab.com/to-be-continuous/python/commit/ef364ef65dad8034b7c7af60bb25195cf20ba466))
+* some log improvement ([aef5a05](https://gitlab.com/to-be-continuous/python/commit/aef5a056d09b22e2c6ce2fa8038632c912e11637))
 ## [6.8.1](https://gitlab.com/to-be-continuous/python/compare/6.8.0...6.8.1) (2024-04-03)

--- a/README.md
+++ b/README.md
@@ -14,7 +14,7 @@ Add the following to your `gitlab-ci.yml`:
 ```yaml
 include:
  # 1: include the component
-  - component: gitlab.com/to-be-continuous/python/gitlab-ci-python@6.8.1
+  - component: gitlab.com/to-be-continuous/python/gitlab-ci-python@6.11.1
    # 2: set/override component inputs
    inputs:
      image: registry.hub.docker.com/library/python:3.10
@@ -29,7 +29,7 @@ Add the following to your `gitlab-ci.yml`:
 include:
  # 1: include the template
  - project: 'to-be-continuous/python'
-    ref: '6.8.1'
+    ref: '6.11.1'
    file: '/templates/gitlab-ci-python.yml'
 variables:
@@ -82,6 +82,10 @@ This job allows building your Python project [distribution packages](https://pac
 It is bound to the `build` stage, it is **disabled by default** and can be enabled by setting `$PYTHON_PACKAGE_ENABLED` to `true`.
+| Input / Variable         | Description                        | Default value     |
+| ------------------------ | ---------------------------------- | ----------------- |
+| `py-package-job-tags` / `PY_PACKAGE_JOB_TAGS` | Tags to be used for selecting runners for the job | `[]`            |
 ### Lint jobs
 #### `py-lint` job
@@ -93,6 +97,7 @@ It is bound to the `build` stage, and uses the following variables:
 | Input / Variable         | Description                        | Default value     |
 | ------------------------ | ---------------------------------- | ----------------- |
+| `pylint-enabled` / `PYLINT_ENABLED` | Set to `true` to enable the `pylint` job                  | _none_ (disabled) |
 | `pylint-args` / `PYLINT_ARGS` | Additional [pylint CLI options](http://pylint.pycqa.org/en/latest/user_guide/run.html#command-line-options) |  _none_           |
 | `pylint-files` / `PYLINT_FILES` | Files or directories to analyse   | _none_ (by default analyses all found python source files) |
@@ -123,6 +128,7 @@ It is bound to the `build` stage, and uses the following variables:
 | Input / Variable         | Description                                                          | Default value           |
 | ------------------------ | -------------------------------------------------------------------- | ----------------------- |
+| `unittest-enabled` / `UNITTEST_ENABLED` | Set to `true` to enable the `unittest` job                  | _none_ (disabled) |
 | `unittest-args` / `UNITTEST_ARGS` | Additional xmlrunner/unittest CLI options                            | _none_                  |
 :information_source: use a `.coveragerc` file at the root of your Python project to control the coverage settings.
@@ -155,7 +161,9 @@ It is bound to the `build` stage, and uses the following variables:
 | Input / Variable         | Description                                                                                                                                   | Default value           |
 | ------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------- |
+| `pytest-enabled` / `PYTEST_ENABLED` | Set to `true` to enable the `pytest` job                  | _none_ (disabled) |
 | `pytest-args` / `PYTEST_ARGS` | Additional [pytest](https://docs.pytest.org/en/stable/usage.html) or [pytest-cov](https://github.com/pytest-dev/pytest-cov#usage) CLI options | _none_                  |
+| `py-pytest-job-tags` / `PY_PYTEST_JOB_TAGS` | Tags to be used for selecting runners for the job | `[]`            |
 :information_source: use a `.coveragerc` file at the root of your Python project to control the coverage settings.
@@ -187,6 +195,7 @@ It is bound to the `build` stage, and uses the following variables:
 | Input / Variable         | Description                                                                             | Default value           |
 | ------------------------ | --------------------------------------------------------------------------------------- | ----------------------- |
+| `nosetests-enabled` / `NOSETESTS_ENABLED` | Set to `true` to enable the `nose` job                  | _none_ (disabled) |
 | `nosetests-args` / `NOSETESTS_ARGS` | Additional [nose CLI options](https://nose.readthedocs.io/en/latest/usage.html#options) | _none_                  |
 By default coverage will be run on all the project directories. You can restrict it to your packages by setting the `$NOSE_COVER_PACKAGE` variable.
@@ -282,6 +291,45 @@ This job **disabled by default** and runs [isort](https://pycqa.github.io/isort/
 | ---------------- | ----------------------------------------------------------------------- | ----------------- |
 | `isort-enabled` / `PYTHON_ISORT_ENABLED` | Set to `true` to enable isort job               | _none_ (disabled) |
+### `py-ruff` job
+This job **disabled by default** and runs [Ruff](https://docs.astral.sh/ruff/) on the repo. It is bound to the build stage.
+| Input / Variable | Description                                                             | Default value     |
+| ---------------- | ----------------------------------------------------------------------- | ----------------- |
+| `ruff-enabled` / `RUFF_ENABLED` | Set to `true` to enable ruff job                  | _none_ (disabled) |
+| `ruff-args` / `RUFF_ARGS` | Additional [Ruff Linter CLI options](https://docs.astral.sh/ruff/configuration/#full-command-line-interface)     | _none_           |
+| `ruff-ext-exclude` / `RUFF_EXT_EXCLUDE` | Define [extend-exclude](https://docs.astral.sh/ruff/settings/#extend-exclude) files                             | _.venv,.cache_   |
+| `py-ruff-job-tags` / `PY_RUFF_JOB_TAGS` | Tags to be used for selecting runners for the job | `[]`            |
+:warning: Ruff can replace isort, Black, Bandit, Pylint and much more. [More info](https://github.com/astral-sh/ruff/blob/main/docs/faq.md#which-tools-does-ruff-replace). 
+In addition to logs in the console, this job produces the following reports, kept for one week:
+| Report         | Format                                                                       | Usage             |
+| -------------- | ---------------------------------------------------------------------------- | ----------------- |
+| `$PYTHON_PROJECT_DIR/reports/py-ruff.gitlab.json` | [GitLab](https://docs.astral.sh/ruff/settings/#output-format) | [GitLab integration](https://docs.gitlab.com/ee/ci/yaml/artifacts_reports.html#artifactsreportscodequality) |
+| `$PYTHON_PROJECT_DIR/reports/py-ruff.native.json` | [JSON](https://docs.astral.sh/ruff/settings/#output-format) | [SonarQube integration](https://docs.sonarqube.org/latest/analysis/external-issues/)<br/>_This report is generated only if SonarQube template is detected_ |
+#### `py-mypy` job
+This job is **disabled by default** and performs code analysis based on [mypy](https://mypy.readthedocs.io/en/stable/).
+It is activated by setting `$MYPY_ENABLED` to `true`.
+It is bound to the `build` stage, and uses the following variables:
+| Input / Variable         | Description                        | Default value     |
+| ------------------------ | ---------------------------------- | ----------------- |
+| `mypy-enabled` / `MYPY_ENABLED` | Set to `true` to enable the `mypy` job                  | _none_ (disabled) |
+| `mypy-args` / `MYPY_ARGS` | Additional [mypy CLI options](https://mypy.readthedocs.io/en/stable/command_line.html) | _none_           |
+| `mypy-files` / `MYPY_FILES` | Files or directories to analyse   | _none_ (by default analyses all found python source files) |
+In addition to a textual report in the console, this job produces the following reports, kept for one day:
+| Report         | Format                                                                       | Usage             |
+| -------------- | ---------------------------------------------------------------------------- | ----------------- |
+| `$PYTHON_PROJECT_DIR/reports/py-mypy.codeclimate.json` | [Code Climate](https://github.com/soul-catcher/mypy-gitlab-code-quality) | [GitLab integration](https://docs.gitlab.com/ee/ci/yaml/artifacts_reports.html#artifactsreportscodequality) |
+| `$PYTHON_PROJECT_DIR/reports/py-mypy.console.txt` | [mypy console output](https://mypy.readthedocs.io/) | [SonarQube integration](https://docs.sonarqube.org/latest/analysis/external-issues/) |
 ### SonarQube analysis
@@ -306,6 +354,10 @@ sonar.python.coverage.reportPaths=reports/py-coverage.cobertura.xml
 sonar.python.pylint.reportPaths=reports/py-lint.parseable.txt
 # Bandit: CSV format (if enabled)
 sonar.python.bandit.reportPaths=reports/py-bandit.bandit.csv
+# Ruff: JSON format (if enabled)
+sonar.python.ruff.reportPaths=reports/py-ruff.native.json
+# mypy: JSON format (if enabled)
+sonar.python.mypy.reportPaths=reports/py-mypy.console.txt
 ```
 More info:
@@ -342,6 +394,7 @@ The release job is bound to the `publish` stage, appears only on production and
 | `repository-url` / `PYTHON_REPOSITORY_URL`| Target PyPI repository to publish packages                              | _[GitLab project's PyPI packages repository](https://docs.gitlab.com/ee/user/packages/pypi_repository/)_ |
 | `PYTHON_REPOSITORY_USERNAME`| Target PyPI repository username credential                              | `gitlab-ci-token` |
 | :lock: `PYTHON_REPOSITORY_PASSWORD`| Target PyPI repository password credential                              | `$CI_JOB_TOKEN` |
+| `py-release-job-tags` / `PY_RELEASE_JOB_TAGS` | Tags to be used for selecting runners for the job | `[]`            |
 #### Setuptools tip
@@ -470,9 +523,9 @@ With:
 ```yaml
 include:
  # main component
-  - component: gitlab.com/to-be-continuous/python/gitlab-ci-python@6.8.1
+  - component: gitlab.com/to-be-continuous/python/gitlab-ci-python@6.11.1
  # Vault variant
-  - component: gitlab.com/to-be-continuous/python/gitlab-ci-python-vault@6.8.1
+  - component: gitlab.com/to-be-continuous/python/gitlab-ci-python-vault@6.11.1
    inputs:
      vault-base-url: "https://vault.acme.host/v1"
      # audience claim for JWT
@@ -504,7 +557,7 @@ The variant requires the additional configuration parameters:
 | Input / Variable | Description                            | Default value     |
 | ----------------- | -------------------------------------- | ----------------- |
-| `gcp-oidc-aud` / `GCP_OIDC_AUD` | The `aud` claim for the JWT token      | `$CI_SERVER_URL` |
+| `gcp-oidc-aud` / `GCP_OIDC_AUD` | The `aud` claim for the JWT token _(only required for [OIDC authentication](https://docs.gitlab.com/ee/ci/cloud_services/google_cloud/))_ | `$CI_SERVER_URL` |
 | `gcp-oidc-provider` / `GCP_OIDC_PROVIDER` | Default Workload Identity Provider associated with GitLab to [authenticate with OpenID Connect](https://docs.gitlab.com/ee/ci/cloud_services/google_cloud/) | _none_ |
 | `gcp-oidc-account` / `GCP_OIDC_ACCOUNT` | Default Service Account to which impersonate with OpenID Connect authentication | _none_ |
@@ -512,13 +565,13 @@ The variant requires the additional configuration parameters:
 ```yaml
 include:
-  - component: gitlab.com/to-be-continuous/python/gitlab-ci-python@6.8.1
+  - component: gitlab.com/to-be-continuous/python/gitlab-ci-python@6.11.1
    # 2: set/override component inputs
    inputs:
      image: registry.hub.docker.com/library/python:3.10
      pytest-enabled: true
-  - component: gitlab.com/to-be-continuous/python/gitlab-ci-python-gcp@6.8.1
+  - component: gitlab.com/to-be-continuous/python/gitlab-ci-python-gcp@6.11.1
    inputs:
      # common OIDC config for non-prod envs
      gcp-oidc-provider: "projects/<gcp_nonprod_proj_id>/locations/global/workloadIdentityPools/<pool_id>/providers/<provider_id>"

--- a/bumpversion.sh
+++ b/bumpversion.sh
@@ -33,7 +33,7 @@ if [[ "$curVer" ]]; then
  # replace in template and variants
  for tmpl in templates/*.yml
  do
-    sed -e "s/command: *\[\"--service\", \"\(.*\)\", \"$curVer\"\]/command: [\"--service\", \"\1\", \"$nextVer\"]/" "$tmpl" > "$tmpl.next"
+    sed -e "s/command: *\[ *\"--service\", *\"\(.*\)\", *\"$curVer\" *\]/command: [\"--service\", \"\1\", \"$nextVer\"]/" "$tmpl" > "$tmpl.next"
    mv -f "$tmpl.next" "$tmpl"
  done
 else

--- a/kicker.json
+++ b/kicker.json
@@ -4,6 +4,7 @@
  "template_path": "templates/gitlab-ci-python.yml",
  "kind": "build",
  "prefix": "python",
+  "job_prefix": "py",
  "is_component": true,
  "variables": [
    {
@@ -57,11 +58,20 @@
      "id":"package",
      "name":"package",
      "description":"This job allows building your Python project [distribution packages](https://packaging.python.org/en/latest/glossary/#term-Distribution-Package).",
-      "enable_with": "PYTHON_PACKAGE_ENABLED"
+      "enable_with": "PYTHON_PACKAGE_ENABLED",
+      "variables": [
+        {
+          "name": "PY_PACKAGE_JOB_TAGS",
+          "description": "Tags to be used for selecting runners for the job",
+          "type": "array",
+          "default": [],
+          "advanced": true
+        }
+      ]
    },
    {
-      "id": "py-lint",
+      "id": "pylint",
-      "name": "py-lint",
+      "name": "pylint",
      "description": "Code analysis based on [pylint](http://pylint.pycqa.org/en/latest/) Python lib.",
      "enable_with": "PYLINT_ENABLED",
      "variables": [
@@ -100,12 +110,19 @@
          "name": "PYTEST_ARGS",
          "description": "Additional [pytest](https://docs.pytest.org/en/stable/usage.html) or [pytest-cov](https://github.com/pytest-dev/pytest-cov#usage) CLI options",
          "advanced": true
+        },
+        {
+          "name": "PY_PYTEST_JOB_TAGS",
+          "description": "Tags to be used for selecting runners for the job",
+          "type": "array",
+          "default": [],
+          "advanced": true
        }
      ]
    },
    {
-      "id": "py-nosetests",
+      "id": "nose",
-      "name": "py-nosetests",
+      "name": "nose",
      "description": "Unit tests based on [nose](https://nose.readthedocs.io/) framework",
      "enable_with": "NOSETESTS_ENABLED",
      "variables": [
@@ -238,6 +255,13 @@
          "description": "Target PyPI repository password credential",
          "secret": true,
          "default": "$CI_JOB_TOKEN"
+        },
+        {
+          "name": "PY_RELEASE_JOB_TAGS",
+          "description": "Tags to be used for selecting runners for the job",
+          "type": "array",
+          "default": [],
+          "advanced": true
        }
      ]
    },
@@ -252,6 +276,49 @@
      "name": "isort",
      "description": "Check imports order with [isort](https://pycqa.github.io/isort)",
      "enable_with": "PYTHON_ISORT_ENABLED"
+    },
+    {
+      "id": "ruff",
+      "name": "Ruff",
+      "description": "An extremely fast Python linter and code formatter, written in Rust. [Ruff](https://docs.astral.sh/ruff/)",
+      "enable_with": "RUFF_ENABLED",
+      "variables": [
+        {
+          "name": "RUFF_ARGS",
+          "description": "Additional [Ruff Linter CLI options](https://docs.astral.sh/ruff/configuration/#full-command-line-interface)",
+          "advanced": true
+        },
+        {
+          "name": "RUFF_EXT_EXCLUDE",
+          "description": "Define [extend-exclude](https://docs.astral.sh/ruff/settings/#extend-exclude) files",
+          "advanced": true
+        },
+        {
+          "name": "PY_RUFF_JOB_TAGS",
+          "description": "Tags to be used for selecting runners for the job",
+          "type": "array",
+          "default": [],
+          "advanced": true
+        }
+      ]
+    },
+    {
+      "id": "mypy",
+      "name": "mypy",
+      "description": "Code analysis based on [mypy](https://mypy.readthedocs.io/).",
+      "enable_with": "MYPY_ENABLED",
+      "variables": [
+        {
+          "name": "MYPY_ARGS",
+          "description": "Additional [mypy CLI options](https://mypy.readthedocs.io/en/stable/command_line.html)",
+          "advanced": true
+        },
+        {
+          "name": "MYPY_FILES",
+          "description": "Files or directories to analyse",
+          "advanced": true
+        }
+      ]
    }
  ],
  "variants": [
@@ -299,7 +366,7 @@
      "variables": [
        {
          "name": "GCP_OIDC_AUD",
-          "description": "The `aud` claim for the JWT token _(only required for [OIDC authentication](https://docs.gitlab.com/ee/ci/cloud_services/aws/))_",
+          "description": "The `aud` claim for the JWT token _(only required for [OIDC authentication](https://docs.gitlab.com/ee/ci/cloud_services/google_cloud/))_",
          "default": "$CI_SERVER_URL",
          "advanced": true
        },

--- a/templates/gitlab-ci-python-gcp.yml
+++ b/templates/gitlab-ci-python-gcp.yml
@@ -21,7 +21,8 @@ variables:
 .gcp-provider-auth:
  before_script:
-    - echo "Installing GCP authentication with env GOOGLE_APPLICATION_CREDENTIALS file"
+    - set -e
+    - echo -e "[\\e[1;94mINFO\\e[0m] Installing GCP authentication with env GOOGLE_APPLICATION_CREDENTIALS file"
    - echo $GCP_JWT > "$CI_BUILDS_DIR/.auth_token.jwt"
    - |-
      cat << EOF > "$CI_BUILDS_DIR/google_application_credentials.json"
@@ -43,7 +44,7 @@ variables:
  image: $PYTHON_IMAGE
  services:
    - name: "$TBC_TRACKING_IMAGE"
-      command: ["--service", "python", "6.8.1"]
+      command: ["--service", "python", "6.11.1"]
  variables:
    GCP_JWT: $GCP_JWT
  before_script:

--- a/templates/gitlab-ci-python-vault.yml
+++ b/templates/gitlab-ci-python-vault.yml
@@ -22,7 +22,7 @@ variables:
 .python-base:
  services:
    - name: "$TBC_TRACKING_IMAGE"
-      command: ["--service", "python", "6.8.1"]
+      command: ["--service", "python", "6.11.1"]
    - name: "$TBC_VAULT_IMAGE"
      alias: "vault-secrets-provider"
  variables:

--- a/templates/gitlab-ci-python.yml
+++ b/templates/gitlab-ci-python.yml
@@ -83,7 +83,7 @@ spec:
      description: Additional [pytest](https://docs.pytest.org/en/stable/usage.html) or [pytest-cov](https://github.com/pytest-dev/pytest-cov#usage) CLI options
      default: ''
    nosetests-enabled:
-      description: Enable nosetest
+      description: Enable nose
      type: boolean
      default: false
    nosetests-args:
@@ -155,12 +155,52 @@ spec:
      description: Enable isort
      type: boolean
      default: false
+    ruff-enabled:
+      description: Enable Ruff
+      type: boolean
+      default: false
+    ruff-args:
+      description: Additional [Ruff Linter CLI options](https://docs.astral.sh/ruff/configuration/#full-command-line-interface)
+      default: ""
+    ruff-ext-exclude:
+      description: Define [extend-exclude](https://docs.astral.sh/ruff/settings/#extend-exclude) files
+      default: ""
+    mypy-enabled:
+      description: Enable mypy
+      type: boolean
+      default: false
+    mypy-args:
+      description: Additional [mypy CLI options](https://mypy.readthedocs.io/en/stable/command_line.html)
+      default: ""
+    mypy-files:
+      description: Files or directories to analyse
+      default: ''
+    py-package-job-tags:
+      description: tags to filter applicable runners for py-package job
+      type: array
+      default: []
+    py-pytest-job-tags:
+      description: tags to filter applicable runners for py-pytest job
+      type: array
+      default: []
+    py-ruff-job-tags:
+      description: tags to filter applicable runners for py-ruff job
+      type: array
+      default: []
+    py-release-job-tags:
+      description: tags to filter applicable runners for py-release job
+      type: array
+      default: []
 ---
 # default workflow rules: Merge Request pipelines
 workflow:
  rules:
-    # prevent branch pipeline when an MR is open (prefer MR pipeline)
+    # prevent MR pipeline originating from production or integration branch(es)
-    - if: '$CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS'
+    - if: '$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME =~ $PROD_REF || $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME =~ $INTEG_REF'
+      when: never
+    # on non-prod, non-integration branches: prefer MR pipeline over branch pipeline
+    - if: '$CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS && $CI_COMMIT_REF_NAME !~ $PROD_REF && $CI_COMMIT_REF_NAME !~ $INTEG_REF'
      when: never
    - if: '$CI_COMMIT_MESSAGE =~ "/\[(ci skip|skip ci) on ([^],]*,)*tag(,[^],]*)*\]/" && $CI_COMMIT_TAG'
      when: never
@@ -201,14 +241,6 @@ variables:
  # variabilized tracking image
  TBC_TRACKING_IMAGE: "registry.gitlab.com/to-be-continuous/tools/tracking:master"
-  # Change pip's cache directory to be inside the project directory since we can
-  # only cache local items.
-  PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
-  # Poetry support: force virtualenv not in project dir & use local cache dir
-  POETRY_CACHE_DIR: "$CI_PROJECT_DIR/.cache/poetry"
-  POETRY_VIRTUALENVS_IN_PROJECT: "false"
-  PIPENV_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pipenv"
  # PYTHON_IMAGE: "registry.hub.docker.com/library/python:3"
  PYTHON_IMAGE: $[[ inputs.image ]]
  # Default Python project root directory
@@ -270,6 +302,12 @@ variables:
  PYTHON_BLACK_ENABLED: $[[ inputs.black-enabled ]]
  PYTHON_ISORT_ENABLED: $[[ inputs.isort-enabled ]]
+  RUFF_ENABLED: $[[ inputs.ruff-enabled ]]
+  RUFF_ARGS: $[[ inputs.ruff-args ]]
+  RUFF_EXT_EXCLUDE: $[[ inputs.ruff-ext-exclude ]]
+  MYPY_ENABLED: $[[ inputs.mypy-enabled ]]
+  MYPY_ARGS: $[[ inputs.mypy-args ]]
+  MYPY_FILES: $[[ inputs.mypy-files ]]
 .python-scripts: &python-scripts |
@@ -759,7 +797,7 @@ variables:
        poetry build ${TRACE+--verbose}
      fi
-      log_info "--- publish packages (poetry)..."
+      log_info "--- publish packages (poetry) to $PYTHON_REPOSITORY_URL with user $PYTHON_REPOSITORY_USERNAME..."
      poetry config repositories.user_defined "$PYTHON_REPOSITORY_URL"
      poetry publish ${TRACE+--verbose} --username "$PYTHON_REPOSITORY_USERNAME" --password "$PYTHON_REPOSITORY_PASSWORD" --repository user_defined
    else
@@ -773,7 +811,7 @@ variables:
        python -m build
      fi
-      log_info "--- publish packages (twine)..."
+      log_info "--- publish packages (twine) to $PYTHON_REPOSITORY_URL with user $PYTHON_REPOSITORY_USERNAME..."
      twine upload ${TRACE+--verbose} --username "$PYTHON_REPOSITORY_USERNAME" --password "$PYTHON_REPOSITORY_PASSWORD" --repository-url "$PYTHON_REPOSITORY_URL" dist/*
    fi
  }
@@ -805,15 +843,19 @@ stages:
  image: $PYTHON_IMAGE
  services:
    - name: "$TBC_TRACKING_IMAGE"
-      command: ["--service", "python", "6.8.1"]
+      command: ["--service", "python", "6.11.1"]
-  # Cache downloaded dependencies and plugins between builds.
+  variables:
-  # To keep cache across branches add 'key: "$CI_JOB_NAME"'
+    # set local cache dir; most Python tools honour XDG specs
+    XDG_CACHE_HOME: "$CI_PROJECT_DIR/.cache"
+    PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
+    POETRY_CACHE_DIR: "$CI_PROJECT_DIR/.cache/poetry"
+    PIPENV_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pipenv"
+    POETRY_VIRTUALENVS_IN_PROJECT: "false"
  cache:
    key: "$CI_COMMIT_REF_SLUG-python"
+    when: always
    paths:
-      - ${PIP_CACHE_DIR}
+      - .cache
-      - ${POETRY_CACHE_DIR}
-      - ${PIPENV_CACHE_DIR}
  before_script:
    - !reference [.python-scripts]
    - install_ca_certs "${CUSTOM_CA_CERTS:-$DEFAULT_CA_CERTS}"
@@ -835,6 +877,7 @@ py-package:
    expire_in: 1 day
  rules:
    - if: '$PYTHON_PACKAGE_ENABLED == "true"'
+  tags: $[[ inputs.py-package-job-tags ]]
 py-lint:
  extends: .python-base
@@ -897,6 +940,66 @@ py-isort:
      when: never
    - !reference [.test-policy, rules]
+py-ruff:
+  extends: .python-base
+  stage: build
+  script:
+    - mkdir -p -m 777 reports
+    - |  
+      if [[  ${BANDIT_ENABLED} == "true" || ${PYLINT_ENABLED} == "true" || ${PYTHON_ISORT_ENABLED} == "true" || ${PYTHON_BLACK_ENABLED} == "true" ]]; then
+        log_warn "Ruff can replace isort, Black, Bandit, Pylint"
+      fi
+    # Ruff is self dependent tool (written in Rust), it can be installed without project dependencies (_pip and _run don't look required here)
+    - pip install ${PIP_OPTS} ruff
+    # JSON output (for SonarQube)
+    - |
+      if [[ "$SONAR_HOST_URL" ]]
+      then
+        ruff check . ${RUFF_ARGS} ${RUFF_EXCLUDE:---extend-exclude .venv,.cache} --exit-zero --output-format json --output-file reports/py-ruff.native.json
+      fi
+    # then GitLab and grouped/console formats
+    - ruff check . ${RUFF_ARGS} ${RUFF_EXCLUDE:---extend-exclude .venv,.cache} --output-format gitlab --output-file reports/py-ruff.gitlab.json || ruff check . ${RUFF_ARGS} ${RUFF_EXCLUDE:---extend-exclude .venv,.cache} --output-format grouped 
+  artifacts:
+    name: "$CI_JOB_NAME artifacts from $CI_PROJECT_NAME on $CI_COMMIT_REF_SLUG"
+    expire_in: 1 day
+    when: always
+    reports:
+      codequality: $PYTHON_PROJECT_DIR/reports/py-ruff.gitlab.json
+    paths:
+      - "$PYTHON_PROJECT_DIR/reports/py-ruff.*"
+  rules:
+    # exclude if $RUFF_ENABLED not set
+    - if: '$RUFF_ENABLED != "true"'
+      when: never
+    - !reference [.test-policy, rules]
+  tags: $[[ inputs.py-ruff-job-tags ]]
+py-mypy:
+  extends: .python-base
+  stage: build
+  variables:
+    MYPY_CACHE_DIR: "$CI_PROJECT_DIR/.cache/mypy"
+  script:
+    - mkdir -p -m 777 reports
+    - install_requirements
+    - _pip install mypy mypy-to-codeclimate
+    - _run mypy ${MYPY_ARGS} ${MYPY_FILES:-$(find -type f -name "*.py" -not -path "./.cache/*")} | tee reports/py-mypy.console.txt || true
+    # mypy-to-codeclimate will fail if any error was found
+    - _run mypy-to-codeclimate reports/py-mypy.console.txt reports/py-mypy.codeclimate.json
+  artifacts:
+    name: "$CI_JOB_NAME artifacts from $CI_PROJECT_NAME on $CI_COMMIT_REF_SLUG"
+    expire_in: 1 day
+    when: always
+    reports:
+      codequality: $PYTHON_PROJECT_DIR/reports/py-mypy.codeclimate.json
+    paths:
+      - "$PYTHON_PROJECT_DIR/reports/py-mypy.*"
+  rules:
+    # exclude if $MYPY_ENABLED not set
+    - if: '$MYPY_ENABLED != "true"'
+      when: never
+    - !reference [.test-policy, rules]
 ###############################################################################################
 #                                      test stage                                             #
 ###############################################################################################
@@ -960,6 +1063,7 @@ py-pytest:
    - if: '$PYTEST_ENABLED != "true"'
      when: never
    - !reference [.test-policy, rules]
+  tags: $[[ inputs.py-pytest-job-tags ]]
 py-nosetests:
  extends: .python-base
@@ -1147,6 +1251,7 @@ py-release:
    - if: '$CI_COMMIT_REF_NAME =~ $PROD_REF || $CI_COMMIT_REF_NAME =~ $INTEG_REF'
      when: manual
      allow_failure: true
+  tags: $[[ inputs.py-release-job-tags ]]
 # (auto from release tag): publishes the Python package(s) to a PyPi registry
 py-publish:
No results found