diff --git a/README.md b/README.md index 852e98e2160a25292792e3ad9886c5e7265e84b6..7d06b4ca272ae560705a54f2ecfcc6b1fe337df8 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ This project implements a generic GitLab CI template for [Python](https://www.python.org/). -It provides several features, usable in different modes (by configuration) following those [recommendations](to-be-continuous.gitlab.io/doc/usage/) +It provides several features, usable in different modes (by configuration). ## Usage @@ -21,34 +21,40 @@ The Python template uses some global configuration used throughout all jobs. | Name | description | default value | | -------------------- | ------------------------------------------------------------------------------------- | ------------------ | -| `PYTHON_IMAGE` | The Docker image used to run Python <br/>:warning: **set the version required by your project** | `python:3` | -| `PIP_INDEX_URL` | Python repository url | _none_ | +| `PYTHON_IMAGE` | The Docker image used to run Python <br/>:warning: **set the version required by your project** | `python:3` | | `PYTHON_PROJECT_DIR` | Python project root directory | `.` | -| `REQUIREMENTS_FILE` | Path to requirements file _(relative to `$PYTHON_PROJECT_DIR`)_ | `requirements.txt` | -| `PIP_OPTS` | pip extra [options](https://pip.pypa.io/en/stable/reference/pip/#general-options) | _none_ | +| `PYTHON_BUILD_SYSTEM`| Python build-system to use to install dependencies, build and package the project (see below) | _none_ (auto-detect) | +| `PIP_INDEX_URL` | Python repository url | _none_ | +| `PIP_OPTS` | pip [extra options](https://pip.pypa.io/en/stable/reference/pip/#general-options) | _none_ | +| `PYTHON_EXTRA_DEPS` | Python extra sets of dependencies to install<br/>For [Setuptools](https://setuptools.pypa.io/en/latest/userguide/dependency_management.html?highlight=extras#optional-dependencies) or [Poetry](https://python-poetry.org/docs/pyproject/#extras) only | _none_ | +| `PYTHON_REQS_FILE` | Main requirements file _(relative to `$PYTHON_PROJECT_DIR`)_<br/>For [Requirements Files](https://pip.pypa.io/en/stable/user_guide/#requirements-files) build-system only | `requirements.txt` | +| `PYTHON_EXTRA_REQS_FILES` | Extra dev requirements file(s) to install _(relative to `$PYTHON_PROJECT_DIR`)_ | `requirements-dev.txt` | -The cache policy also declares the `.cache/pip` directory as cached (not to download Python dependencies over and over again). +The cache policy also makes the necessary to manage pip cache (not to download Python dependencies over and over again). -Default configuration follows [this Python project structure](https://docs.python-guide.org/writing/structure/) +## Multi build-system support -### Poetry support +The Python template supports the most popular dependency management & build systems. -The Python template supports [Poetry](https://python-poetry.org/) as packaging and dependency management tool. +By default it tries to auto-detect the build system used by the project (based on the presence of `pyproject.toml` +and/or `setup.py` and/or `requirements.txt`), but the build system might also be set explicitly using the +`$PYTHON_BUILD_SYSTEM` variable: -If a `pyproject.toml` file is detected at the root of your Python project, requirements will automatically be generated from Poetry. -Poetry support can be explicitly disabled by setting `PYTHON_POETRY_DISABLED` to `true`. +| Value | Build System (scope) | +| ---------------- | ---------------------------------------------------------- | +| _none_ (default) or `auto` | The template tries to **auto-detect** the actual build system | +| `setuptools` | [Setuptools](https://setuptools.pypa.io/) (dependencies, build & packaging) | +| `poetry` | [Poetry](https://python-poetry.org/) (dependencies, build, test & packaging) | +| `pipenv` | [Pipenv](https://pipenv.pypa.io/) (dependencies only) | +| `reqfile` | [Requirements Files](https://pip.pypa.io/en/stable/user_guide/#requirements-files) (dependencies only) | -:warning: If no `poetry.lock` file is found, the template will emit a (non-blocking) warning message, to enforce [Poetry recommendation](https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control): +## Jobs -> You should commit the `poetry.lock` file to your project repo so that all people working on the project are locked to the same versions of dependencies. +### `py-package` job -Poetry support uses the following variables: +This job allows building your Python project [distribution packages](https://packaging.python.org/en/latest/glossary/#term-Distribution-Package). -| Name | description | default value | -| ------------------------ | ---------------------------------------------------------- | ----------------- | -| `PYTHON_POETRY_EXTRAS` | Poetry [extra sets of dependencies](https://python-poetry.org/docs/pyproject/#extras) to include, space separated | _none_ | - -## Jobs +It is bound to the `build` stage, it is **disabled by default** and can be enabled by setting `$PYTHON_PACKAGE_ENABLED` to `true`. ### Lint jobs @@ -88,7 +94,6 @@ It is bound to the `build` stage, and uses the following variables: | Name | description | default value | | ------------------------ | -------------------------------------------------------------------- | ----------------------- | -| `TEST_REQUIREMENTS_FILE` | Path to test requirements file _(relative to `$PYTHON_PROJECT_DIR`)_ | `test-requirements.txt` | | `UNITTEST_ARGS` | Additional xmlrunner/unittest CLI options | _none_ | This job produces the following artifacts, kept for one day: @@ -119,7 +124,6 @@ It is bound to the `build` stage, and uses the following variables: | Name | description | default value | | ------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------- | -| `TEST_REQUIREMENTS_FILE` | Path to test requirements file _(relative `$PYTHON_PROJECT_DIR`)_ | `test-requirements.txt` | | `PYTEST_ARGS` | Additional [pytest](https://docs.pytest.org/en/stable/usage.html) or [pytest-cov](https://github.com/pytest-dev/pytest-cov#usage) CLI options | _none_ | This job produces the following artifacts, kept for one day: @@ -150,7 +154,6 @@ It is bound to the `build` stage, and uses the following variables: | Name | description | default value | | ------------------------ | --------------------------------------------------------------------------------------- | ----------------------- | -| `TEST_REQUIREMENTS_FILE` | Path to test requirements file _(relative to `$PYTHON_PROJECT_DIR`)_ | `test-requirements.txt` | | `NOSETESTS_ARGS` | Additional [nose CLI options](https://nose.readthedocs.io/en/latest/usage.html#options) | _none_ | By default coverage will be run on all the directory. You can restrict it to your packages by setting NOSE_COVER_PACKAGE variable. @@ -209,7 +212,7 @@ It is bound to the `test` stage, and uses the following variables: | Name | description | default value | | ---------------- | ---------------------------------------------------------------------- | ----------------- | -| `BANDIT_ENABLED` | Set to `true` to enable Bandit analysis | _none_ (disabled) | +| `BANDIT_ENABLED` | Set to `true` to enable Bandit analysis | _none_ (disabled) | | `BANDIT_ARGS` | Additional [Bandit CLI options](https://github.com/PyCQA/bandit#usage) | `--recursive .` | This job outputs a **textual report** in the console, and in case of failure also exports a JSON report in the `reports/` @@ -223,7 +226,7 @@ It is bound to the `test` stage, and uses the following variables: | Name | description | default value | | ---------------- | ----------------------------------------------------------------------- | ----------------- | -| `SAFETY_ENABLED` | Set to `true` to enable Safety job | _none_ (disabled) | +| `SAFETY_ENABLED` | Set to `true` to enable Safety job | _none_ (disabled) | | `SAFETY_ARGS` | Additional [Safety CLI options](https://github.com/pyupio/safety#usage) | `--full-report` | This job outputs a **textual report** in the console, and in case of failure also exports a JSON report in the `reports/` @@ -237,70 +240,92 @@ It is bound to the `test` stage, and uses the following variables: | Name | description | default value | | ---------------- | ----------------------------------------------------------------------- | ----------------- | -| `PYTHON_TRIVY_ENABLED` | Set to `true` to enable Trivy job | _none_ (disabled) | +| `PYTHON_TRIVY_ENABLED` | Set to `true` to enable Trivy job | _none_ (disabled) | | `PYTHON_TRIVY_ARGS` | Additional [Trivy CLI options](https://aquasecurity.github.io/trivy/v0.21.1/getting-started/cli/fs/) | `--vuln-type library` | This job outputs a **textual report** in the console, and in case of failure also exports a JSON report in the `reports/` directory _(relative to project root dir)_. -### Package jobs +### `py-release` job -#### `py-package` job +This job is **disabled by default** and allows to perform a complete release of your Python code: -This job is **disabled by default** and performs a packaging of your Python code. +1. increase the Python project version, +2. Git commit changes and create a Git tag with the new version number, +3. build the [Python packages](https://packaging.python.org/), +4. publish the built packages to a PyPI compatible repository ([GitLab packages](https://docs.gitlab.com/ee/user/packages/pypi_repository/) by default). -It is bound to the `package-build` stage, applies only on git tags and uses the following variables: +The Python template supports two packaging systems: -| Name | description | default value | -| --------------- | ---------------------------------------------------- | ------------- | -| `PYTHON_FORCE_PACKAGE` | Set to `true` to force the packaging even if not on tag related event | _none_ (disabled) | +* [Poetry](https://python-poetry.org/): uses Poetry-specific [version](https://python-poetry.org/docs/cli/#version), [build](https://python-poetry.org/docs/cli/#build) and [publish](https://python-poetry.org/docs/cli/#publish) commands. +* [Setuptools](https://setuptools.pypa.io/): uses [Bumpversion](https://github.com/peritus/bumpversion) as version management, [build](https://pypa-build.readthedocs.io/) as package builder and [Twine](https://twine.readthedocs.io/) to publish. -### Publish jobs +The release job is bound to the `publish` stage, appears only on production and integration branches and uses the following variables: -#### `py-release` job +| Name | description | default value | +| ----------------------- | ----------------------------------------------------------------------- | ----------------- | +| `PYTHON_RELEASE_ENABLED`| Set to `true` to enable the release job | _none_ (disabled) | +| `PYTHON_RELEASE_NEXT` | The part of the version to increase (one of: `major`, `minor`, `patch`) | `minor` | +| `PYTHON_SEMREL_RELEASE_DISABLED`| Set to `true` to disable [semantic-release integration](#semantic-release-integration) | _none_ (disabled) | +| `GIT_USERNAME` | Git username for Git push operations (see below) | _none_ | +| :lock: `GIT_PASSWORD` | Git password for Git push operations (see below) | _none_ | +| :lock: `GIT_PRIVATE_KEY`| SSH key for Git push operations (see below) | _none_ | +| `PYTHON_REPOSITORY_URL`| Target PyPI repository to publish packages | _[GitLab project's PyPI packages repository](https://docs.gitlab.com/ee/user/packages/pypi_repository/)_ | +| `PYTHON_REPOSITORY_USERNAME`| Target PyPI repository username credential | `gitlab-ci-token` | +| :lock: `PYTHON_REPOSITORY_PASSWORD`| Target PyPI repository password credential | `$CI_JOB_TOKEN` | -This job is **disabled by default** and performs an automatic tagging of your Python code. +#### Setuptools tip -* [Bumpversion](https://github.com/peritus/bumpversion) Python library is used for version management. -* Looks for an existing `.bumpversion.cfg` at the project root. If found, it will be the configuration used by bumpversion. If not, the `$RELEASE_VERSION_PART` variable and `setup.py` will be used instead. -* Creating a Git tag involves an authenticated and authorized Git user. +If you're using a `setup.cfg` declarative file for your project Setuptools configuration, then you will have to write a +`.bumpversion.cfg` file to workaround a bug that prevents Bumpversion from updating the project version in your `setup.cfg` file. -**Don't use your personal password !!! -Use an [access token](https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html) with write_repository rights. -If you have a generic account, add it to the project and generate access token from this account.** +Example of `.bumpversion.cfg` file: -It is bound to the `publish` stage, applies only on master branch and uses the following variables: +```ini +[bumpversion] +# same version as in your setup.cfg +current_version = 0.5.0 -| Name | description | default value | -| ---------------------- | ----------------------------------------------------------------------- | ----------------- | -| `RELEASE_VERSION_PART` | The part of the version to increase (one of: `major`, `minor`, `patch`) | `minor` | -| `RELEASE_USERNAME` | Username credential for git push | _none_ (disabled) | -| `RELEASE_ACCESS_TOKEN` | Password credential for git push | _none_ | +[bumpversion:file:setup.cfg] +# any additional config here +# see: https://github.com/peritus/bumpversion#file-specific-configuration +``` -#### `py-publish` job +#### `semantic-release` integration -This job is **disabled by default** and performs a publication of your Python code. +If you activate the [`semantic-release-info` job from the `semantic-release` template](https://gitlab.com/to-be-continuous/semantic-release/#semantic-release-info-job), the `py-release` job will rely on the generated next version info. +Thus, a release will be performed only if a next semantic release is present. -It is bound to the `publish` stage, applies only on git tags and uses the following variables: +You should disable the `semantic-release` job (as it's the `py-release` job that will perform the release and so we only need the `semantic-release-info` job) by setting `SEMREL_RELEASE_DISABLED` to `true`. -| Name | description | default value | -| ---------------------- | -------------------------------------------------------- | ----------------- | -| `PYTHON_PUBLISH_ENABLED`| Set to `true` to enable the publish job | _none_ (disabled) | -| `TWINE_REPOSITORY_URL` | Where to publish your Python project | GitLab Project's Pypi Packages registry | -| `TWINE_USERNAME` | Username credential to publish to \$TWINE_REPOSITORY_URL | `gitlab-ci-token` | -| `TWINE_PASSWORD` | Password credential to publish to \$TWINE_REPOSITORY_URL | `$CI_JOB_TOKEN` | +Finally, the semantic-release integration can be disabled with the `PYTHON_SEMREL_RELEASE_DISABLED` variable. -More info: +#### -* [Python Packaging User Guide](https://packaging.python.org/) -* [PyPI packages in the Package Registry](https://docs.gitlab.com/ee/user/packages/pypi_repository/) +#### Git authentication -If you want to automatically create tag and publish your Python package, please have a look [here](#release-python) +A Python release involves some Git push operations. + +You can either use a SSH key or user/password credentials. + +##### Using a SSH key + +We recommend you to use a [project deploy key](https://docs.gitlab.com/ee/user/project/deploy_keys/#project-deploy-keys) with write access to your project. + +The key should not have a passphrase (see [how to generate a new SSH key pair](https://docs.gitlab.com/ce/ssh/README.html#generating-a-new-ssh-key-pair)). + +Specify :lock: `$GIT_PRIVATE_KEY` as secret project variable with the private part of the deploy key. + +```PEM +-----BEGIN OPENSSH PRIVATE KEY----- +blablabla +-----END OPENSSH PRIVATE KEY----- +``` -#### `py-docs` job +The template handles both classic variable and file variable. -This job is no longer supported in this version of the template. It might come back later on with a more generic & configurable implementation. +##### Using user/password credentials -## GitLab compatibility +Simply specify :lock: `$GIT_USERNAME` and :lock: `$GIT_PASSWORD` as secret project variables. -:information_source: This template is actually tested and validated on GitLab Community Edition instance version 13.12.11 +Note that the password should be an access token (preferably a [project](https://docs.gitlab.com/ee/user/project/settings/project_access_tokens.html) or [group](https://docs.gitlab.com/ee/user/group/settings/group_access_tokens.html) access token) with `read_repository` and `write_repository` scopes. diff --git a/kicker.json b/kicker.json index 5983e8b79dd449a66f5343f21c3eb4dc6322b180..da0fc1ee4a5257f65937e8a5868b6cadae33d5fe 100644 --- a/kicker.json +++ b/kicker.json @@ -15,11 +15,25 @@ "default": "." }, { - "name": "REQUIREMENTS_FILE", - "description": "Full path to `requirements.txt` file _(relative to `$PYTHON_PROJECT_DIR`)_", + "name": "PYTHON_BUILD_SYSTEM", + "description": "Python build-system to use to install dependencies, build and package the project", + "type": "enum", + "values": ["auto", "setuptools", "poetry", "pipenv", "reqfile"], + "default": "auto", + "advanced": true + }, + { + "name": "PYTHON_REQS_FILE", + "description": "Main requirements file _(relative to `$PYTHON_PROJECT_DIR`)_\n\nFor [Requirements Files](https://pip.pypa.io/en/stable/user_guide/#requirements-files) build-system only", "default": "requirements.txt", "advanced": true }, + { + "name": "PYTHON_EXTRA_REQS_FILES", + "description": "Extra dev requirements file(s) to install _(relative to `$PYTHON_PROJECT_DIR`)_\n\nFor [Requirements Files](https://pip.pypa.io/en/stable/user_guide/#requirements-files) build-system only", + "default": "requirements-dev.txt", + "advanced": true + }, { "name": "PYTHON_COMPILE_ARGS", "description": "[`compileall` CLI options](https://docs.python.org/3/library/compileall.html)", @@ -32,15 +46,8 @@ "advanced": true }, { - "name": "PYTHON_POETRY_DISABLED", - "description": "Disable poetry support", - "type": "boolean", - "advanced": true - }, - { - "name": "PYTHON_POETRY_EXTRAS", - "description": "Poetry [extra sets of dependencies](https://python-poetry.org/docs/pyproject/#extras) to include, space separated", - "advanced": true + "name": "PYTHON_EXTRA_DEPS", + "description": "Extra sets of dependencies to install\n\nFor [Setuptools](https://setuptools.pypa.io/en/latest/userguide/dependency_management.html?highlight=extras#optional-dependencies) or [Poetry](https://python-poetry.org/docs/pyproject/#extras) only" } ], "features": [ @@ -68,12 +75,6 @@ "description": "Unit tests based on [unittest](https://docs.python.org/3/library/unittest.html) framework", "enable_with": "UNITTEST_ENABLED", "variables": [ - { - "name": "TEST_REQUIREMENTS_FILE", - "description": "Path to test requirements file _(relative to `$PYTHON_PROJECT_DIR`)_", - "default": "test-requirements.txt", - "advanced": true - }, { "name": "UNITTEST_ARGS", "description": "Additional xmlrunner/unittest CLI options", @@ -87,12 +88,6 @@ "description": "Unit tests based on [pytest](https://docs.pytest.org/) framework", "enable_with": "PYTEST_ENABLED", "variables": [ - { - "name": "TEST_REQUIREMENTS_FILE", - "description": "Path to test requirements file _(relative to `$PYTHON_PROJECT_DIR`)_", - "default": "test-requirements.txt", - "advanced": true - }, { "name": "PYTEST_ARGS", "description": "Additional [pytest](https://docs.pytest.org/en/stable/usage.html) or [pytest-cov](https://github.com/pytest-dev/pytest-cov#usage) CLI options", @@ -106,12 +101,6 @@ "description": "Unit tests based on [nose](https://nose.readthedocs.io/) framework", "enable_with": "NOSETESTS_ENABLED", "variables": [ - { - "name": "TEST_REQUIREMENTS_FILE", - "description": "Path to test requirements file _(relative to `$PYTHON_PROJECT_DIR`)_", - "default": "test-requirements.txt", - "advanced": true - }, { "name": "NOSETESTS_ARGS", "description": "Additional [nose CLI options](https://nose.readthedocs.io/en/latest/usage.html#options)", @@ -161,51 +150,14 @@ } ] }, - { - "id": "package", - "name": "package", - "description": "Packaging of your Python code", - "variables": [ - { - "name": "PYTHON_FORCE_PACKAGE", - "description": "Force the packaging even if not on tag related event", - "type": "boolean" - } - ] - }, - { - "id": "publish", - "name": "Publish", - "description": "Publish your code to a [Twine](https://pypi.org/project/twine/) repository", - "enable_with": "PYTHON_PUBLISH_ENABLED", - "variables": [ - { - "name": "TWINE_REPOSITORY_URL", - "type": "url", - "description": "Twine repository url to publish you python project", - "default": "${CI_SERVER_URL}/api/v4/projects/${CI_PROJECT_ID}/packages/pypi" - }, - { - "name": "TWINE_USERNAME", - "description": "Twine repository username credential", - "secret": true, - "default": "gitlab-ci-token" - }, - { - "name": "TWINE_PASSWORD", - "description": "Twine repository password credential", - "secret": true, - "default": "$CI_JOB_TOKEN" - } - ] - }, { "id": "release", "name": "Release", "description": "Manually trigger a release of your code (uses [bumpversion](https://pypi.org/project/bumpversion/))", + "enable_with": "PYTHON_RELEASE_ENABLED", "variables": [ { - "name": "RELEASE_VERSION_PART", + "name": "PYTHON_RELEASE_NEXT", "type": "enum", "values": [ "", @@ -218,16 +170,43 @@ "advanced": true }, { - "name": "RELEASE_USERNAME", - "description": "Username credential for Git push", + "name": "PYTHON_SEMREL_RELEASE_DISABLED", + "description": "Disable semantic-release integration", + "type": "boolean", + "advanced": true + }, + { + "name": "GIT_USERNAME", + "description": "Git username for Git push operations", + "secret": true + }, + { + "name": "GIT_PASSWORD", + "description": "Git password for Git push operations", + "secret": true + }, + { + "name": "GIT_PRIVATE_KEY", + "description": "SSH key for Git push operations", + "secret": true + }, + { + "name": "PYTHON_REPOSITORY_URL", + "type": "url", + "description": "Target PyPI repository to publish packages.\n\n_defaults to [GitLab project's packages repository](https://docs.gitlab.com/ee/user/packages/pypi_repository/)_", + "default": "${CI_SERVER_URL}/api/v4/projects/${CI_PROJECT_ID}/packages/pypi" + }, + { + "name": "PYTHON_REPOSITORY_USERNAME", + "description": "Target PyPI repository username credential", "secret": true, - "mandatory": true + "default": "gitlab-ci-token" }, { - "name": "RELEASE_ACCESS_TOKEN", - "description": "Password credential for Git push", + "name": "PYTHON_REPOSITORY_PASSWORD", + "description": "Target PyPI repository password credential", "secret": true, - "mandatory": true + "default": "$CI_JOB_TOKEN" } ] } diff --git a/templates/gitlab-ci-python.yml b/templates/gitlab-ci-python.yml index 9b9bda375ab50478b010c3b8700a1de22e81beb1..a8188ebe1b0e05b09b9bd40f88994b95cb9d865d 100644 --- a/templates/gitlab-ci-python.yml +++ b/templates/gitlab-ci-python.yml @@ -17,12 +17,18 @@ variables: # Change pip's cache directory to be inside the project directory since we can # only cache local items. PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip" + # Poetry support: force virtualenv not in project dir & use local cache dir + POETRY_CACHE_DIR: "$CI_PROJECT_DIR/.cache/poetry" + POETRY_VIRTUALENVS_IN_PROJECT: "false" + PIPENV_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pipenv" + PYTHON_IMAGE: python:3 # Default Python project root directory PYTHON_PROJECT_DIR: . - REQUIREMENTS_FILE: requirements.txt - TEST_REQUIREMENTS_FILE: test-requirements.txt - SETUP_PY_DIR: "." + + PYTHON_REQS_FILE: requirements.txt + PYTHON_EXTRA_REQS_FILES: "requirements-dev.txt" + # default production ref name (pattern) PROD_REF: '/^(master|main)$/' # default integration ref name (pattern) @@ -40,20 +46,13 @@ variables: PYTHON_TRIVY_IMAGE: aquasec/trivy:latest PYTHON_TRIVY_ARGS: "--vuln-type library" - - # Docs - DOCS_REQUIREMENTS_FILE: docs-requirements.txt - DOCS_DIRECTORY: docs - DOCS_BUILD_DIR: public - DOCS_MAKE_ARGS: html BUILDDIR=${DOCS_BUILD_DIR} - - RELEASE_VERSION_PART: "minor" + PYTHON_RELEASE_NEXT: "minor" # By default, publish on the Packages registry of the project # https://docs.gitlab.com/ee/user/packages/pypi_repository/#authenticate-with-a-ci-job-token - TWINE_REPOSITORY_URL: ${CI_SERVER_URL}/api/v4/projects/${CI_PROJECT_ID}/packages/pypi - TWINE_USERNAME: 'gitlab-ci-token' - TWINE_PASSWORD: $CI_JOB_TOKEN + PYTHON_REPOSITORY_URL: ${CI_SERVER_URL}/api/v4/projects/${CI_PROJECT_ID}/packages/pypi + PYTHON_REPOSITORY_USERNAME: 'gitlab-ci-token' + PYTHON_REPOSITORY_PASSWORD: $CI_JOB_TOKEN .python-scripts: &python-scripts | @@ -215,46 +214,130 @@ variables: log_info "... done" } + function guess_build_system() { + case "${PYTHON_BUILD_SYSTEM:-auto}" in + auto) + ;; + poetry) + log_info "--- Build system explictly declared: Poetry" + return + ;; + setuptools) + log_info "--- Build system explictly declared: Setuptools" + return + ;; + pipenv) + log_info "--- Build system explictly declared: Pipenv" + return + ;; + reqfile) + log_info "--- Build system explictly declared: requirements file" + return + ;; + *) + log_warn "--- Unknown declared build system: \\e[33;1m${PYTHON_BUILD_SYSTEM}\\e[0m: please read template doc" + ;; + esac + + if [[ -f "${PYTHON_REQS_FILE}" ]] + then + log_info "--- Build system auto-detected: requirements file" + export PYTHON_BUILD_SYSTEM="reqfile" + return + fi + + if [[ -f "pyproject.toml" ]] + then + # that might be PEP 517 if a build-backend is specified + # otherwise it might be only used as configuration file for development tools... + build_backend=$(sed -rn 's/^build-backend *= *"([^"]*)".*/\1/p' pyproject.toml) + if [[ "$build_backend" ]] + then + case "$build_backend" in + poetry.core.masonry.api) + log_info "--- Build system auto-detected: PEP 517 with Poetry backend" + export PYTHON_BUILD_SYSTEM="poetry" + return + ;; + setuptools.build_meta) + log_info "--- Build system auto-detected: PEP 517 with Setuptools backend" + export PYTHON_BUILD_SYSTEM="setuptools" + return + ;; + *) + log_error "--- Build system auto-detected: PEP 517 with unsupported backend \\e[33;1m${build_backend}\\e[0m: please read template doc" + exit 1 + ;; + esac + fi + fi + + if [[ -f "setup.py" ]] + then + log_info "--- Build system auto-detected: Setuptools (legacy)" + export PYTHON_BUILD_SYSTEM="setuptools" + elif [[ -f "Pipfile" ]] + then + log_info "--- Build system auto-detected: Pipenv" + export PYTHON_BUILD_SYSTEM="pipenv" + else + log_error "--- Build system auto-detect failed: please read template doc" + exit 1 + fi + } + # install requirements - # arg1: 'build' (build only) or 'test' (build + test) function install_requirements() { - target=$1 - if [[ -f "pyproject.toml" ]] && [[ "${PYTHON_POETRY_DISABLED}" != "true" ]]; then + case "$PYTHON_BUILD_SYSTEM" in + poetry) if [[ ! -f "poetry.lock" ]]; then - log_warn "Poetry detected but \\e[33;1mpoetry.lock\\e[0m file not found: you shall commit it with your project files" + log_warn "Using Poetry but \\e[33;1mpoetry.lock\\e[0m file not found: you shall commit it with your project files" fi - pip install poetry - if [[ "$target" == "build" ]]; then - log_info "--- Poetry detected: install build only requirements" - poetry install --no-dev ${PYTHON_POETRY_EXTRAS:+--extras "$PYTHON_POETRY_EXTRAS"} + # shellcheck disable=SC2086 + pip install ${PIP_OPTS} poetry + poetry install ${PYTHON_EXTRA_DEPS:+--extras "$PYTHON_EXTRA_DEPS"} + ;; + setuptools) + # shellcheck disable=SC2086 + pip install ${PIP_OPTS} setuptools + # shellcheck disable=SC2086 + pip install ${PIP_OPTS} ".${PYTHON_EXTRA_DEPS:+[$PYTHON_EXTRA_DEPS]}" + ;; + pipenv) + # shellcheck disable=SC2086 + pip install ${PIP_OPTS} pipenv + if [[ ! -f "Pipfile.lock" ]]; then + log_warn "Using Pipenv but \\e[33;1mPipfile.lock\\e[0m file not found: you shall commit it with your project files" + pipenv install --dev --system else - log_info "--- Poetry detected: install build and dev requirements" - poetry install ${PYTHON_POETRY_EXTRAS:+--extras "$PYTHON_POETRY_EXTRAS"} + pipenv sync --dev --system fi - elif [[ -f "${REQUIREMENTS_FILE}" ]]; then - log_info "--- installing build requirements from \\e[33;1m${REQUIREMENTS_FILE}\\e[0m" - # shellcheck disable=SC2086 - pip install ${PIP_OPTS} -r "${REQUIREMENTS_FILE}" - if [[ "$target" == "test" ]] && [[ -f "${TEST_REQUIREMENTS_FILE}" ]]; then - log_info "--- installing test requirements from \\e[33;1m${TEST_REQUIREMENTS_FILE}\\e[0m" + ;; + reqfile) + if [[ -f "${PYTHON_REQS_FILE}" ]]; then + log_info "--- installing main requirements from \\e[33;1m${PYTHON_REQS_FILE}\\e[0m" + # shellcheck disable=SC2086 + pip install ${PIP_OPTS} -r "${PYTHON_REQS_FILE}" # shellcheck disable=SC2086 - pip install ${PIP_OPTS} -r "${TEST_REQUIREMENTS_FILE}" + found_reqs_files=$(eval ls -1 $PYTHON_EXTRA_REQS_FILES 2>/dev/null || echo "") + # shellcheck disable=SC2116 + for extrareqsfile in $(echo "$found_reqs_files"); do + log_info "--- installing extra requirements from \\e[33;1m${extrareqsfile}\\e[0m" + # shellcheck disable=SC2086 + pip install ${PIP_OPTS} -r "${extrareqsfile}" + done + else + log_warn "--- requirements build system defined, but no ${PYTHON_REQS_FILE} file found" fi - elif [[ -f "${SETUP_PY_DIR}/setup.py" ]]; then - log_info "--- installing requirements from \\e[33;1m${SETUP_PY_DIR}/setup.py\\e[0m" - # shellcheck disable=SC2086 - pip install ${PIP_OPTS} "${SETUP_PY_DIR}/" - else - log_info "--- no dependency management tool, nor requirements file nor setup.py file found: skip install dependencies" - fi + ;; + esac } function _run() { - if [[ -f "pyproject.toml" ]] && [[ "${PYTHON_POETRY_DISABLED}" != "true" ]]; then - if ! command -v poetry > /dev/null - then - pip install poetry - fi + if [[ "${PYTHON_BUILD_SYSTEM}" == "poetry" ]] + then + # shellcheck disable=SC2086 + if ! command -v poetry > /dev/null; then pip install ${PIP_OPTS} poetry; fi poetry run "$@" else "$@" @@ -266,61 +349,178 @@ variables: } function _pip() { - _run pip "$@" + # shellcheck disable=SC2086 + _run pip ${PIP_OPTS} "$@" } - function _package(){ - if [[ -f "pyproject.toml" ]] && [[ "${PYTHON_POETRY_DISABLED}" != "true" ]]; then - pip install poetry + function _package() { + case "$PYTHON_BUILD_SYSTEM" in + poetry) + # shellcheck disable=SC2086 + if ! command -v poetry > /dev/null; then pip install ${PIP_OPTS} poetry; fi poetry build - else - pip install setuptools - python setup.py sdist bdist_wheel - fi + ;; + *) + # shellcheck disable=SC2086 + pip install ${PIP_OPTS} build + python -m build + ;; + esac } - function _publish() { - if [[ -f "pyproject.toml" ]] && [[ "${PYTHON_POETRY_DISABLED}" != "true" ]]; then - pip install poetry - poetry config repositories.user_defined "$TWINE_REPOSITORY_URL" - poetry publish --username "$TWINE_USERNAME" --password "$TWINE_PASSWORD" --repository user_defined - else - pip install twine - pip list - twine upload --verbose dist/*.tar.gz - twine upload --verbose dist/*.whl + function configure_scm_auth() { + git_base_url=$(echo "$CI_REPOSITORY_URL" | cut -d\@ -f2) + if [[ -n "${GIT_USERNAME}" ]] && [[ -n "${GIT_PASSWORD}" ]]; then + log_info "--- using https protocol with SCM credentials from env (\$GIT_USERNAME and \$GIT_PASSWORD)..." + export git_auth_url="https://${GIT_USERNAME}:${GIT_PASSWORD}@${git_base_url}" + elif [[ -n "${GIT_PRIVATE_KEY}" ]]; then + log_info "--- using ssh protocol with SSH key from env (\$GIT_PRIVATE_KEY)..." + mkdir -m 700 "${HOME}/.ssh" + ssh-keyscan -H "${CI_SERVER_HOST}" >> ~/.ssh/known_hosts + eval "$(ssh-agent -s)" + # Handle file variable + if [[ -f "${GIT_PRIVATE_KEY}" ]]; then + tr -d '\r' < "${GIT_PRIVATE_KEY}" | ssh-add - + else + echo "${GIT_PRIVATE_KEY}" | tr -d '\r' | ssh-add - + fi + export git_auth_url="git@${git_base_url/\//:}" + else + log_error "--- Please specify either \$GIT_USERNAME and \$GIT_PASSWORD or \$GIT_PRIVATE_KEY variables to enable release (see doc)." + exit 1 fi } function _release() { - if [[ -f "pyproject.toml" ]] && [[ "${PYTHON_POETRY_DISABLED}" != "true" ]]; then - pip install poetry - poetry version "${RELEASE_VERSION_PART}" - else - pip install bumpversion - release_args - bumpversion "${bumpversion_args}" + # 0: guess packaging system + if [[ -f "pyproject.toml" ]] + then + # that might be PEP 517 if a build-backend is specified + # otherwise it might be only used as configuration file for development tools... + build_backend=$(sed -rn 's/^build-backend *= *"([^"]*)".*/\1/p' pyproject.toml) + if [[ "$build_backend" ]] + then + case "$build_backend" in + poetry.core.masonry.api) + log_info "--- Packaging system auto-detected: Poetry" + pkg_system="poetry" + ;; + setuptools.build_meta) + log_info "--- Packaging system auto-detected: Setuptools (PEP 517)" + pkg_system="setuptools" + ;; + *) + log_error "--- Unsupported PEP 517 backend \\e[33;1m${build_backend}\\e[0m: abort" + exit 1 + ;; + esac + fi fi - } - function release_args() { - if [[ -f ".bumpversion.cfg" ]]; then - log_info "--- .bumpversion.cfg file found " - export bumpversion_args="${RELEASE_VERSION_PART} --verbose" + + if [[ -z "$pkg_system" ]] + then + if [[ -f "setup.py" ]] + then + log_info "--- Packaging system auto-detected: Setuptools (legacy)" + pkg_system="setuptools" + else + log_error "--- Couldn't find any supported packaging system: abort" + exit 1 + fi + fi + + # 1: retrieve next release info from semantic-release + if [ "$SEMREL_INFO_ON" ] && [ "$PYTHON_SEMREL_RELEASE_DISABLED" != "true" ] + then + if [ -z "$SEMREL_INFO_NEXT_VERSION" ] + then + log_info "[semantic-release] no new version to release: skip" + exit 0 + else + py_cur_version="$SEMREL_INFO_LAST_VERSION" + py_next_version="$SEMREL_INFO_NEXT_VERSION" + py_release_part="$SEMREL_INFO_NEXT_VERSION_TYPE" + log_info "[semantic-release] new ($py_release_part) release required \\e[1;94m${py_cur_version}\\e[0m → \\e[1;94m${py_next_version}\\e[0m" + fi + fi + + # 2: bumpversion (+ Git commit & tag) + if [[ "$pkg_system" == "poetry" ]] + then + # shellcheck disable=SC2086 + if ! command -v poetry > /dev/null; then pip install ${PIP_OPTS} poetry; fi + if [[ -z "$py_next_version" ]] + then + py_cur_version=$(poetry version --short) + py_next_version="$PYTHON_RELEASE_NEXT" + fi + log_info "[Poetry] change version \\e[1;94m${py_cur_version}\\e[0m → \\e[1;94m${py_next_version}\\e[0m" + poetry version ${TRACE+--verbose} "$py_next_version" + # eval exact next version + py_next_version=$(poetry version --short) + git add pyproject.toml + git commit -m "chore(python-release): $py_cur_version → $py_next_version [ci skip]" + git tag "$py_next_version" else - log_info "--- No .bumpversion.cfg file found " - if [[ -f "setup.py" ]]; then - log_info "--- Getting current version of setup.py file " - current_version=$(python setup.py --version) - export bumpversion_args=" --verbose --current-version ${current_version} --tag --tag-name {new_version} --commit ${RELEASE_VERSION_PART} setup.py" + # Setuptools / bumpversion + # shellcheck disable=SC2086 + pip install ${PIP_OPTS} bumpversion + py_commit_message="chore(python-release): {current_version} → {new_version} [ci skip]" + if [[ "$py_next_version" ]] + then + # explicit release version (semantic-release) + log_info "[Setuptools] bumpversion \\e[1;94m${py_cur_version}\\e[0m → \\e[1;94m${py_next_version}\\e[0m" + # create cfg in case it doesn't exist - will be updated by bumpversion + touch .bumpversion.cfg + bumpversion ${TRACE+--verbose} --current-version "$py_cur_version" --commit --message "$py_commit_message" --tag --tag-name "{new_version}" "$py_release_part" + elif [[ -f "setup.py" ]] + then + # retrieve current version from setup.py + py_cur_version=$(python setup.py --version) + py_release_part="$PYTHON_RELEASE_NEXT" + log_info "[Setuptools] bumpversion ($py_release_part) from \\e[1;94m${py_cur_version}\\e[0m" + bumpversion ${TRACE+--verbose} --current-version "$py_cur_version" --commit --message "$py_commit_message" --tag --tag-name "{new_version}" "$py_release_part" + elif [[ -f ".bumpversion.cfg" ]] + then + # current version shall be set in .bumpversion.cfg + py_release_part="$PYTHON_RELEASE_NEXT" + log_info "[bumpversion] increase \\e[1;94m${py_release_part}\\e[0m" + bumpversion ${TRACE+--verbose} --commit --message "$py_commit_message" --tag --tag-name "{new_version}" "$py_release_part" else - log_warn "--- No setup.py file found. Cannot perform release." + log_error "--- setup.py or .bumpversion.cfg file required to retrieve current version: cannot perform release" + exit 1 fi fi - log_info "--- Release args: ${bumpversion_args}" - } + # 3: Git commit, tag and push + log_info "--- git push commit and tag..." + git push "$git_auth_url" "$CI_BUILD_REF_NAME" + git push "$git_auth_url" --tags + # 4: build new version distribution + log_info "--- build distribution packages..." + if [[ "$pkg_system" == "poetry" ]] + then + poetry build ${TRACE+--verbose} + else + # shellcheck disable=SC2086 + pip install ${PIP_OPTS} build + rm -rf dist + python -m build + fi + # 5: publish packages + log_info "--- publish distribution packages..." + if [[ "$pkg_system" == "poetry" ]] + then + poetry config repositories.user_defined "$PYTHON_REPOSITORY_URL" + poetry publish ${TRACE+--verbose} --username "$PYTHON_REPOSITORY_USERNAME" --password "$PYTHON_REPOSITORY_PASSWORD" --repository user_defined + else + # shellcheck disable=SC2086 + pip install ${PIP_OPTS} twine + twine upload ${TRACE+--verbose} --username "$PYTHON_REPOSITORY_USERNAME" --password "$PYTHON_REPOSITORY_PASSWORD" --repository-url "$PYTHON_REPOSITORY_URL" dist/* + fi + } function get_latest_template_version() { tag_json=$(wget -T 5 -q -O - "$CI_API_V4_URL/projects/to-be-continuous%2F$1/repository/tags?per_page=1" || echo "") @@ -359,10 +559,13 @@ variables: key: "$CI_COMMIT_REF_SLUG-python" paths: - ${PIP_CACHE_DIR} + - ${POETRY_CACHE_DIR} + - ${PIPENV_CACHE_DIR} before_script: - *python-scripts - install_ca_certs "${CUSTOM_CA_CERTS:-$DEFAULT_CA_CERTS}" - cd ${PYTHON_PROJECT_DIR} + - guess_build_system ############################################################################################### # stages definition # @@ -370,19 +573,33 @@ variables: stages: - build - test - - package-build - publish ############################################################################################### # build stage # ############################################################################################### +# build Python packages as artifacts +py-package: + extends: .python-base + stage: build + script: + - _package + artifacts: + paths: + - $PYTHON_PROJECT_DIR/dist/* + rules: + # exclude merge requests + - if: $CI_MERGE_REQUEST_ID + when: never + - if: '$PYTHON_PACKAGE_ENABLED == "true"' + py-lint: extends: .python-base stage: build script: - mkdir -p reports - chmod o+rwx reports - - install_requirements build + - install_requirements - _pip install pylint_gitlab - | if ! _run pylint --ignore=.cache --output-format=text ${PYLINT_ARGS} ${PYLINT_FILES:-$(find -type f -name "*.py")} @@ -418,7 +635,7 @@ py-compile: extends: .python-base stage: build script: - - install_requirements build + - install_requirements - _python -m compileall $PYTHON_COMPILE_ARGS rules: # exclude merge requests @@ -436,7 +653,7 @@ py-unittest: script: - mkdir -p reports - chmod o+rwx reports - - install_requirements test + - install_requirements # code coverage - _pip install coverage # JUnit XML report @@ -468,7 +685,7 @@ py-pytest: script: - mkdir -p reports - chmod o+rwx reports - - install_requirements test + - install_requirements - _pip install pytest pytest-cov coverage - _python -m pytest --junit-xml=reports/TEST-pytests.xml --cov --cov-report term --cov-report xml:reports/coverage.xml ${PYTEST_ARGS} coverage: /^TOTAL.+?(\d+\%)$/ @@ -495,7 +712,7 @@ py-nosetests: script: - mkdir -p reports - chmod o+rwx reports - - install_requirements test + - install_requirements - _run nosetests --with-xunit --xunit-file=reports/TEST-nosetests.xml --with-coverage --cover-erase --cover-xml --cover-xml-file=reports/coverage.xml --cover-html --cover-html-dir=reports/coverage ${NOSETESTS_ARGS} coverage: /^TOTAL.+?(\d+\%)$/ artifacts: @@ -524,6 +741,7 @@ py-bandit: script: - mkdir -p reports - chmod o+rwx reports + - install_requirements - _pip install bandit - | if ! _run bandit ${TRACE+--verbose} ${BANDIT_ARGS} @@ -560,8 +778,8 @@ py-safety: script: - mkdir -p reports - chmod o+rwx reports + - install_requirements - _pip install safety - - install_requirements build - | if ! _pip freeze | _run safety check --stdin ${SAFETY_ARGS} then @@ -609,7 +827,6 @@ py-trivy: fi trivy fs ${PYTHON_TRIVY_ARGS} --format table --exit-code 0 $PYTHON_PROJECT_DIR trivy fs ${PYTHON_TRIVY_ARGS} --format json --output reports/trivy-python.json --exit-code 1 $PYTHON_PROJECT_DIR - artifacts: name: "$CI_JOB_NAME artifacts from $CI_PROJECT_NAME on $CI_COMMIT_REF_SLUG" expire_in: 1 day @@ -631,65 +848,30 @@ py-trivy: when: manual allow_failure: true -############################################################################################### -# package stage # -############################################################################################### - -# (on tag creation): create packages as artifacts -py-package: - extends: .python-base - stage: package-build - script: - - _package - artifacts: - paths: - - $PYTHON_PROJECT_DIR/dist/*.tar.gz - - $PYTHON_PROJECT_DIR/dist/*.whl - rules: - # on tags - - if: '$CI_COMMIT_TAG' - - if: '$PYTHON_FORCE_PACKAGE == "true"' - - -############################################################################################### -# publish stage # -############################################################################################### - -# (on tag creation): performs a release -py-publish: - extends: .python-base - stage: publish - script: - - assert_defined "$TWINE_USERNAME" 'Missing required env $TWINE_USERNAME' - - assert_defined "$TWINE_PASSWORD" 'Missing required env $TWINE_PASSWORD' - - _publish - rules: - # on tags with $PYTHON_PUBLISH_ENABLED set - - if: '$PYTHON_PUBLISH_ENABLED == "true" && $CI_COMMIT_TAG' - - - # (manual from master branch): triggers a release (tag creation) py-release: extends: .python-base stage: publish script: - - git config --global user.email '$GITLAB_USER_EMAIL' - - git config --global user.name '$GITLAB_USER_LOGIN' + - git config --global user.email "$GITLAB_USER_EMAIL" + - git config --global user.name "$GITLAB_USER_LOGIN" - git checkout -B $CI_BUILD_REF_NAME + - configure_scm_auth - _release - - git_url_base=`echo ${CI_REPOSITORY_URL} | cut -d\@ -f2` - - git push https://${RELEASE_USERNAME}:${RELEASE_ACCESS_TOKEN}@${git_url_base} --tags - - git push https://${RELEASE_USERNAME}:${RELEASE_ACCESS_TOKEN}@${git_url_base} $CI_BUILD_REF_NAME + artifacts: + paths: + - $PYTHON_PROJECT_DIR/dist/* rules: # exclude merge requests - if: $CI_MERGE_REQUEST_ID when: never - # on production branch(es): manual & non-blocking if $RELEASE_USERNAME is set - - if: '$RELEASE_USERNAME && $CI_COMMIT_REF_NAME =~ $PROD_REF' - when: manual - allow_failure: true - # on integration branch(es): manual & non-blocking if $RELEASE_USERNAME is set - - if: '$RELEASE_USERNAME && $CI_COMMIT_REF_NAME =~ $INTEG_REF' + # exclude if $PYTHON_RELEASE_ENABLED not set + - if: '$PYTHON_RELEASE_ENABLED != "true"' + when: never + # exclude on non-prod, non-integ branches + - if: '$CI_COMMIT_REF_NAME !~ $PROD_REF && $CI_COMMIT_REF_NAME !~ $INTEG_REF' + when: never + # else: manual + - if: '$PYTHON_RELEASE_ENABLED == "true"' # useless but prevents GitLab warning when: manual allow_failure: true