diff --git a/README.md b/README.md index 6fb9abd4ed5ce47f9464039d11f46de8f499831c..351c50d34680e4e984073185b2fc0fd3732eac60 100644 --- a/README.md +++ b/README.md @@ -306,6 +306,7 @@ The release job is bound to the `publish` stage, appears only on production and | `GIT_USERNAME` | Git username for Git push operations (see below) | _none_ | | :lock: `GIT_PASSWORD` | Git password for Git push operations (see below) | _none_ | | :lock: `GIT_PRIVATE_KEY`| SSH key for Git push operations (see below) | _none_ | +| `PYTHON_RELEASE_COMMIT_MESSAGE`| The Git commit message to use on the release commit. This is templated using the [Python Format String Syntax](http://docs.python.org/2/library/string.html#format-string-syntax). Available in the template context are current_version and new_version. | `chore(python-release): {current_version} → {new_version}` | | `PYTHON_REPOSITORY_URL`| Target PyPI repository to publish packages | _[GitLab project's PyPI packages repository](https://docs.gitlab.com/ee/user/packages/pypi_repository/)_ | | `PYTHON_REPOSITORY_USERNAME`| Target PyPI repository username credential | `gitlab-ci-token` | | :lock: `PYTHON_REPOSITORY_PASSWORD`| Target PyPI repository password credential | `$CI_JOB_TOKEN` | diff --git a/kicker.json b/kicker.json index 60821c84041cf352567abc2c4bb5829e328221a7..65ac497aff67d777ef1bca716e6159e29a6b2633 100644 --- a/kicker.json +++ b/kicker.json @@ -194,6 +194,12 @@ "type": "boolean", "advanced": true }, + { + "name": "PYTHON_RELEASE_COMMIT_MESSAGE", + "description": "The Git commit message to use on the release commit. This is templated using the [Python Format String Syntax](http://docs.python.org/2/library/string.html#format-string-syntax). Available in the template context are current_version and new_version.", + "default": "chore(python-release): {current_version} → {new_version}", + "advanced": true + }, { "name": "GIT_USERNAME", "description": "Git username for Git push operations", diff --git a/templates/gitlab-ci-python.yml b/templates/gitlab-ci-python.yml index 09ee2579ab6ab77c58302808b3cb9082b987940f..4b443f9f5d464b7938da03d1d638a5212b9124e1 100644 --- a/templates/gitlab-ci-python.yml +++ b/templates/gitlab-ci-python.yml @@ -77,6 +77,8 @@ variables: PROD_REF: '/^(master|main)$/' # default integration ref name (pattern) INTEG_REF: '/^develop$/' + # default release tag name (pattern) + RELEASE_REF: '/^v?[0-9]+\.[0-9]+\.[0-9]+$/' # compileall PYTHON_COMPILE_ARGS: "*" @@ -93,6 +95,7 @@ variables: PYTHON_SBOM_OPTS: "--catalogers python-index-cataloger" PYTHON_RELEASE_NEXT: "minor" + PYTHON_RELEASE_COMMIT_MESSAGE: "chore(python-release): {current_version} → {new_version}" # By default, publish on the Packages registry of the project # https://docs.gitlab.com/ee/user/packages/pypi_repository/#authenticate-with-a-ci-job-token @@ -263,15 +266,7 @@ variables: case "${PYTHON_BUILD_SYSTEM:-auto}" in auto) ;; - poetry*) - log_info "--- Build system explictly declared: ${PYTHON_BUILD_SYSTEM}" - return - ;; - setuptools*) - log_info "--- Build system explictly declared: ${PYTHON_BUILD_SYSTEM}" - return - ;; - pipenv*) + poetry*|setuptools*|pipenv*) log_info "--- Build system explictly declared: ${PYTHON_BUILD_SYSTEM}" return ;; @@ -332,6 +327,14 @@ variables: fi } + function maybe_install_poetry() { + if [[ "$PYTHON_BUILD_SYSTEM" == poetry* ]] && ! command -v poetry > /dev/null + then + # shellcheck disable=SC2086 + pip install ${PIP_OPTS} "$PYTHON_BUILD_SYSTEM" + fi + } + # install requirements function install_requirements() { case "$PYTHON_BUILD_SYSTEM" in @@ -339,8 +342,7 @@ variables: if [[ ! -f "poetry.lock" ]]; then log_warn "Using Poetry but \\e[33;1mpoetry.lock\\e[0m file not found: you shall commit it with your project files" fi - # shellcheck disable=SC2086 - pip install ${PIP_OPTS} "$PYTHON_BUILD_SYSTEM" + maybe_install_poetry poetry install ${PYTHON_EXTRA_DEPS:+--extras "$PYTHON_EXTRA_DEPS"} ;; setuptools*) @@ -380,10 +382,9 @@ variables: } function _run() { - if [[ "${PYTHON_BUILD_SYSTEM}" =~ poetry.* ]] + if [[ "$PYTHON_BUILD_SYSTEM" == poetry* ]] then - # shellcheck disable=SC2086 - if ! command -v poetry > /dev/null; then pip install ${PIP_OPTS} poetry; fi + maybe_install_poetry poetry run "$@" else "$@" @@ -399,19 +400,16 @@ variables: _run pip ${PIP_OPTS} "$@" } - function _package() { - case "$PYTHON_BUILD_SYSTEM" in - poetry) - # shellcheck disable=SC2086 - if ! command -v poetry > /dev/null; then pip install ${PIP_OPTS} poetry; fi + function py_package() { + if [[ "$PYTHON_BUILD_SYSTEM" == poetry* ]] + then + maybe_install_poetry poetry build - ;; - *) + else # shellcheck disable=SC2086 pip install ${PIP_OPTS} build python -m build - ;; - esac + fi } function configure_scm_auth() { @@ -437,44 +435,7 @@ variables: fi } - function _release() { - # 0: guess packaging system - if [[ -f "pyproject.toml" ]] - then - # that might be PEP 517 if a build-backend is specified - # otherwise it might be only used as configuration file for development tools... - build_backend=$(sed -rn 's/^build-backend *= *"([^"]*)".*/\1/p' pyproject.toml) - if [[ "$build_backend" ]] - then - case "$build_backend" in - poetry.core.masonry.api) - log_info "--- Packaging system auto-detected: Poetry" - pkg_system="poetry" - ;; - setuptools.build_meta) - log_info "--- Packaging system auto-detected: Setuptools (PEP 517)" - pkg_system="setuptools" - ;; - *) - log_error "--- Unsupported PEP 517 backend \\e[33;1m${build_backend}\\e[0m: abort" - exit 1 - ;; - esac - fi - fi - - if [[ -z "$pkg_system" ]] - then - if [[ -f "setup.py" ]] - then - log_info "--- Packaging system auto-detected: Setuptools (legacy)" - pkg_system="setuptools" - else - log_error "--- Couldn't find any supported packaging system: abort" - exit 1 - fi - fi - + function py_release() { # 1: retrieve next release info from semantic-release if [ "$SEMREL_INFO_ON" ] && [ "$PYTHON_SEMREL_RELEASE_DISABLED" != "true" ] then @@ -491,47 +452,48 @@ variables: fi # 2: bumpversion (+ Git commit & tag) - if [[ "$pkg_system" == "poetry" ]] + if [[ "$PYTHON_BUILD_SYSTEM" == poetry* ]] then - # shellcheck disable=SC2086 - if ! command -v poetry > /dev/null; then pip install ${PIP_OPTS} poetry; fi + maybe_install_poetry if [[ -z "$py_next_version" ]] then py_cur_version=$(poetry version --short) py_next_version="$PYTHON_RELEASE_NEXT" fi - log_info "[Poetry] change version \\e[1;94m${py_cur_version}\\e[0m → \\e[1;94m${py_next_version}\\e[0m" + log_info "[poetry] change version \\e[1;94m${py_cur_version}\\e[0m → \\e[1;94m${py_next_version}\\e[0m" poetry version ${TRACE+--verbose} "$py_next_version" # eval exact next version py_next_version=$(poetry version --short) + # Git commit and tag git add pyproject.toml - git commit -m "chore(python-release): $py_cur_version → $py_next_version [ci skip]" + # emulate Bumpversion to generate commit message + py_commit_message=$(python -c "print('$PYTHON_RELEASE_COMMIT_MESSAGE'.format(current_version='$py_cur_version', new_version='$py_next_version'))") + git commit -m "$py_commit_message" git tag "$py_next_version" else # Setuptools / bumpversion # shellcheck disable=SC2086 pip install ${PIP_OPTS} bumpversion - py_commit_message="chore(python-release): {current_version} → {new_version} [ci skip]" if [[ "$py_next_version" ]] then # explicit release version (semantic-release) - log_info "[Setuptools] bumpversion \\e[1;94m${py_cur_version}\\e[0m → \\e[1;94m${py_next_version}\\e[0m" + log_info "[bumpversion] change version \\e[1;94m${py_cur_version}\\e[0m → \\e[1;94m${py_next_version}\\e[0m" # create cfg in case it doesn't exist - will be updated by bumpversion touch .bumpversion.cfg - bumpversion ${TRACE+--verbose} --current-version "$py_cur_version" --commit --message "$py_commit_message" --tag --tag-name "{new_version}" "$py_release_part" + bumpversion ${TRACE+--verbose} --current-version "$py_cur_version" --commit ${PYTHON_RELEASE_COMMIT_MESSAGE+--message "$PYTHON_RELEASE_COMMIT_MESSAGE"} --tag --tag-name "{new_version}" "$py_release_part" elif [[ -f ".bumpversion.cfg" ]] then # current version shall be set in .bumpversion.cfg py_release_part="$PYTHON_RELEASE_NEXT" log_info "[bumpversion] increase \\e[1;94m${py_release_part}\\e[0m" - bumpversion ${TRACE+--verbose} --commit --message "$py_commit_message" --tag --tag-name "{new_version}" "$py_release_part" + bumpversion ${TRACE+--verbose} --commit ${PYTHON_RELEASE_COMMIT_MESSAGE+--message "$PYTHON_RELEASE_COMMIT_MESSAGE"} "$PYTHON_RELEASE_COMMIT_MESSAGE" --tag --tag-name "{new_version}" "$py_release_part" elif [[ -f "setup.py" ]] then # retrieve current version from setup.py py_cur_version=$(python setup.py --version) py_release_part="$PYTHON_RELEASE_NEXT" - log_info "[Setuptools] bumpversion ($py_release_part) from \\e[1;94m${py_cur_version}\\e[0m" - bumpversion ${TRACE+--verbose} --current-version "$py_cur_version" --commit --message "$py_commit_message" --tag --tag-name "{new_version}" "$py_release_part" setup.py + log_info "[bumpversion] increase \\e[1;94m${py_release_part}\\e[0m (from current \\e[1;94m${py_cur_version}\\e[0m)" + bumpversion ${TRACE+--verbose} --current-version "$py_cur_version" --commit ${PYTHON_RELEASE_COMMIT_MESSAGE+--message "$PYTHON_RELEASE_COMMIT_MESSAGE"} --tag --tag-name "{new_version}" "$py_release_part" setup.py else log_error "--- setup.py or .bumpversion.cfg file required to retrieve current version: cannot perform release" exit 1 @@ -542,28 +504,28 @@ variables: log_info "--- git push commit and tag..." git push "$git_auth_url" "$CI_COMMIT_REF_NAME" git push "$git_auth_url" --tags + } - # 4: build new version distribution - log_info "--- build distribution packages..." - if [[ "$pkg_system" == "poetry" ]] + function py_publish() { + if [[ "$PYTHON_BUILD_SYSTEM" == poetry* ]] then + maybe_install_poetry + + log_info "--- build packages (poetry)..." poetry build ${TRACE+--verbose} + + log_info "--- publish packages (poetry)..." + poetry config repositories.user_defined "$PYTHON_REPOSITORY_URL" + poetry publish ${TRACE+--verbose} --username "$PYTHON_REPOSITORY_USERNAME" --password "$PYTHON_REPOSITORY_PASSWORD" --repository user_defined else # shellcheck disable=SC2086 - pip install ${PIP_OPTS} build + pip install ${PIP_OPTS} build twine + + log_info "--- build packages (build)..." rm -rf dist python -m build - fi - # 5: publish packages - log_info "--- publish distribution packages..." - if [[ "$pkg_system" == "poetry" ]] - then - poetry config repositories.user_defined "$PYTHON_REPOSITORY_URL" - poetry publish ${TRACE+--verbose} --username "$PYTHON_REPOSITORY_USERNAME" --password "$PYTHON_REPOSITORY_PASSWORD" --repository user_defined - else - # shellcheck disable=SC2086 - pip install ${PIP_OPTS} twine + log_info "--- publish packages (twine)..." twine upload ${TRACE+--verbose} --username "$PYTHON_REPOSITORY_USERNAME" --password "$PYTHON_REPOSITORY_PASSWORD" --repository-url "$PYTHON_REPOSITORY_URL" dist/* fi } @@ -572,6 +534,14 @@ variables: # ENDSCRIPT +############################################################################################### +# stages definition # +############################################################################################### +stages: + - build + - test + - publish + ############################################################################################### # Generic python job # ############################################################################################### @@ -594,14 +564,6 @@ variables: - cd ${PYTHON_PROJECT_DIR} - guess_build_system -############################################################################################### -# stages definition # -############################################################################################### -stages: - - build - - test - - publish - ############################################################################################### # build stage # ############################################################################################### @@ -610,7 +572,7 @@ py-package: extends: .python-base stage: build script: - - _package + - py_package artifacts: paths: - $PYTHON_PROJECT_DIR/dist/* @@ -811,7 +773,7 @@ py-trivy: - apt-get update - apt-get install trivy - | - if [[ "$PYTHON_BUILD_SYSTEM" =~ poetry.* ]] + if [[ "$PYTHON_BUILD_SYSTEM" == poetry* ]] then # When using Poetry, `pip freeze` outputs a requirements.txt with @file URLs for each wheel # These @file URLs in requirements.txt are not supported by Trivy @@ -862,7 +824,7 @@ py-sbom: - install_requirements - | case "$PYTHON_BUILD_SYSTEM" in - setuptools* | reqfile) + setuptools*|reqfile) _pip freeze > "${PYTHON_REQS_FILE}" ;; esac @@ -906,7 +868,7 @@ py-release: - git config --global user.name "$GITLAB_USER_LOGIN" - git checkout -B $CI_COMMIT_REF_NAME - configure_scm_auth - - _release + - py_release artifacts: paths: - $PYTHON_PROJECT_DIR/dist/* @@ -918,3 +880,19 @@ py-release: - if: '$CI_COMMIT_REF_NAME =~ $PROD_REF || $CI_COMMIT_REF_NAME =~ $INTEG_REF' when: manual allow_failure: true + +# (auto from release tag): publishes the Python package(s) to a PyPi registry +py-publish: + extends: .python-base + stage: publish + script: + - py_publish $CI_COMMIT_TAG + artifacts: + paths: + - $PYTHON_PROJECT_DIR/dist/* + rules: + # exclude if $PYTHON_RELEASE_ENABLED not set + - if: '$PYTHON_RELEASE_ENABLED != "true"' + when: never + # on tag with release pattern: auto + - if: '$CI_COMMIT_TAG =~ $RELEASE_REF'