Merge branch 'feat/new-release' into 'master'

new release Closes #46 and #45 See merge request to-be-continuous/python!78

Merge branch 'feat/new-release' into 'master'
54ffeb7a · Pierre Smeyers · 46c024b3 · 991a9b5a · 54ffeb7a · 54ffeb7a
Commit 54ffeb7a authored May 28, 2023 by Pierre Smeyers
--- a/README.md
+++ b/README.md
@@ -306,6 +306,7 @@ The release job is bound to the `publish` stage, appears only on production and
 | `GIT_USERNAME`          | Git username for Git push operations (see below)                        | _none_            |
 | :lock: `GIT_PASSWORD`   | Git password for Git push operations (see below)                        | _none_            |
 | :lock: `GIT_PRIVATE_KEY`| SSH key for Git push operations (see below)                             | _none_            |
+| `PYTHON_RELEASE_COMMIT_MESSAGE`| The Git commit message to use on the release commit. This is templated using the [Python Format String Syntax](http://docs.python.org/2/library/string.html#format-string-syntax). Available in the template context are current_version and new_version. | `chore(python-release): {current_version} → {new_version}` |
 | `PYTHON_REPOSITORY_URL`| Target PyPI repository to publish packages                              | _[GitLab project's PyPI packages repository](https://docs.gitlab.com/ee/user/packages/pypi_repository/)_ |
 | `PYTHON_REPOSITORY_USERNAME`| Target PyPI repository username credential                              | `gitlab-ci-token` |
 | :lock: `PYTHON_REPOSITORY_PASSWORD`| Target PyPI repository password credential                              | `$CI_JOB_TOKEN` |

--- a/kicker.json
+++ b/kicker.json
@@ -194,6 +194,12 @@
          "type": "boolean",
          "advanced": true
        },
+        {
+          "name": "PYTHON_RELEASE_COMMIT_MESSAGE",
+          "description": "The Git commit message to use on the release commit. This is templated using the [Python Format String Syntax](http://docs.python.org/2/library/string.html#format-string-syntax). Available in the template context are current_version and new_version.",
+          "default": "chore(python-release): {current_version} → {new_version}",
+          "advanced": true
+        },
        {
          "name": "GIT_USERNAME",
          "description": "Git username for Git push operations",

--- a/templates/gitlab-ci-python.yml
+++ b/templates/gitlab-ci-python.yml
@@ -77,6 +77,8 @@ variables:
  PROD_REF: '/^(master|main)$/'
  # default integration ref name (pattern)
  INTEG_REF: '/^develop$/'
+  # default release tag name (pattern)
+  RELEASE_REF: '/^v?[0-9]+\.[0-9]+\.[0-9]+$/'

  # compileall
  PYTHON_COMPILE_ARGS: "*"
@@ -93,6 +95,7 @@ variables:
  PYTHON_SBOM_OPTS: "--catalogers python-index-cataloger"

  PYTHON_RELEASE_NEXT: "minor"
+  PYTHON_RELEASE_COMMIT_MESSAGE: "chore(python-release): {current_version} → {new_version}"

  # By default, publish on the Packages registry of the project
  # https://docs.gitlab.com/ee/user/packages/pypi_repository/#authenticate-with-a-ci-job-token
@@ -263,15 +266,7 @@ variables:
    case "${PYTHON_BUILD_SYSTEM:-auto}" in
    auto)
      ;;
-    poetry*)
-      log_info "--- Build system explictly declared: ${PYTHON_BUILD_SYSTEM}"
-      return
-      ;;
-    setuptools*)
-      log_info "--- Build system explictly declared: ${PYTHON_BUILD_SYSTEM}"
-      return
-      ;;
-    pipenv*)
+    poetry*|setuptools*|pipenv*)
      log_info "--- Build system explictly declared: ${PYTHON_BUILD_SYSTEM}"
      return
      ;;
@@ -332,6 +327,14 @@ variables:
    fi
  }

+  function maybe_install_poetry() {
+    if [[ "$PYTHON_BUILD_SYSTEM" == poetry* ]] && ! command -v poetry > /dev/null
+    then
+      # shellcheck disable=SC2086
+      pip install ${PIP_OPTS} "$PYTHON_BUILD_SYSTEM"
+    fi
+  }
+
  # install requirements
  function install_requirements() {
    case "$PYTHON_BUILD_SYSTEM" in
@@ -339,8 +342,7 @@ variables:
      if  [[ ! -f "poetry.lock" ]]; then
        log_warn "Using Poetry but \\e[33;1mpoetry.lock\\e[0m file not found: you shall commit it with your project files"
      fi
-      # shellcheck disable=SC2086
-      pip install ${PIP_OPTS} "$PYTHON_BUILD_SYSTEM"
+      maybe_install_poetry
      poetry install ${PYTHON_EXTRA_DEPS:+--extras "$PYTHON_EXTRA_DEPS"}
      ;;
    setuptools*)
@@ -380,10 +382,9 @@ variables:
  }

  function _run() {
-    if [[ "${PYTHON_BUILD_SYSTEM}" =~ poetry.* ]]
+    if [[ "$PYTHON_BUILD_SYSTEM" == poetry* ]]
    then
-      # shellcheck disable=SC2086
-      if ! command -v poetry > /dev/null; then pip install ${PIP_OPTS} poetry; fi
+      maybe_install_poetry
      poetry run "$@"
    else
      "$@"
@@ -399,19 +400,16 @@ variables:
    _run pip ${PIP_OPTS} "$@"
  }

-  function _package() {
-    case "$PYTHON_BUILD_SYSTEM" in
-    poetry)
-      # shellcheck disable=SC2086
-      if ! command -v poetry > /dev/null; then pip install ${PIP_OPTS} poetry; fi
+  function py_package() {
+    if [[ "$PYTHON_BUILD_SYSTEM" == poetry* ]]
+    then
+      maybe_install_poetry
      poetry build
-      ;;
-    *)
+    else
      # shellcheck disable=SC2086
      pip install ${PIP_OPTS} build
      python -m build
-      ;;
-    esac
+    fi
  }

  function configure_scm_auth() {
@@ -437,44 +435,7 @@ variables:
    fi
  }

-  function _release() {
-    # 0: guess packaging system
-    if [[ -f "pyproject.toml" ]]
-    then
-      # that might be PEP 517 if a build-backend is specified
-      # otherwise it might be only used as configuration file for development tools...
-      build_backend=$(sed -rn 's/^build-backend *= *"([^"]*)".*/\1/p' pyproject.toml)
-      if [[ "$build_backend" ]]
-      then
-        case "$build_backend" in
-        poetry.core.masonry.api)
-          log_info "--- Packaging system auto-detected: Poetry"
-          pkg_system="poetry"
-          ;;
-        setuptools.build_meta)
-          log_info "--- Packaging system auto-detected: Setuptools (PEP 517)"
-          pkg_system="setuptools"
-          ;;
-        *)
-          log_error "--- Unsupported PEP 517 backend \\e[33;1m${build_backend}\\e[0m: abort"
-          exit 1
-          ;;
-        esac
-      fi
-    fi
-
-    if [[ -z "$pkg_system" ]]
-    then
-      if [[ -f "setup.py" ]]
-      then
-        log_info "--- Packaging system auto-detected: Setuptools (legacy)"
-        pkg_system="setuptools"
-      else
-        log_error "--- Couldn't find any supported packaging system: abort"
-        exit 1
-      fi
-    fi
-
+  function py_release() {
    # 1: retrieve next release info from semantic-release
    if [ "$SEMREL_INFO_ON" ] && [ "$PYTHON_SEMREL_RELEASE_DISABLED" != "true" ]
    then
@@ -491,47 +452,48 @@ variables:
    fi

    # 2: bumpversion (+ Git commit & tag)
-    if [[ "$pkg_system" == "poetry" ]]
+    if [[ "$PYTHON_BUILD_SYSTEM" == poetry* ]]
    then
-      # shellcheck disable=SC2086
-      if ! command -v poetry > /dev/null; then pip install ${PIP_OPTS} poetry; fi
+      maybe_install_poetry
      if [[ -z "$py_next_version" ]]
      then
        py_cur_version=$(poetry version --short)
        py_next_version="$PYTHON_RELEASE_NEXT"
      fi
-      log_info "[Poetry] change version \\e[1;94m${py_cur_version}\\e[0m → \\e[1;94m${py_next_version}\\e[0m"
+      log_info "[poetry] change version \\e[1;94m${py_cur_version}\\e[0m → \\e[1;94m${py_next_version}\\e[0m"
      poetry version ${TRACE+--verbose} "$py_next_version"
      # eval exact next version
      py_next_version=$(poetry version --short)
+      # Git commit and tag
      git add pyproject.toml
-      git commit -m "chore(python-release): $py_cur_version → $py_next_version [ci skip]"
+      # emulate Bumpversion to generate commit message
+      py_commit_message=$(python -c "print('$PYTHON_RELEASE_COMMIT_MESSAGE'.format(current_version='$py_cur_version', new_version='$py_next_version'))")
+      git commit -m "$py_commit_message"
      git tag "$py_next_version"
    else
      # Setuptools / bumpversion
      # shellcheck disable=SC2086
      pip install ${PIP_OPTS} bumpversion
-      py_commit_message="chore(python-release): {current_version} → {new_version} [ci skip]"
      if [[ "$py_next_version" ]]
      then
        # explicit release version (semantic-release)
-        log_info "[Setuptools] bumpversion \\e[1;94m${py_cur_version}\\e[0m → \\e[1;94m${py_next_version}\\e[0m"
+        log_info "[bumpversion] change version \\e[1;94m${py_cur_version}\\e[0m → \\e[1;94m${py_next_version}\\e[0m"
        # create cfg in case it doesn't exist - will be updated by bumpversion
        touch .bumpversion.cfg
-        bumpversion ${TRACE+--verbose} --current-version "$py_cur_version" --commit --message "$py_commit_message" --tag --tag-name "{new_version}" "$py_release_part"
+        bumpversion ${TRACE+--verbose} --current-version "$py_cur_version" --commit ${PYTHON_RELEASE_COMMIT_MESSAGE+--message "$PYTHON_RELEASE_COMMIT_MESSAGE"} --tag --tag-name "{new_version}" "$py_release_part"
      elif [[ -f ".bumpversion.cfg" ]]
      then
        # current version shall be set in .bumpversion.cfg
        py_release_part="$PYTHON_RELEASE_NEXT"
        log_info "[bumpversion] increase \\e[1;94m${py_release_part}\\e[0m"
-        bumpversion ${TRACE+--verbose} --commit --message "$py_commit_message" --tag --tag-name "{new_version}" "$py_release_part"
+        bumpversion ${TRACE+--verbose} --commit ${PYTHON_RELEASE_COMMIT_MESSAGE+--message "$PYTHON_RELEASE_COMMIT_MESSAGE"} "$PYTHON_RELEASE_COMMIT_MESSAGE" --tag --tag-name "{new_version}" "$py_release_part"
      elif [[ -f "setup.py" ]]
      then
        # retrieve current version from setup.py
        py_cur_version=$(python setup.py --version)
        py_release_part="$PYTHON_RELEASE_NEXT"
-        log_info "[Setuptools] bumpversion ($py_release_part) from \\e[1;94m${py_cur_version}\\e[0m"
-        bumpversion ${TRACE+--verbose} --current-version "$py_cur_version" --commit --message "$py_commit_message" --tag --tag-name "{new_version}" "$py_release_part" setup.py
+        log_info "[bumpversion] increase \\e[1;94m${py_release_part}\\e[0m (from current \\e[1;94m${py_cur_version}\\e[0m)"
+        bumpversion ${TRACE+--verbose} --current-version "$py_cur_version" --commit ${PYTHON_RELEASE_COMMIT_MESSAGE+--message "$PYTHON_RELEASE_COMMIT_MESSAGE"} --tag --tag-name "{new_version}" "$py_release_part" setup.py
      else
        log_error "--- setup.py or .bumpversion.cfg file required to retrieve current version: cannot perform release"
        exit 1
@@ -542,28 +504,28 @@ variables:
    log_info "--- git push commit and tag..."
    git push "$git_auth_url" "$CI_COMMIT_REF_NAME"
    git push "$git_auth_url" --tags
+  }

-    # 4: build new version distribution
-    log_info "--- build distribution packages..."
-    if [[ "$pkg_system" == "poetry" ]]
+  function py_publish() {
+    if [[ "$PYTHON_BUILD_SYSTEM" == poetry* ]]
    then
+      maybe_install_poetry
+ 
+      log_info "--- build packages (poetry)..."
      poetry build ${TRACE+--verbose}
-    else
-      # shellcheck disable=SC2086
-      pip install ${PIP_OPTS} build
-      rm -rf dist
-      python -m build
-    fi

-    # 5: publish packages
-    log_info "--- publish distribution packages..."
-    if [[ "$pkg_system" == "poetry" ]]
-    then
+      log_info "--- publish packages (poetry)..."
      poetry config repositories.user_defined "$PYTHON_REPOSITORY_URL"
      poetry publish ${TRACE+--verbose} --username "$PYTHON_REPOSITORY_USERNAME" --password "$PYTHON_REPOSITORY_PASSWORD" --repository user_defined
    else
      # shellcheck disable=SC2086
-      pip install ${PIP_OPTS} twine
+      pip install ${PIP_OPTS} build twine
+ 
+      log_info "--- build packages (build)..."
+      rm -rf dist
+      python -m build
+
+      log_info "--- publish packages (twine)..."
      twine upload ${TRACE+--verbose} --username "$PYTHON_REPOSITORY_USERNAME" --password "$PYTHON_REPOSITORY_PASSWORD" --repository-url "$PYTHON_REPOSITORY_URL" dist/*
    fi
  }
@@ -572,6 +534,14 @@ variables:

  # ENDSCRIPT

+###############################################################################################
+#                                      stages definition                                      #
+###############################################################################################
+stages:
+  - build
+  - test
+  - publish
+
 ###############################################################################################
 #                                      Generic python job                                     #
 ###############################################################################################
@@ -594,14 +564,6 @@ variables:
    - cd ${PYTHON_PROJECT_DIR}
    - guess_build_system

-###############################################################################################
-#                                      stages definition                                      #
-###############################################################################################
-stages:
-  - build
-  - test
-  - publish
-
 ###############################################################################################
 #                                      build stage                                             #
 ###############################################################################################
@@ -610,7 +572,7 @@ py-package:
  extends: .python-base
  stage: build
  script:
-    - _package
+    - py_package
  artifacts:
    paths:
      - $PYTHON_PROJECT_DIR/dist/*
@@ -811,7 +773,7 @@ py-trivy:
    - apt-get update
    - apt-get install trivy
    - |
-      if [[ "$PYTHON_BUILD_SYSTEM" =~ poetry.* ]]
+      if [[ "$PYTHON_BUILD_SYSTEM" == poetry* ]]
      then
        # When using Poetry, `pip freeze` outputs a requirements.txt with @file URLs for each wheel
        # These @file URLs in requirements.txt are not supported by Trivy
@@ -906,7 +868,7 @@ py-release:
    - git config --global user.name "$GITLAB_USER_LOGIN"
    - git checkout -B $CI_COMMIT_REF_NAME
    - configure_scm_auth
-    - _release
+    - py_release
  artifacts:
    paths:
      - $PYTHON_PROJECT_DIR/dist/*
@@ -918,3 +880,19 @@ py-release:
    - if: '$CI_COMMIT_REF_NAME =~ $PROD_REF || $CI_COMMIT_REF_NAME =~ $INTEG_REF'
      when: manual
      allow_failure: true
+
+# (auto from release tag): publishes the Python package(s) to a PyPi registry
+py-publish:
+  extends: .python-base
+  stage: publish
+  script:
+    - py_publish $CI_COMMIT_TAG
+  artifacts:
+    paths:
+      - $PYTHON_PROJECT_DIR/dist/*
+  rules:
+    # exclude if $PYTHON_RELEASE_ENABLED not set
+    - if: '$PYTHON_RELEASE_ENABLED != "true"'
+      when: never
+    # on tag with release pattern: auto
+    - if: '$CI_COMMIT_TAG =~ $RELEASE_REF'