diff --git a/README.md b/README.md index 25b5f172590c07b9aec9fcc20114c1f0fc669ef4..94c6b879f4808675826d6411848a160a1ca45830 100644 --- a/README.md +++ b/README.md @@ -226,18 +226,17 @@ It is bound to the `test` stage, and uses the following variables: This job outputs a **textual report** in the console, and in case of failure also exports a JSON report in the `reports/` directory _(relative to project root dir)_. - ### Package jobs #### `py-package` job -This job is performs a packaging of your Python code. +This job is **disabled by default** and performs a packaging of your Python code. It is bound to the `package-build` stage, applies only on git tags and uses the following variables: | Name | description | default value | | --------------- | ---------------------------------------------------- | ------------- | -| `PYTHON_FORCE_PACKAGE` | Force the packaging even if not on tag related event | _none_ | +| `PYTHON_FORCE_PACKAGE` | Set to `true` to force the packaging even if not on tag related event | _none_ (disabled) | ### Publish jobs diff --git a/templates/gitlab-ci-python.yml b/templates/gitlab-ci-python.yml index 022ce5aa068b8809e3ab386887a2b86e3d580800..59329dc7a6d115685939952e1e928c83e6eb762a 100644 --- a/templates/gitlab-ci-python.yml +++ b/templates/gitlab-ci-python.yml @@ -69,90 +69,9 @@ variables: fi } - function install_test_requirements() { - if [[ -f "pyproject.toml" ]] && [[ "${PYTHON_POETRY_DISABLED}" != "true" ]]; then - if [[ ! -f "poetry.lock" ]]; then - log_error "Poetry detected but \\e[33;1mpoetry.lock\\e[0m file not found: you shall commit it with your project files" - exit 1 - fi - log_info "--- Poetry detected: generating \\e[33;1m${TEST_REQUIREMENTS_FILE}\\e[0m from poetry.lock" - pip install poetry - poetry export --without-hashes ${PYTHON_POETRY_EXTRAS:+--extras "$PYTHON_POETRY_EXTRAS"} --dev -f requirements.txt --output "${TEST_REQUIREMENTS_FILE}" - fi - - if [[ -f "${TEST_REQUIREMENTS_FILE}" ]]; then - log_info "--- installing from ${TEST_REQUIREMENTS_FILE} file" - # shellcheck disable=SC2086 - pip install ${PIP_OPTS} -r "${TEST_REQUIREMENTS_FILE}" - else - log_info "--- no test requirements file found from env or file ${TEST_REQUIREMENTS_FILE} does not exist" - fi - } - function install_requirements() { - if [[ -f "pyproject.toml" ]] && [[ "${PYTHON_POETRY_DISABLED}" != "true" ]]; then - if [[ ! -f "poetry.lock" ]]; then - log_error "Poetry detected but \\e[33;1mpoetry.lock\\e[0m file not found: you shall commit it with your project files" - exit 1 - fi - log_info "--- Poetry detected: generating \\e[33;1m${REQUIREMENTS_FILE}\\e[0m from poetry.lock" - pip install poetry - poetry export --without-hashes ${PYTHON_POETRY_EXTRAS:+--extras "$PYTHON_POETRY_EXTRAS"} -f requirements.txt --output "${REQUIREMENTS_FILE}" - fi - if [[ -f "${REQUIREMENTS_FILE}" ]]; then - log_info "--- installing from ${REQUIREMENTS_FILE} file" - # shellcheck disable=SC2086 - pip install ${PIP_OPTS} -r "${REQUIREMENTS_FILE}" - elif [[ -f "${SETUP_PY_DIR}/setup.py" ]]; then - log_info "--- installing from ${SETUP_PY_DIR}/setup.py file" - # shellcheck disable=SC2086 - pip install ${PIP_OPTS} "${SETUP_PY_DIR}/" - else - log_info "--- no requirements or setup.py file found from env or file ${REQUIREMENTS_FILE} - ${SETUP_PY_DIR}/setup.py does not exist" - fi - } - - function install_doc_requirements() { - if [[ -f "pyproject.toml" ]] && [[ "${PYTHON_POETRY_DISABLED}" != "true" ]]; then - if [[ ! -f "poetry.lock" ]]; then - log_error "Poetry detected but \\e[33;1mpoetry.lock\\e[0m file not found: you shall commit it with your project files" - exit 1 - fi - log_info "--- Poetry detected: generating \\e[33;1m${TEST_REQUIREMENTS_FILE}\\e[0m from poetry.lock" - pip install poetry - poetry export --without-hashes ${PYTHON_POETRY_EXTRAS:+--extras "$PYTHON_POETRY_EXTRAS"} -f requirements.txt --output "${DOCS_REQUIREMENTS_FILE}" - fi - - if [[ -f "${DOCS_REQUIREMENTS_FILE}" ]]; then - log_info "--- installing from ${DOCS_REQUIREMENTS_FILE} file" - # shellcheck disable=SC2086 - pip install ${PIP_OPTS} -r "${DOCS_REQUIREMENTS_FILE}" - elif [[ -f "${SETUP_PY_DIR}/setup.py" ]]; then - log_info "--- installing from ${SETUP_PY_DIR}/setup.py file" - # shellcheck disable=SC2086 - pip install ${PIP_OPTS} "${SETUP_PY_DIR}/" - else - log_info "--- no doc requirements file found from env or file ${DOCS_REQUIREMENTS_FILE} - ${SETUP_PY_DIR}/setup.py does not exist" - fi - } - function release_args() { - if [[ -f ".bumpversion.cfg" ]]; then - log_info "--- .bumpversion.cfg file found " - export bumpversion_args="${RELEASE_VERSION_PART} --verbose" - else - log_info "--- No .bumpversion.cfg file found " - if [[ -f "setup.py" ]]; then - log_info "--- Getting current version of setup.py file " - current_version=$(python setup.py --version) - export bumpversion_args=" --verbose --current-version ${current_version} --tag --tag-name {new_version} --commit ${RELEASE_VERSION_PART} setup.py" - else - log_warn "--- No setup.py file found. Cannot perform release." - fi - fi - log_info "--- Release args: ${bumpversion_args}" - } function install_ca_certs() { certs=$1 @@ -289,6 +208,113 @@ variables: log_info "... done" } + # install requirements + # arg1: 'build' (build only) or 'test' (build + test) + function install_requirements() { + target=$1 + if [[ -f "pyproject.toml" ]] && [[ "${PYTHON_POETRY_DISABLED}" != "true" ]]; then + if [[ ! -f "poetry.lock" ]]; then + log_error "Poetry detected but \\e[33;1mpoetry.lock\\e[0m file not found: you shall commit it with your project files" + exit 1 + fi + pip install -U poetry + if [[ "$target" == "build" ]]; then + log_info "--- Poetry detected: install build only requirements" + poetry install --no-dev ${PYTHON_POETRY_EXTRAS:+--extras "$PYTHON_POETRY_EXTRAS"} + else + log_info "--- Poetry detected: install build and dev requirements" + poetry install ${PYTHON_POETRY_EXTRAS:+--extras "$PYTHON_POETRY_EXTRAS"} + fi + elif [[ -f "${REQUIREMENTS_FILE}" ]]; then + log_info "--- installing build requirements from \\e[33;1m${REQUIREMENTS_FILE}\\e[0m" + # shellcheck disable=SC2086 + pip install ${PIP_OPTS} -r "${REQUIREMENTS_FILE}" + if [[ "$target" == "test" ]] && [[ -f "${TEST_REQUIREMENTS_FILE}" ]]; then + log_info "--- installing test requirements from \\e[33;1m${TEST_REQUIREMENTS_FILE}\\e[0m" + # shellcheck disable=SC2086 + pip install ${PIP_OPTS} -r "${TEST_REQUIREMENTS_FILE}" + fi + elif [[ -f "${SETUP_PY_DIR}/setup.py" ]]; then + log_info "--- installing requirements from \\e[33;1m${SETUP_PY_DIR}/setup.py\\e[0m" + # shellcheck disable=SC2086 + pip install ${PIP_OPTS} "${SETUP_PY_DIR}/" + else + log_info "--- no dependency management tool, nor requirements file nor setup.py file found: skip install dependencies" + fi + } + + function _run() { + if [[ -f "poetry.lock" ]] && [[ "${PYTHON_POETRY_DISABLED}" != "true" ]]; then + if ! command -v poetry > /dev/null + then + pip install -U poetry + fi + poetry run "$@" + else + "$@" + fi + } + + function _python() { + _run python "$@" + } + + function _pip() { + _run pip "$@" + } + + function _package(){ + if [[ -f "poetry.lock" ]] && [[ "${PYTHON_POETRY_DISABLED}" != "true" ]]; then + pip install -U poetry + poetry build + else + python setup.py sdist bdist_wheel + fi + } + function _publish() { + if [[ -f "poetry.lock" ]] && [[ "${PYTHON_POETRY_DISABLED}" != "true" ]]; then + pip install -U poetry + poetry config repositories.user_defined "$TWINE_REPOSITORY_URL" + poetry publish --username "$TWINE_USERNAME" --password "$TWINE_PASSWORD" --repository user_defined + else + pip install -U twine setuptools + pip list + + twine upload --verbose dist/*.tar.gz + twine upload --verbose dist/*.whl + fi + } + + function _release() { + if [[ -f "poetry.lock" ]] && [[ "${PYTHON_POETRY_DISABLED}" != "true" ]]; then + pip install -U poetry + poetry version "${RELEASE_VERSION_PART}" + else + pip install -U bumpversion + release_args + bumpversion "${bumpversion_args}" + fi + } + function release_args() { + if [[ -f ".bumpversion.cfg" ]]; then + log_info "--- .bumpversion.cfg file found " + export bumpversion_args="${RELEASE_VERSION_PART} --verbose" + else + log_info "--- No .bumpversion.cfg file found " + if [[ -f "setup.py" ]]; then + log_info "--- Getting current version of setup.py file " + current_version=$(python setup.py --version) + export bumpversion_args=" --verbose --current-version ${current_version} --tag --tag-name {new_version} --commit ${RELEASE_VERSION_PART} setup.py" + else + log_warn "--- No setup.py file found. Cannot perform release." + fi + fi + log_info "--- Release args: ${bumpversion_args}" + } + + + + function get_latest_template_version() { tag_json=$(wget -T 5 -q -O - "$CI_API_V4_URL/projects/to-be-continuous%2F$1/repository/tags?per_page=1" || echo "") echo "$tag_json" | sed -rn 's/^.*"name":"([^"]*)".*$/\1/p' @@ -347,20 +373,20 @@ py-lint: extends: .python-base stage: build script: - - install_requirements - - pip install pylint_gitlab + - mkdir -p reports + - chmod o+rwx reports + - install_requirements build + - _pip install -U pylint_gitlab - | - if ! pylint --ignore=.cache --output-format=text ${PYLINT_ARGS} ${PYLINT_FILES:-$(find -type f -name "*.py")} + if ! _run pylint --ignore=.cache --output-format=text ${PYLINT_ARGS} ${PYLINT_FILES:-$(find -type f -name "*.py")} then # failed: also generate codeclimate report - mkdir -p reports - chmod o+rwx reports - pylint --ignore=.cache --output-format=pylint_gitlab.GitlabCodeClimateReporter ${PYLINT_ARGS} ${PYLINT_FILES:-$(find -type f -name "*.py")} > reports/pylint-codeclimate.json + + _run pylint --ignore=.cache --output-format=pylint_gitlab.GitlabCodeClimateReporter ${PYLINT_ARGS} ${PYLINT_FILES:-$(find -type f -name "*.py")} > reports/pylint-codeclimate.json exit 1 else # success: generate empty codeclimate report (required by GitLab :( ) - mkdir -p reports - chmod o+rwx reports + echo "[]" > reports/pylint-codeclimate.json fi artifacts: @@ -387,8 +413,8 @@ py-compile: extends: .python-base stage: build script: - - install_requirements - - python -m compileall $PYTHON_COMPILE_ARGS + - install_requirements build + - _python -m compileall $PYTHON_COMPILE_ARGS rules: # exclude merge requests - if: $CI_MERGE_REQUEST_ID @@ -405,15 +431,14 @@ py-unittest: script: - mkdir -p reports - chmod o+rwx reports - - install_requirements - - install_test_requirements + - install_requirements test # code coverage - - pip install -U coverage + - _pip install -U coverage # JUnit XML report - - pip install -U unittest-xml-reporting - - coverage run -m xmlrunner discover -o "reports/" $UNITTEST_ARGS - - coverage report -m - - coverage xml -o "reports/coverage.xml" + - _pip install -U unittest-xml-reporting + - _run coverage run -m xmlrunner discover -o "reports/" $UNITTEST_ARGS + - _run coverage report -m + - _run coverage xml -o "reports/coverage.xml" coverage: /^TOTAL.+?(\d+\%)$/ artifacts: name: "$CI_JOB_NAME artifacts from $CI_PROJECT_NAME on $CI_COMMIT_REF_SLUG" @@ -436,12 +461,11 @@ py-pytest: extends: .python-base stage: build script: - - install_requirements - - install_test_requirements - mkdir -p reports - chmod o+rwx reports - - pip install -U pytest pytest-cov coverage - - python -m pytest --junit-xml=reports/TEST-pytests.xml --cov --cov-report term --cov-report xml:reports/coverage.xml ${PYTEST_ARGS} + - install_requirements test + - _pip install -U pytest pytest-cov coverage + - _python -m pytest --junit-xml=reports/TEST-pytests.xml --cov --cov-report term --cov-report xml:reports/coverage.xml ${PYTEST_ARGS} coverage: /^TOTAL.+?(\d+\%)$/ artifacts: name: "$CI_JOB_NAME artifacts from $CI_PROJECT_NAME on $CI_COMMIT_REF_SLUG" @@ -464,11 +488,10 @@ py-nosetests: extends: .python-base stage: build script: - - install_requirements - - install_test_requirements - mkdir -p reports - chmod o+rwx reports - - nosetests --with-xunit --xunit-file=reports/TEST-nosetests.xml --with-coverage --cover-erase --cover-xml --cover-xml-file=reports/coverage.xml --cover-html --cover-html-dir=reports/coverage ${NOSETESTS_ARGS} + - install_requirements test + - _run nosetests --with-xunit --xunit-file=reports/TEST-nosetests.xml --with-coverage --cover-erase --cover-xml --cover-xml-file=reports/coverage.xml --cover-html --cover-html-dir=reports/coverage ${NOSETESTS_ARGS} coverage: /^TOTAL.+?(\d+\%)$/ artifacts: name: "$CI_JOB_NAME artifacts from $CI_PROJECT_NAME on $CI_COMMIT_REF_SLUG" @@ -494,14 +517,14 @@ py-bandit: # force no dependencies dependencies: [] script: - - pip install -U bandit + - mkdir -p reports + - chmod o+rwx reports + - _pip install -U bandit - | - if ! bandit ${TRACE+--verbose} ${BANDIT_ARGS} + if ! _run bandit ${TRACE+--verbose} ${BANDIT_ARGS} then # failed: also generate JSON report - mkdir -p reports - chmod o+rwx reports - bandit ${TRACE+--verbose} --format json --output reports/bandit.json ${BANDIT_ARGS} + _run bandit ${TRACE+--verbose} --format json --output reports/bandit.json ${BANDIT_ARGS} exit 1 fi artifacts: @@ -531,14 +554,15 @@ py-safety: # force no dependencies dependencies: [] script: - - install_requirements + - mkdir -p reports + - chmod o+rwx reports + - install_requirements build - | - if ! pip freeze | safety check --stdin ${SAFETY_ARGS} + if ! _pip freeze | safety check --stdin ${SAFETY_ARGS} then # failed: also generate JSON report - mkdir -p reports - chmod o+rwx reports - pip freeze | safety check --stdin --json --output reports/safety.json ${SAFETY_ARGS} + + _pip freeze | safety check --stdin --json --output reports/safety.json ${SAFETY_ARGS} exit 1 fi artifacts: @@ -559,10 +583,8 @@ py-safety: - if: '$SAFETY_ENABLED == "true"' when: manual allow_failure: true - - ############################################################################################### -# pakage stage # +# package stage # ############################################################################################### # (on tag creation): create packages as artifacts @@ -570,7 +592,7 @@ py-package: extends: .python-base stage: package-build script: - - python setup.py sdist bdist_wheel + - _package artifacts: paths: - $PYTHON_PROJECT_DIR/dist/*.tar.gz @@ -580,6 +602,7 @@ py-package: - if: '$CI_COMMIT_TAG' - if: '$PYTHON_FORCE_PACKAGE == "true"' + ############################################################################################### # publish stage # ############################################################################################### @@ -591,27 +614,23 @@ py-publish: script: - assert_defined "$TWINE_USERNAME" 'Missing required env $TWINE_USERNAME' - assert_defined "$TWINE_PASSWORD" 'Missing required env $TWINE_PASSWORD' - - pip install -U twine setuptools - - pip list - - twine upload --verbose dist/*.tar.gz - - twine upload --verbose dist/*.whl + - _publish rules: # on tags with $TWINE_USERNAME set - if: '$TWINE_USERNAME && $CI_COMMIT_TAG' - # (on tag creation): generates the documentation py-docs: extends: .python-base stage: publish script: - install_doc_requirements - - pip install -U sphinx + - run_python -m pip install -U sphinx - cd ${DOCS_DIRECTORY} - make ${DOCS_MAKE_ARGS} artifacts: name: "$CI_JOB_NAME artifacts from $CI_PROJECT_NAME on $CI_COMMIT_REF_SLUG" paths: - - $DOCS_BUILD_DIR + - ${DOCS_DIRECTORY}/$DOCS_BUILD_DIR rules: # on tags with $DOCS_ENABLED set - if: '$DOCS_ENABLED == "true" && $CI_COMMIT_TAG' @@ -624,9 +643,7 @@ py-release: - git config --global user.email '$GITLAB_USER_EMAIL' - git config --global user.name '$GITLAB_USER_LOGIN' - git checkout -B $CI_BUILD_REF_NAME - - pip install --upgrade bumpversion - - release_args - - bumpversion ${bumpversion_args} + - _release - git_url_base=`echo ${CI_REPOSITORY_URL} | cut -d\@ -f2` - git push https://${RELEASE_USERNAME}:${RELEASE_ACCESS_TOKEN}@${git_url_base} --tags - git push https://${RELEASE_USERNAME}:${RELEASE_ACCESS_TOKEN}@${git_url_base} $CI_BUILD_REF_NAME