From 470bc2290ce5146e19d32ddf8c361cdaaba62103 Mon Sep 17 00:00:00 2001 From: Jon Azpiazu <jon.azpiazu@tecnalia.com> Date: Mon, 13 Jun 2022 15:43:34 +0200 Subject: [PATCH 01/10] New jobs to substitute the old Python check --- ci-templates/syntax-check.yml | 45 ++++++++++++++++++++++++++++------- 1 file changed, 36 insertions(+), 9 deletions(-) diff --git a/ci-templates/syntax-check.yml b/ci-templates/syntax-check.yml index 4ecbab1..3c413c2 100644 --- a/ci-templates/syntax-check.yml +++ b/ci-templates/syntax-check.yml @@ -16,16 +16,43 @@ clang_format: when: always ########################################################## -## Simple check for basic Python syntax errors; this is much less thorough than -## other options like pylint, but much faster and less strict +## Several Python checks ########################################################## -python_syntax: - image: alpine:3.15 # python2 is no longer available in alpine:3.16 - before_script: - - apk add --update python2 - stage: build - script: python -m compileall -q . - when: always +flake8: + stage: .post + image: python:3-alpine + before_script: + - pip3 install flake8 + script: + - flake8 --max-line-length 120 --statistics --show-source . + tags: + - docker + when: always + +flake8_extended: + stage: .post + image: python:3-alpine + before_script: + - pip3 install flake8 flake8-isort flake8-builtins flake8-eradicate flake8-functions-names flake8-return flake8-functions + script: + - flake8 --max-line-length 120 --statistics --show-source . + tags: + - docker + when: always + allow_failure: true + +python_security: + stage: .post + image: python:3-alpine + before_script: + - pip3 install bandit safety + script: + - bandit -r . || RET=1 + - safety check || RET=1 + - exit $RET + tags: + - docker + when: always ########################################################## ## Check for errors in bash/sh scripts -- GitLab From df025d18f1a5afc1abb1dc0e2945b19211ba370f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?I=C3=B1igo=20Moreno?= <inigo.moreno@tecnalia.com> Date: Fri, 17 Jun 2022 12:32:45 +0200 Subject: [PATCH 02/10] Make jobs python2 or python3 depending on distro --- ci-templates/auto-rules/no-default.yml | 20 ++++++++ ci-templates/core.yml | 1 + ci-templates/python.yml | 63 ++++++++++++++++++++++++++ ci-templates/syntax-check.yml | 39 ---------------- 4 files changed, 84 insertions(+), 39 deletions(-) create mode 100644 ci-templates/python.yml diff --git a/ci-templates/auto-rules/no-default.yml b/ci-templates/auto-rules/no-default.yml index 011021a..51bac7f 100644 --- a/ci-templates/auto-rules/no-default.yml +++ b/ci-templates/auto-rules/no-default.yml @@ -38,6 +38,26 @@ industrial_ci_noetic: - if: $DEFAULT_DISTRO == "noetic" - if: $BUILD_NOETIC + +.py3_template: + rules: + - if: $DEFAULT_DISTRO == "noetic" + when: always + - if: $BUILD_NOETIC + when: always + when: always + +.py2_template: + rules: + - if: $DEFAULT_DISTRO == "kinetic" + when: always + - if: $BUILD_KINETIC + when: always + - if: $DEFAULT_DISTRO == "melodic" + when: always + - if: $BUILD_MELODIC + when: always + ddeploy: extends: .ddeploy rules: diff --git a/ci-templates/core.yml b/ci-templates/core.yml index 8cacbee..2c198d1 100644 --- a/ci-templates/core.yml +++ b/ci-templates/core.yml @@ -24,6 +24,7 @@ before_script: include: - ci-templates/syntax-check.yml + - ci-templates/python.yml - ci-templates/industrial-ci-templates.yml default: diff --git a/ci-templates/python.yml b/ci-templates/python.yml new file mode 100644 index 0000000..a9d1082 --- /dev/null +++ b/ci-templates/python.yml @@ -0,0 +1,63 @@ + +.flake8_template: + stage: build + before_script: ${PYTHON_VERSION} -m pip -q install flake8 + script: ${PYTHON_VERSION} -m flake8 --max-line-length 120 --statistics --show-source . + + +.flake8_extended_template: + stage: .post + before_script: + - ${PYTHON_VERSION} -m pip install flake8 flake8-isort flake8-builtins + # The following flake8 extensions only work on python3 + - ${PYTHON_VERSION} -m pip install flake8-eradicate flake8-functions-names flake8-return flake8-functions || true + script: ${PYTHON_VERSION} -m flake8 --max-line-length 120 --statistics --show-source . + allow_failure: true + +.security_template: + stage: .post + before_script: + - ${PYTHON_VERSION} -m pip install bandit safety + script: + - bandit -r . || RET=1 + - safety check || RET=1 + - exit $RET + when: always + allow_failure: true + +.py3_template: + image: python:3-alpine + variables: + PYTHON_VERSION: python3 + when: never + +.py2_template: + image: python:2-alpine + variables: + PYTHON_VERSION: python2 + when: always + +py3-flake8: + extends: + - .flake8_template + - .py3_template + +py2-flake8: + extends: + - .flake8_template + - .py2_template + +py3-flake8_extended: + extends: + - .flake8_extended_template + - .py3_template + +py2-flake8_extended: + extends: + - .flake8_extended_template + - .py2_template + +py3-security: + extends: + - .security_template + - .py3_template \ No newline at end of file diff --git a/ci-templates/syntax-check.yml b/ci-templates/syntax-check.yml index 3c413c2..f834495 100644 --- a/ci-templates/syntax-check.yml +++ b/ci-templates/syntax-check.yml @@ -15,45 +15,6 @@ clang_format: CLANG_FORMAT_VERSION: "3.8" when: always -########################################################## -## Several Python checks -########################################################## -flake8: - stage: .post - image: python:3-alpine - before_script: - - pip3 install flake8 - script: - - flake8 --max-line-length 120 --statistics --show-source . - tags: - - docker - when: always - -flake8_extended: - stage: .post - image: python:3-alpine - before_script: - - pip3 install flake8 flake8-isort flake8-builtins flake8-eradicate flake8-functions-names flake8-return flake8-functions - script: - - flake8 --max-line-length 120 --statistics --show-source . - tags: - - docker - when: always - allow_failure: true - -python_security: - stage: .post - image: python:3-alpine - before_script: - - pip3 install bandit safety - script: - - bandit -r . || RET=1 - - safety check || RET=1 - - exit $RET - tags: - - docker - when: always - ########################################################## ## Check for errors in bash/sh scripts ## Uses https://github.com/koalaman/shellcheck -- GitLab From eb815306ceb460ccac9c4b65129013efe502ae4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?I=C3=B1igo=20Moreno?= <inigo.moreno@tecnalia.com> Date: Fri, 17 Jun 2022 12:34:16 +0200 Subject: [PATCH 03/10] Fix yaml-lint on python.yml --- ci-templates/python.yml | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/ci-templates/python.yml b/ci-templates/python.yml index a9d1082..e4d00ed 100644 --- a/ci-templates/python.yml +++ b/ci-templates/python.yml @@ -38,26 +38,26 @@ when: always py3-flake8: - extends: - - .flake8_template - - .py3_template + extends: + - .flake8_template + - .py3_template py2-flake8: - extends: - - .flake8_template - - .py2_template + extends: + - .flake8_template + - .py2_template py3-flake8_extended: - extends: - - .flake8_extended_template - - .py3_template + extends: + - .flake8_extended_template + - .py3_template py2-flake8_extended: - extends: - - .flake8_extended_template - - .py2_template + extends: + - .flake8_extended_template + - .py2_template py3-security: - extends: - - .security_template - - .py3_template \ No newline at end of file + extends: + - .security_template + - .py3_template -- GitLab From 8c0a161e4c1c1dc02ebe89076f038750e83f95fa Mon Sep 17 00:00:00 2001 From: Miguel Prada <miguel.prada@tecnalia.com> Date: Wed, 27 Sep 2023 11:30:33 +0200 Subject: [PATCH 04/10] Add 'humble' into distro-based python job selection logic --- ci-templates/auto-rules/no-default.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ci-templates/auto-rules/no-default.yml b/ci-templates/auto-rules/no-default.yml index f67a44b..6515130 100644 --- a/ci-templates/auto-rules/no-default.yml +++ b/ci-templates/auto-rules/no-default.yml @@ -55,6 +55,10 @@ industrial_ci_humble: when: always - if: $BUILD_NOETIC when: always + - if: $DEFAULT_DISTRO == "humble" + when: always + - if: $BUILD_HUMBLE + when: always when: always .py2_template: -- GitLab From fb49ba3869f2ea3077d3d62fec08ba85af400706 Mon Sep 17 00:00:00 2001 From: Miguel Prada <miguel.prada@tecnalia.com> Date: Mon, 18 Dec 2023 16:26:30 +0100 Subject: [PATCH 05/10] Add jobs testing the templates on top of test ROS repository --- .gitlab-ci.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3d2cc4d..8ca6df2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -20,3 +20,20 @@ deploy_devcontainer: expire_in: 1 year tags: - docker + +.simple_ros_test: + variables: + TEMPLATES_PIPELINE_REF: $CI_COMMIT_REF_NAME + trigger: + project: tecnalia_robotics/ci_test_repos/simple_ros_test + strategy: depend + +simple_ros_melodic_test: + extends: .simple_ros_test + variables: + TEMPLATE_TO_USE: ci-templates/auto-rules/melodic-default.yml + +simple_ros_noetic_test: + extends: .simple_ros_test + variables: + TEMPLATE_TO_USE: ci-templates/auto-rules/noetic-default.yml -- GitLab From 6b93b65706509705218f6bd46bc0e22b9b86b546 Mon Sep 17 00:00:00 2001 From: Miguel Prada <miguel.prada@tecnalia.com> Date: Mon, 18 Dec 2023 16:37:39 +0100 Subject: [PATCH 06/10] Add job testing usage of 'core.yml' --- .gitlab-ci.yml | 5 +++++ test/core_with_noetic.yml | 11 +++++++++++ 2 files changed, 16 insertions(+) create mode 100644 test/core_with_noetic.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8ca6df2..ffe0a5a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -37,3 +37,8 @@ simple_ros_noetic_test: extends: .simple_ros_test variables: TEMPLATE_TO_USE: ci-templates/auto-rules/noetic-default.yml + +simple_ros_core_with_noetic_test: + extends: .simple_ros_test + variables: + TEMPLATE_TO_USE: test/core_with_noetic.yml diff --git a/test/core_with_noetic.yml b/test/core_with_noetic.yml new file mode 100644 index 0000000..ce063ed --- /dev/null +++ b/test/core_with_noetic.yml @@ -0,0 +1,11 @@ +include: ci-templates/core.yml + +industrial_ci_melodic: + extends: .industrial_ci + variables: + ROS_DISTRO: melodic + +industrial_ci_noetic: + extends: .industrial_ci + variables: + ROS_DISTRO: noetic -- GitLab From b713a597c99bc8c1ff5b66642cc41a641dd62ba3 Mon Sep 17 00:00:00 2001 From: Miguel Prada <miguel.prada@tecnalia.com> Date: Mon, 18 Dec 2023 17:04:56 +0100 Subject: [PATCH 07/10] Modify python job logic Create hidden templates in core and make conditionally make them visible in auto-rules --- ci-templates/auto-rules/no-default.yml | 6 ++++-- ci-templates/python.yml | 13 +++++-------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/ci-templates/auto-rules/no-default.yml b/ci-templates/auto-rules/no-default.yml index 6515130..6171018 100644 --- a/ci-templates/auto-rules/no-default.yml +++ b/ci-templates/auto-rules/no-default.yml @@ -49,7 +49,8 @@ industrial_ci_humble: - if: $DEFAULT_DISTRO == "humble" - if: $BUILD_HUMBLE -.py3_template: +py3_template: + extends: .py3_template rules: - if: $DEFAULT_DISTRO == "noetic" when: always @@ -61,7 +62,8 @@ industrial_ci_humble: when: always when: always -.py2_template: +py2_template: + extends: .py2_template rules: - if: $DEFAULT_DISTRO == "kinetic" when: always diff --git a/ci-templates/python.yml b/ci-templates/python.yml index e4d00ed..61d476a 100644 --- a/ci-templates/python.yml +++ b/ci-templates/python.yml @@ -22,42 +22,39 @@ - bandit -r . || RET=1 - safety check || RET=1 - exit $RET - when: always allow_failure: true .py3_template: image: python:3-alpine variables: PYTHON_VERSION: python3 - when: never .py2_template: image: python:2-alpine variables: PYTHON_VERSION: python2 - when: always -py3-flake8: +.py3-flake8: extends: - .flake8_template - .py3_template -py2-flake8: +.py2-flake8: extends: - .flake8_template - .py2_template -py3-flake8_extended: +.py3-flake8_extended: extends: - .flake8_extended_template - .py3_template -py2-flake8_extended: +.py2-flake8_extended: extends: - .flake8_extended_template - .py2_template -py3-security: +.py3-security: extends: - .security_template - .py3_template -- GitLab From 30d754f0f8596e47f25c45920d1add885cdfba4d Mon Sep 17 00:00:00 2001 From: Miguel Prada <miguel.prada@tecnalia.com> Date: Mon, 18 Dec 2023 17:10:19 +0100 Subject: [PATCH 08/10] Fix errors in python auto-rules logic --- ci-templates/auto-rules/no-default.yml | 32 ++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/ci-templates/auto-rules/no-default.yml b/ci-templates/auto-rules/no-default.yml index 6171018..4921c97 100644 --- a/ci-templates/auto-rules/no-default.yml +++ b/ci-templates/auto-rules/no-default.yml @@ -49,8 +49,7 @@ industrial_ci_humble: - if: $DEFAULT_DISTRO == "humble" - if: $BUILD_HUMBLE -py3_template: - extends: .py3_template +.py3_rules: rules: - if: $DEFAULT_DISTRO == "noetic" when: always @@ -60,10 +59,23 @@ py3_template: when: always - if: $BUILD_HUMBLE when: always - when: always -py2_template: - extends: .py2_template +py3-flake8: + extends: + - .py3_rules + - .py3-flake8 + +py3-flake8_extended: + extends: + - .py3_rules + - .py3-flake8_extended + +py3-security: + extends: + - .py3_rules + - .py3-security + +.py2_rules: rules: - if: $DEFAULT_DISTRO == "kinetic" when: always @@ -74,6 +86,16 @@ py2_template: - if: $BUILD_MELODIC when: always +py2-flake8: + extends: + - .py2_rules + - .py2-flake8 + +py2-flake8_extended: + extends: + - .py2_rules + - .py2-flake8_extended + ddeploy: extends: .ddeploy rules: -- GitLab From 3c683127607837734ddda55ba8dee06c8271b2a7 Mon Sep 17 00:00:00 2001 From: Miguel Prada <miguel.prada@tecnalia.com> Date: Mon, 18 Dec 2023 17:13:33 +0100 Subject: [PATCH 09/10] Add job testing two distros using auto-rules --- .gitlab-ci.yml | 5 +++++ test/auto_rules_melodic_noetic.yml | 5 +++++ 2 files changed, 10 insertions(+) create mode 100644 test/auto_rules_melodic_noetic.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ffe0a5a..e64ddbe 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -42,3 +42,8 @@ simple_ros_core_with_noetic_test: extends: .simple_ros_test variables: TEMPLATE_TO_USE: test/core_with_noetic.yml + +simple_ros_auto_rules_melodic_noetic_test: + extends: .simple_ros_test + variables: + TEMPLATE_TO_USE: test/auto_rules_melodic_noetic.yml diff --git a/test/auto_rules_melodic_noetic.yml b/test/auto_rules_melodic_noetic.yml new file mode 100644 index 0000000..a3301c4 --- /dev/null +++ b/test/auto_rules_melodic_noetic.yml @@ -0,0 +1,5 @@ +include: ci-templates/auto-rules/no-default.yml + +variables: + BUILD_MELODIC: true + BUILD_NOETIC: true -- GitLab From a8ff0415cfc2bdceb8e259d168c6e525abaf371e Mon Sep 17 00:00:00 2001 From: Miguel Prada <miguel.prada@tecnalia.com> Date: Mon, 18 Dec 2023 17:15:25 +0100 Subject: [PATCH 10/10] Use 1 instead of true in CI template --- test/auto_rules_melodic_noetic.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/auto_rules_melodic_noetic.yml b/test/auto_rules_melodic_noetic.yml index a3301c4..26e5143 100644 --- a/test/auto_rules_melodic_noetic.yml +++ b/test/auto_rules_melodic_noetic.yml @@ -1,5 +1,5 @@ include: ci-templates/auto-rules/no-default.yml variables: - BUILD_MELODIC: true - BUILD_NOETIC: true + BUILD_MELODIC: 1 + BUILD_NOETIC: 1 -- GitLab