From 470bc2290ce5146e19d32ddf8c361cdaaba62103 Mon Sep 17 00:00:00 2001
From: Jon Azpiazu <jon.azpiazu@tecnalia.com>
Date: Mon, 13 Jun 2022 15:43:34 +0200
Subject: [PATCH 01/10] New jobs to substitute the old Python check
---
ci-templates/syntax-check.yml | 45 ++++++++++++++++++++++++++++-------
1 file changed, 36 insertions(+), 9 deletions(-)
diff --git a/ci-templates/syntax-check.yml b/ci-templates/syntax-check.yml
index 4ecbab1..3c413c2 100644
--- a/ci-templates/syntax-check.yml
+++ b/ci-templates/syntax-check.yml
@@ -16,16 +16,43 @@ clang_format:
when: always
##########################################################
-## Simple check for basic Python syntax errors; this is much less thorough than
-## other options like pylint, but much faster and less strict
+## Several Python checks
##########################################################
-python_syntax:
- image: alpine:3.15 # python2 is no longer available in alpine:3.16
- before_script:
- - apk add --update python2
- stage: build
- script: python -m compileall -q .
- when: always
+flake8:
+ stage: .post
+ image: python:3-alpine
+ before_script:
+ - pip3 install flake8
+ script:
+ - flake8 --max-line-length 120 --statistics --show-source .
+ tags:
+ - docker
+ when: always
+
+flake8_extended:
+ stage: .post
+ image: python:3-alpine
+ before_script:
+ - pip3 install flake8 flake8-isort flake8-builtins flake8-eradicate flake8-functions-names flake8-return flake8-functions
+ script:
+ - flake8 --max-line-length 120 --statistics --show-source .
+ tags:
+ - docker
+ when: always
+ allow_failure: true
+
+python_security:
+ stage: .post
+ image: python:3-alpine
+ before_script:
+ - pip3 install bandit safety
+ script:
+ - bandit -r . || RET=1
+ - safety check || RET=1
+ - exit $RET
+ tags:
+ - docker
+ when: always
##########################################################
## Check for errors in bash/sh scripts
--
GitLab
From df025d18f1a5afc1abb1dc0e2945b19211ba370f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?I=C3=B1igo=20Moreno?= <inigo.moreno@tecnalia.com>
Date: Fri, 17 Jun 2022 12:32:45 +0200
Subject: [PATCH 02/10] Make jobs python2 or python3 depending on distro
---
ci-templates/auto-rules/no-default.yml | 20 ++++++++
ci-templates/core.yml | 1 +
ci-templates/python.yml | 63 ++++++++++++++++++++++++++
ci-templates/syntax-check.yml | 39 ----------------
4 files changed, 84 insertions(+), 39 deletions(-)
create mode 100644 ci-templates/python.yml
diff --git a/ci-templates/auto-rules/no-default.yml b/ci-templates/auto-rules/no-default.yml
index 011021a..51bac7f 100644
--- a/ci-templates/auto-rules/no-default.yml
+++ b/ci-templates/auto-rules/no-default.yml
@@ -38,6 +38,26 @@ industrial_ci_noetic:
- if: $DEFAULT_DISTRO == "noetic"
- if: $BUILD_NOETIC
+
+.py3_template:
+ rules:
+ - if: $DEFAULT_DISTRO == "noetic"
+ when: always
+ - if: $BUILD_NOETIC
+ when: always
+ when: always
+
+.py2_template:
+ rules:
+ - if: $DEFAULT_DISTRO == "kinetic"
+ when: always
+ - if: $BUILD_KINETIC
+ when: always
+ - if: $DEFAULT_DISTRO == "melodic"
+ when: always
+ - if: $BUILD_MELODIC
+ when: always
+
ddeploy:
extends: .ddeploy
rules:
diff --git a/ci-templates/core.yml b/ci-templates/core.yml
index 8cacbee..2c198d1 100644
--- a/ci-templates/core.yml
+++ b/ci-templates/core.yml
@@ -24,6 +24,7 @@ before_script:
include:
- ci-templates/syntax-check.yml
+ - ci-templates/python.yml
- ci-templates/industrial-ci-templates.yml
default:
diff --git a/ci-templates/python.yml b/ci-templates/python.yml
new file mode 100644
index 0000000..a9d1082
--- /dev/null
+++ b/ci-templates/python.yml
@@ -0,0 +1,63 @@
+
+.flake8_template:
+ stage: build
+ before_script: ${PYTHON_VERSION} -m pip -q install flake8
+ script: ${PYTHON_VERSION} -m flake8 --max-line-length 120 --statistics --show-source .
+
+
+.flake8_extended_template:
+ stage: .post
+ before_script:
+ - ${PYTHON_VERSION} -m pip install flake8 flake8-isort flake8-builtins
+ # The following flake8 extensions only work on python3
+ - ${PYTHON_VERSION} -m pip install flake8-eradicate flake8-functions-names flake8-return flake8-functions || true
+ script: ${PYTHON_VERSION} -m flake8 --max-line-length 120 --statistics --show-source .
+ allow_failure: true
+
+.security_template:
+ stage: .post
+ before_script:
+ - ${PYTHON_VERSION} -m pip install bandit safety
+ script:
+ - bandit -r . || RET=1
+ - safety check || RET=1
+ - exit $RET
+ when: always
+ allow_failure: true
+
+.py3_template:
+ image: python:3-alpine
+ variables:
+ PYTHON_VERSION: python3
+ when: never
+
+.py2_template:
+ image: python:2-alpine
+ variables:
+ PYTHON_VERSION: python2
+ when: always
+
+py3-flake8:
+ extends:
+ - .flake8_template
+ - .py3_template
+
+py2-flake8:
+ extends:
+ - .flake8_template
+ - .py2_template
+
+py3-flake8_extended:
+ extends:
+ - .flake8_extended_template
+ - .py3_template
+
+py2-flake8_extended:
+ extends:
+ - .flake8_extended_template
+ - .py2_template
+
+py3-security:
+ extends:
+ - .security_template
+ - .py3_template
\ No newline at end of file
diff --git a/ci-templates/syntax-check.yml b/ci-templates/syntax-check.yml
index 3c413c2..f834495 100644
--- a/ci-templates/syntax-check.yml
+++ b/ci-templates/syntax-check.yml
@@ -15,45 +15,6 @@ clang_format:
CLANG_FORMAT_VERSION: "3.8"
when: always
-##########################################################
-## Several Python checks
-##########################################################
-flake8:
- stage: .post
- image: python:3-alpine
- before_script:
- - pip3 install flake8
- script:
- - flake8 --max-line-length 120 --statistics --show-source .
- tags:
- - docker
- when: always
-
-flake8_extended:
- stage: .post
- image: python:3-alpine
- before_script:
- - pip3 install flake8 flake8-isort flake8-builtins flake8-eradicate flake8-functions-names flake8-return flake8-functions
- script:
- - flake8 --max-line-length 120 --statistics --show-source .
- tags:
- - docker
- when: always
- allow_failure: true
-
-python_security:
- stage: .post
- image: python:3-alpine
- before_script:
- - pip3 install bandit safety
- script:
- - bandit -r . || RET=1
- - safety check || RET=1
- - exit $RET
- tags:
- - docker
- when: always
-
##########################################################
## Check for errors in bash/sh scripts
## Uses https://github.com/koalaman/shellcheck
--
GitLab
From eb815306ceb460ccac9c4b65129013efe502ae4c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?I=C3=B1igo=20Moreno?= <inigo.moreno@tecnalia.com>
Date: Fri, 17 Jun 2022 12:34:16 +0200
Subject: [PATCH 03/10] Fix yaml-lint on python.yml
---
ci-templates/python.yml | 30 +++++++++++++++---------------
1 file changed, 15 insertions(+), 15 deletions(-)
diff --git a/ci-templates/python.yml b/ci-templates/python.yml
index a9d1082..e4d00ed 100644
--- a/ci-templates/python.yml
+++ b/ci-templates/python.yml
@@ -38,26 +38,26 @@
when: always
py3-flake8:
- extends:
- - .flake8_template
- - .py3_template
+ extends:
+ - .flake8_template
+ - .py3_template
py2-flake8:
- extends:
- - .flake8_template
- - .py2_template
+ extends:
+ - .flake8_template
+ - .py2_template
py3-flake8_extended:
- extends:
- - .flake8_extended_template
- - .py3_template
+ extends:
+ - .flake8_extended_template
+ - .py3_template
py2-flake8_extended:
- extends:
- - .flake8_extended_template
- - .py2_template
+ extends:
+ - .flake8_extended_template
+ - .py2_template
py3-security:
- extends:
- - .security_template
- - .py3_template
\ No newline at end of file
+ extends:
+ - .security_template
+ - .py3_template
--
GitLab
From 8c0a161e4c1c1dc02ebe89076f038750e83f95fa Mon Sep 17 00:00:00 2001
From: Miguel Prada <miguel.prada@tecnalia.com>
Date: Wed, 27 Sep 2023 11:30:33 +0200
Subject: [PATCH 04/10] Add 'humble' into distro-based python job selection
logic
---
ci-templates/auto-rules/no-default.yml | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/ci-templates/auto-rules/no-default.yml b/ci-templates/auto-rules/no-default.yml
index f67a44b..6515130 100644
--- a/ci-templates/auto-rules/no-default.yml
+++ b/ci-templates/auto-rules/no-default.yml
@@ -55,6 +55,10 @@ industrial_ci_humble:
when: always
- if: $BUILD_NOETIC
when: always
+ - if: $DEFAULT_DISTRO == "humble"
+ when: always
+ - if: $BUILD_HUMBLE
+ when: always
when: always
.py2_template:
--
GitLab
From fb49ba3869f2ea3077d3d62fec08ba85af400706 Mon Sep 17 00:00:00 2001
From: Miguel Prada <miguel.prada@tecnalia.com>
Date: Mon, 18 Dec 2023 16:26:30 +0100
Subject: [PATCH 05/10] Add jobs testing the templates on top of test ROS
repository
---
.gitlab-ci.yml | 17 +++++++++++++++++
1 file changed, 17 insertions(+)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 3d2cc4d..8ca6df2 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -20,3 +20,20 @@ deploy_devcontainer:
expire_in: 1 year
tags:
- docker
+
+.simple_ros_test:
+ variables:
+ TEMPLATES_PIPELINE_REF: $CI_COMMIT_REF_NAME
+ trigger:
+ project: tecnalia_robotics/ci_test_repos/simple_ros_test
+ strategy: depend
+
+simple_ros_melodic_test:
+ extends: .simple_ros_test
+ variables:
+ TEMPLATE_TO_USE: ci-templates/auto-rules/melodic-default.yml
+
+simple_ros_noetic_test:
+ extends: .simple_ros_test
+ variables:
+ TEMPLATE_TO_USE: ci-templates/auto-rules/noetic-default.yml
--
GitLab
From 6b93b65706509705218f6bd46bc0e22b9b86b546 Mon Sep 17 00:00:00 2001
From: Miguel Prada <miguel.prada@tecnalia.com>
Date: Mon, 18 Dec 2023 16:37:39 +0100
Subject: [PATCH 06/10] Add job testing usage of 'core.yml'
---
.gitlab-ci.yml | 5 +++++
test/core_with_noetic.yml | 11 +++++++++++
2 files changed, 16 insertions(+)
create mode 100644 test/core_with_noetic.yml
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 8ca6df2..ffe0a5a 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -37,3 +37,8 @@ simple_ros_noetic_test:
extends: .simple_ros_test
variables:
TEMPLATE_TO_USE: ci-templates/auto-rules/noetic-default.yml
+
+simple_ros_core_with_noetic_test:
+ extends: .simple_ros_test
+ variables:
+ TEMPLATE_TO_USE: test/core_with_noetic.yml
diff --git a/test/core_with_noetic.yml b/test/core_with_noetic.yml
new file mode 100644
index 0000000..ce063ed
--- /dev/null
+++ b/test/core_with_noetic.yml
@@ -0,0 +1,11 @@
+include: ci-templates/core.yml
+
+industrial_ci_melodic:
+ extends: .industrial_ci
+ variables:
+ ROS_DISTRO: melodic
+
+industrial_ci_noetic:
+ extends: .industrial_ci
+ variables:
+ ROS_DISTRO: noetic
--
GitLab
From b713a597c99bc8c1ff5b66642cc41a641dd62ba3 Mon Sep 17 00:00:00 2001
From: Miguel Prada <miguel.prada@tecnalia.com>
Date: Mon, 18 Dec 2023 17:04:56 +0100
Subject: [PATCH 07/10] Modify python job logic
Create hidden templates in core and make conditionally make them visible in auto-rules
---
ci-templates/auto-rules/no-default.yml | 6 ++++--
ci-templates/python.yml | 13 +++++--------
2 files changed, 9 insertions(+), 10 deletions(-)
diff --git a/ci-templates/auto-rules/no-default.yml b/ci-templates/auto-rules/no-default.yml
index 6515130..6171018 100644
--- a/ci-templates/auto-rules/no-default.yml
+++ b/ci-templates/auto-rules/no-default.yml
@@ -49,7 +49,8 @@ industrial_ci_humble:
- if: $DEFAULT_DISTRO == "humble"
- if: $BUILD_HUMBLE
-.py3_template:
+py3_template:
+ extends: .py3_template
rules:
- if: $DEFAULT_DISTRO == "noetic"
when: always
@@ -61,7 +62,8 @@ industrial_ci_humble:
when: always
when: always
-.py2_template:
+py2_template:
+ extends: .py2_template
rules:
- if: $DEFAULT_DISTRO == "kinetic"
when: always
diff --git a/ci-templates/python.yml b/ci-templates/python.yml
index e4d00ed..61d476a 100644
--- a/ci-templates/python.yml
+++ b/ci-templates/python.yml
@@ -22,42 +22,39 @@
- bandit -r . || RET=1
- safety check || RET=1
- exit $RET
- when: always
allow_failure: true
.py3_template:
image: python:3-alpine
variables:
PYTHON_VERSION: python3
- when: never
.py2_template:
image: python:2-alpine
variables:
PYTHON_VERSION: python2
- when: always
-py3-flake8:
+.py3-flake8:
extends:
- .flake8_template
- .py3_template
-py2-flake8:
+.py2-flake8:
extends:
- .flake8_template
- .py2_template
-py3-flake8_extended:
+.py3-flake8_extended:
extends:
- .flake8_extended_template
- .py3_template
-py2-flake8_extended:
+.py2-flake8_extended:
extends:
- .flake8_extended_template
- .py2_template
-py3-security:
+.py3-security:
extends:
- .security_template
- .py3_template
--
GitLab
From 30d754f0f8596e47f25c45920d1add885cdfba4d Mon Sep 17 00:00:00 2001
From: Miguel Prada <miguel.prada@tecnalia.com>
Date: Mon, 18 Dec 2023 17:10:19 +0100
Subject: [PATCH 08/10] Fix errors in python auto-rules logic
---
ci-templates/auto-rules/no-default.yml | 32 ++++++++++++++++++++++----
1 file changed, 27 insertions(+), 5 deletions(-)
diff --git a/ci-templates/auto-rules/no-default.yml b/ci-templates/auto-rules/no-default.yml
index 6171018..4921c97 100644
--- a/ci-templates/auto-rules/no-default.yml
+++ b/ci-templates/auto-rules/no-default.yml
@@ -49,8 +49,7 @@ industrial_ci_humble:
- if: $DEFAULT_DISTRO == "humble"
- if: $BUILD_HUMBLE
-py3_template:
- extends: .py3_template
+.py3_rules:
rules:
- if: $DEFAULT_DISTRO == "noetic"
when: always
@@ -60,10 +59,23 @@ py3_template:
when: always
- if: $BUILD_HUMBLE
when: always
- when: always
-py2_template:
- extends: .py2_template
+py3-flake8:
+ extends:
+ - .py3_rules
+ - .py3-flake8
+
+py3-flake8_extended:
+ extends:
+ - .py3_rules
+ - .py3-flake8_extended
+
+py3-security:
+ extends:
+ - .py3_rules
+ - .py3-security
+
+.py2_rules:
rules:
- if: $DEFAULT_DISTRO == "kinetic"
when: always
@@ -74,6 +86,16 @@ py2_template:
- if: $BUILD_MELODIC
when: always
+py2-flake8:
+ extends:
+ - .py2_rules
+ - .py2-flake8
+
+py2-flake8_extended:
+ extends:
+ - .py2_rules
+ - .py2-flake8_extended
+
ddeploy:
extends: .ddeploy
rules:
--
GitLab
From 3c683127607837734ddda55ba8dee06c8271b2a7 Mon Sep 17 00:00:00 2001
From: Miguel Prada <miguel.prada@tecnalia.com>
Date: Mon, 18 Dec 2023 17:13:33 +0100
Subject: [PATCH 09/10] Add job testing two distros using auto-rules
---
.gitlab-ci.yml | 5 +++++
test/auto_rules_melodic_noetic.yml | 5 +++++
2 files changed, 10 insertions(+)
create mode 100644 test/auto_rules_melodic_noetic.yml
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index ffe0a5a..e64ddbe 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -42,3 +42,8 @@ simple_ros_core_with_noetic_test:
extends: .simple_ros_test
variables:
TEMPLATE_TO_USE: test/core_with_noetic.yml
+
+simple_ros_auto_rules_melodic_noetic_test:
+ extends: .simple_ros_test
+ variables:
+ TEMPLATE_TO_USE: test/auto_rules_melodic_noetic.yml
diff --git a/test/auto_rules_melodic_noetic.yml b/test/auto_rules_melodic_noetic.yml
new file mode 100644
index 0000000..a3301c4
--- /dev/null
+++ b/test/auto_rules_melodic_noetic.yml
@@ -0,0 +1,5 @@
+include: ci-templates/auto-rules/no-default.yml
+
+variables:
+ BUILD_MELODIC: true
+ BUILD_NOETIC: true
--
GitLab
From a8ff0415cfc2bdceb8e259d168c6e525abaf371e Mon Sep 17 00:00:00 2001
From: Miguel Prada <miguel.prada@tecnalia.com>
Date: Mon, 18 Dec 2023 17:15:25 +0100
Subject: [PATCH 10/10] Use 1 instead of true in CI template
---
test/auto_rules_melodic_noetic.yml | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/test/auto_rules_melodic_noetic.yml b/test/auto_rules_melodic_noetic.yml
index a3301c4..26e5143 100644
--- a/test/auto_rules_melodic_noetic.yml
+++ b/test/auto_rules_melodic_noetic.yml
@@ -1,5 +1,5 @@
include: ci-templates/auto-rules/no-default.yml
variables:
- BUILD_MELODIC: true
- BUILD_NOETIC: true
+ BUILD_MELODIC: 1
+ BUILD_NOETIC: 1
--
GitLab