From 470bc2290ce5146e19d32ddf8c361cdaaba62103 Mon Sep 17 00:00:00 2001
From: Jon Azpiazu <jon.azpiazu@tecnalia.com>
Date: Mon, 13 Jun 2022 15:43:34 +0200
Subject: [PATCH 01/10] New jobs to substitute the old Python check

---
 ci-templates/syntax-check.yml | 45 ++++++++++++++++++++++++++++-------
 1 file changed, 36 insertions(+), 9 deletions(-)

diff --git a/ci-templates/syntax-check.yml b/ci-templates/syntax-check.yml
index 4ecbab1..3c413c2 100644
--- a/ci-templates/syntax-check.yml
+++ b/ci-templates/syntax-check.yml
@@ -16,16 +16,43 @@ clang_format:
   when: always
 
 ##########################################################
-## Simple check for basic Python syntax errors; this is much less thorough than
-## other options like pylint, but much faster and less strict
+## Several Python checks
 ##########################################################
-python_syntax:
-  image: alpine:3.15 # python2 is no longer available in alpine:3.16
-  before_script:
-    - apk add --update python2
-  stage: build
-  script: python -m compileall -q .
-  when: always
+flake8:
+    stage: .post
+    image: python:3-alpine
+    before_script:
+        - pip3 install flake8
+    script:
+        - flake8 --max-line-length 120 --statistics --show-source .
+    tags:
+        - docker
+    when: always
+
+flake8_extended:
+    stage: .post
+    image: python:3-alpine
+    before_script:
+        - pip3 install flake8 flake8-isort flake8-builtins flake8-eradicate flake8-functions-names flake8-return flake8-functions
+    script:
+        - flake8 --max-line-length 120 --statistics --show-source .
+    tags:
+        - docker
+    when: always
+    allow_failure: true
+
+python_security:
+    stage: .post
+    image: python:3-alpine
+    before_script:
+        - pip3 install bandit safety
+    script:
+        - bandit -r . || RET=1
+        - safety check || RET=1
+        - exit $RET
+    tags:
+        - docker
+    when: always
 
 ##########################################################
 ## Check for errors in bash/sh scripts
-- 
GitLab


From df025d18f1a5afc1abb1dc0e2945b19211ba370f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?I=C3=B1igo=20Moreno?= <inigo.moreno@tecnalia.com>
Date: Fri, 17 Jun 2022 12:32:45 +0200
Subject: [PATCH 02/10] Make jobs python2 or python3 depending on distro

---
 ci-templates/auto-rules/no-default.yml | 20 ++++++++
 ci-templates/core.yml                  |  1 +
 ci-templates/python.yml                | 63 ++++++++++++++++++++++++++
 ci-templates/syntax-check.yml          | 39 ----------------
 4 files changed, 84 insertions(+), 39 deletions(-)
 create mode 100644 ci-templates/python.yml

diff --git a/ci-templates/auto-rules/no-default.yml b/ci-templates/auto-rules/no-default.yml
index 011021a..51bac7f 100644
--- a/ci-templates/auto-rules/no-default.yml
+++ b/ci-templates/auto-rules/no-default.yml
@@ -38,6 +38,26 @@ industrial_ci_noetic:
     - if: $DEFAULT_DISTRO == "noetic"
     - if: $BUILD_NOETIC
 
+
+.py3_template:
+  rules:
+    - if: $DEFAULT_DISTRO == "noetic"
+      when: always
+    - if: $BUILD_NOETIC
+      when: always
+  when: always
+
+.py2_template:
+  rules:
+    - if: $DEFAULT_DISTRO == "kinetic"
+      when: always
+    - if: $BUILD_KINETIC
+      when: always
+    - if: $DEFAULT_DISTRO == "melodic"
+      when: always
+    - if: $BUILD_MELODIC
+      when: always
+
 ddeploy:
   extends: .ddeploy
   rules:
diff --git a/ci-templates/core.yml b/ci-templates/core.yml
index 8cacbee..2c198d1 100644
--- a/ci-templates/core.yml
+++ b/ci-templates/core.yml
@@ -24,6 +24,7 @@ before_script:
 
 include:
   - ci-templates/syntax-check.yml
+  - ci-templates/python.yml
   - ci-templates/industrial-ci-templates.yml
 
 default:
diff --git a/ci-templates/python.yml b/ci-templates/python.yml
new file mode 100644
index 0000000..a9d1082
--- /dev/null
+++ b/ci-templates/python.yml
@@ -0,0 +1,63 @@
+
+.flake8_template:
+  stage: build
+  before_script: ${PYTHON_VERSION} -m pip -q install flake8
+  script: ${PYTHON_VERSION} -m flake8 --max-line-length 120 --statistics --show-source .
+
+
+.flake8_extended_template:
+  stage: .post
+  before_script:
+    - ${PYTHON_VERSION} -m pip install flake8 flake8-isort flake8-builtins
+    # The following flake8 extensions only work on python3
+    - ${PYTHON_VERSION} -m pip install flake8-eradicate flake8-functions-names flake8-return flake8-functions || true
+  script: ${PYTHON_VERSION} -m flake8 --max-line-length 120 --statistics --show-source .
+  allow_failure: true
+
+.security_template:
+  stage: .post
+  before_script:
+    - ${PYTHON_VERSION} -m pip install bandit safety
+  script:
+    - bandit -r . || RET=1
+    - safety check || RET=1
+    - exit $RET
+  when: always
+  allow_failure: true
+
+.py3_template:
+  image: python:3-alpine
+  variables:
+    PYTHON_VERSION: python3
+  when: never
+
+.py2_template:
+  image: python:2-alpine
+  variables:
+    PYTHON_VERSION: python2
+  when: always
+
+py3-flake8:
+    extends:
+      - .flake8_template
+      - .py3_template
+
+py2-flake8:
+    extends:
+      - .flake8_template
+      - .py2_template
+
+py3-flake8_extended:
+    extends:
+      - .flake8_extended_template
+      - .py3_template
+
+py2-flake8_extended:
+    extends:
+      - .flake8_extended_template
+      - .py2_template
+
+py3-security:
+    extends:
+      - .security_template
+      - .py3_template
\ No newline at end of file
diff --git a/ci-templates/syntax-check.yml b/ci-templates/syntax-check.yml
index 3c413c2..f834495 100644
--- a/ci-templates/syntax-check.yml
+++ b/ci-templates/syntax-check.yml
@@ -15,45 +15,6 @@ clang_format:
     CLANG_FORMAT_VERSION: "3.8"
   when: always
 
-##########################################################
-## Several Python checks
-##########################################################
-flake8:
-    stage: .post
-    image: python:3-alpine
-    before_script:
-        - pip3 install flake8
-    script:
-        - flake8 --max-line-length 120 --statistics --show-source .
-    tags:
-        - docker
-    when: always
-
-flake8_extended:
-    stage: .post
-    image: python:3-alpine
-    before_script:
-        - pip3 install flake8 flake8-isort flake8-builtins flake8-eradicate flake8-functions-names flake8-return flake8-functions
-    script:
-        - flake8 --max-line-length 120 --statistics --show-source .
-    tags:
-        - docker
-    when: always
-    allow_failure: true
-
-python_security:
-    stage: .post
-    image: python:3-alpine
-    before_script:
-        - pip3 install bandit safety
-    script:
-        - bandit -r . || RET=1
-        - safety check || RET=1
-        - exit $RET
-    tags:
-        - docker
-    when: always
-
 ##########################################################
 ## Check for errors in bash/sh scripts
 ## Uses https://github.com/koalaman/shellcheck
-- 
GitLab


From eb815306ceb460ccac9c4b65129013efe502ae4c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?I=C3=B1igo=20Moreno?= <inigo.moreno@tecnalia.com>
Date: Fri, 17 Jun 2022 12:34:16 +0200
Subject: [PATCH 03/10] Fix yaml-lint on python.yml

---
 ci-templates/python.yml | 30 +++++++++++++++---------------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/ci-templates/python.yml b/ci-templates/python.yml
index a9d1082..e4d00ed 100644
--- a/ci-templates/python.yml
+++ b/ci-templates/python.yml
@@ -38,26 +38,26 @@
   when: always
 
 py3-flake8:
-    extends:
-      - .flake8_template
-      - .py3_template
+  extends:
+    - .flake8_template
+    - .py3_template
 
 py2-flake8:
-    extends:
-      - .flake8_template
-      - .py2_template
+  extends:
+    - .flake8_template
+    - .py2_template
 
 py3-flake8_extended:
-    extends:
-      - .flake8_extended_template
-      - .py3_template
+  extends:
+    - .flake8_extended_template
+    - .py3_template
 
 py2-flake8_extended:
-    extends:
-      - .flake8_extended_template
-      - .py2_template
+  extends:
+    - .flake8_extended_template
+    - .py2_template
 
 py3-security:
-    extends:
-      - .security_template
-      - .py3_template
\ No newline at end of file
+  extends:
+    - .security_template
+    - .py3_template
-- 
GitLab


From 8c0a161e4c1c1dc02ebe89076f038750e83f95fa Mon Sep 17 00:00:00 2001
From: Miguel Prada <miguel.prada@tecnalia.com>
Date: Wed, 27 Sep 2023 11:30:33 +0200
Subject: [PATCH 04/10] Add 'humble' into distro-based python job selection
 logic

---
 ci-templates/auto-rules/no-default.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/ci-templates/auto-rules/no-default.yml b/ci-templates/auto-rules/no-default.yml
index f67a44b..6515130 100644
--- a/ci-templates/auto-rules/no-default.yml
+++ b/ci-templates/auto-rules/no-default.yml
@@ -55,6 +55,10 @@ industrial_ci_humble:
       when: always
     - if: $BUILD_NOETIC
       when: always
+    - if: $DEFAULT_DISTRO == "humble"
+      when: always
+    - if: $BUILD_HUMBLE
+      when: always
   when: always
 
 .py2_template:
-- 
GitLab


From fb49ba3869f2ea3077d3d62fec08ba85af400706 Mon Sep 17 00:00:00 2001
From: Miguel Prada <miguel.prada@tecnalia.com>
Date: Mon, 18 Dec 2023 16:26:30 +0100
Subject: [PATCH 05/10] Add jobs testing the templates on top of test ROS
 repository

---
 .gitlab-ci.yml | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 3d2cc4d..8ca6df2 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -20,3 +20,20 @@ deploy_devcontainer:
     expire_in: 1 year
   tags:
     - docker
+
+.simple_ros_test:
+  variables:
+    TEMPLATES_PIPELINE_REF: $CI_COMMIT_REF_NAME
+  trigger:
+    project: tecnalia_robotics/ci_test_repos/simple_ros_test
+    strategy: depend
+
+simple_ros_melodic_test:
+  extends: .simple_ros_test
+  variables:
+    TEMPLATE_TO_USE: ci-templates/auto-rules/melodic-default.yml
+
+simple_ros_noetic_test:
+  extends: .simple_ros_test
+  variables:
+    TEMPLATE_TO_USE: ci-templates/auto-rules/noetic-default.yml
-- 
GitLab


From 6b93b65706509705218f6bd46bc0e22b9b86b546 Mon Sep 17 00:00:00 2001
From: Miguel Prada <miguel.prada@tecnalia.com>
Date: Mon, 18 Dec 2023 16:37:39 +0100
Subject: [PATCH 06/10] Add job testing usage of 'core.yml'

---
 .gitlab-ci.yml            |  5 +++++
 test/core_with_noetic.yml | 11 +++++++++++
 2 files changed, 16 insertions(+)
 create mode 100644 test/core_with_noetic.yml

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 8ca6df2..ffe0a5a 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -37,3 +37,8 @@ simple_ros_noetic_test:
   extends: .simple_ros_test
   variables:
     TEMPLATE_TO_USE: ci-templates/auto-rules/noetic-default.yml
+
+simple_ros_core_with_noetic_test:
+  extends: .simple_ros_test
+  variables:
+    TEMPLATE_TO_USE: test/core_with_noetic.yml
diff --git a/test/core_with_noetic.yml b/test/core_with_noetic.yml
new file mode 100644
index 0000000..ce063ed
--- /dev/null
+++ b/test/core_with_noetic.yml
@@ -0,0 +1,11 @@
+include: ci-templates/core.yml
+
+industrial_ci_melodic:
+  extends: .industrial_ci
+  variables:
+    ROS_DISTRO: melodic
+
+industrial_ci_noetic:
+  extends: .industrial_ci
+  variables:
+    ROS_DISTRO: noetic
-- 
GitLab


From b713a597c99bc8c1ff5b66642cc41a641dd62ba3 Mon Sep 17 00:00:00 2001
From: Miguel Prada <miguel.prada@tecnalia.com>
Date: Mon, 18 Dec 2023 17:04:56 +0100
Subject: [PATCH 07/10] Modify python job logic

Create hidden templates in core and make conditionally make them visible in auto-rules
---
 ci-templates/auto-rules/no-default.yml |  6 ++++--
 ci-templates/python.yml                | 13 +++++--------
 2 files changed, 9 insertions(+), 10 deletions(-)

diff --git a/ci-templates/auto-rules/no-default.yml b/ci-templates/auto-rules/no-default.yml
index 6515130..6171018 100644
--- a/ci-templates/auto-rules/no-default.yml
+++ b/ci-templates/auto-rules/no-default.yml
@@ -49,7 +49,8 @@ industrial_ci_humble:
     - if: $DEFAULT_DISTRO == "humble"
     - if: $BUILD_HUMBLE
 
-.py3_template:
+py3_template:
+  extends: .py3_template
   rules:
     - if: $DEFAULT_DISTRO == "noetic"
       when: always
@@ -61,7 +62,8 @@ industrial_ci_humble:
       when: always
   when: always
 
-.py2_template:
+py2_template:
+  extends: .py2_template
   rules:
     - if: $DEFAULT_DISTRO == "kinetic"
       when: always
diff --git a/ci-templates/python.yml b/ci-templates/python.yml
index e4d00ed..61d476a 100644
--- a/ci-templates/python.yml
+++ b/ci-templates/python.yml
@@ -22,42 +22,39 @@
     - bandit -r . || RET=1
     - safety check || RET=1
     - exit $RET
-  when: always
   allow_failure: true
 
 .py3_template:
   image: python:3-alpine
   variables:
     PYTHON_VERSION: python3
-  when: never
 
 .py2_template:
   image: python:2-alpine
   variables:
     PYTHON_VERSION: python2
-  when: always
 
-py3-flake8:
+.py3-flake8:
   extends:
     - .flake8_template
     - .py3_template
 
-py2-flake8:
+.py2-flake8:
   extends:
     - .flake8_template
     - .py2_template
 
-py3-flake8_extended:
+.py3-flake8_extended:
   extends:
     - .flake8_extended_template
     - .py3_template
 
-py2-flake8_extended:
+.py2-flake8_extended:
   extends:
     - .flake8_extended_template
     - .py2_template
 
-py3-security:
+.py3-security:
   extends:
     - .security_template
     - .py3_template
-- 
GitLab


From 30d754f0f8596e47f25c45920d1add885cdfba4d Mon Sep 17 00:00:00 2001
From: Miguel Prada <miguel.prada@tecnalia.com>
Date: Mon, 18 Dec 2023 17:10:19 +0100
Subject: [PATCH 08/10] Fix errors in python auto-rules logic

---
 ci-templates/auto-rules/no-default.yml | 32 ++++++++++++++++++++++----
 1 file changed, 27 insertions(+), 5 deletions(-)

diff --git a/ci-templates/auto-rules/no-default.yml b/ci-templates/auto-rules/no-default.yml
index 6171018..4921c97 100644
--- a/ci-templates/auto-rules/no-default.yml
+++ b/ci-templates/auto-rules/no-default.yml
@@ -49,8 +49,7 @@ industrial_ci_humble:
     - if: $DEFAULT_DISTRO == "humble"
     - if: $BUILD_HUMBLE
 
-py3_template:
-  extends: .py3_template
+.py3_rules:
   rules:
     - if: $DEFAULT_DISTRO == "noetic"
       when: always
@@ -60,10 +59,23 @@ py3_template:
       when: always
     - if: $BUILD_HUMBLE
       when: always
-  when: always
 
-py2_template:
-  extends: .py2_template
+py3-flake8:
+  extends:
+    - .py3_rules
+    - .py3-flake8
+
+py3-flake8_extended:
+  extends:
+    - .py3_rules
+    - .py3-flake8_extended
+
+py3-security:
+  extends:
+    - .py3_rules
+    - .py3-security
+
+.py2_rules:
   rules:
     - if: $DEFAULT_DISTRO == "kinetic"
       when: always
@@ -74,6 +86,16 @@ py2_template:
     - if: $BUILD_MELODIC
       when: always
 
+py2-flake8:
+  extends:
+    - .py2_rules
+    - .py2-flake8
+
+py2-flake8_extended:
+  extends:
+    - .py2_rules
+    - .py2-flake8_extended
+
 ddeploy:
   extends: .ddeploy
   rules:
-- 
GitLab


From 3c683127607837734ddda55ba8dee06c8271b2a7 Mon Sep 17 00:00:00 2001
From: Miguel Prada <miguel.prada@tecnalia.com>
Date: Mon, 18 Dec 2023 17:13:33 +0100
Subject: [PATCH 09/10] Add job testing two distros using auto-rules

---
 .gitlab-ci.yml                     | 5 +++++
 test/auto_rules_melodic_noetic.yml | 5 +++++
 2 files changed, 10 insertions(+)
 create mode 100644 test/auto_rules_melodic_noetic.yml

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index ffe0a5a..e64ddbe 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -42,3 +42,8 @@ simple_ros_core_with_noetic_test:
   extends: .simple_ros_test
   variables:
     TEMPLATE_TO_USE: test/core_with_noetic.yml
+
+simple_ros_auto_rules_melodic_noetic_test:
+  extends: .simple_ros_test
+  variables:
+    TEMPLATE_TO_USE: test/auto_rules_melodic_noetic.yml
diff --git a/test/auto_rules_melodic_noetic.yml b/test/auto_rules_melodic_noetic.yml
new file mode 100644
index 0000000..a3301c4
--- /dev/null
+++ b/test/auto_rules_melodic_noetic.yml
@@ -0,0 +1,5 @@
+include: ci-templates/auto-rules/no-default.yml
+
+variables:
+  BUILD_MELODIC: true
+  BUILD_NOETIC: true
-- 
GitLab


From a8ff0415cfc2bdceb8e259d168c6e525abaf371e Mon Sep 17 00:00:00 2001
From: Miguel Prada <miguel.prada@tecnalia.com>
Date: Mon, 18 Dec 2023 17:15:25 +0100
Subject: [PATCH 10/10] Use 1 instead of true in CI template

---
 test/auto_rules_melodic_noetic.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/auto_rules_melodic_noetic.yml b/test/auto_rules_melodic_noetic.yml
index a3301c4..26e5143 100644
--- a/test/auto_rules_melodic_noetic.yml
+++ b/test/auto_rules_melodic_noetic.yml
@@ -1,5 +1,5 @@
 include: ci-templates/auto-rules/no-default.yml
 
 variables:
-  BUILD_MELODIC: true
-  BUILD_NOETIC: true
+  BUILD_MELODIC: 1
+  BUILD_NOETIC: 1
-- 
GitLab