From 0671f49c2d8b4f1cf88be84cd091c9be1118fca4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Moreno=20i=20Caireta=2C=20I=C3=B1igo?=
<inigo.moreno@tecnalia.com>
Date: Tue, 21 Sep 2021 15:10:54 +0200
Subject: [PATCH] Squashed: "Improve CI template organization"
---
...ab-industrial-ci-kinetic-no-rosinstall.yml | 191 +----------------
...-industrial-ci-kinetic-with-rosinstall.yml | 193 +-----------------
README.adoc | 28 +--
ci-templates/README.adoc | 54 +++++
ci-templates/auto-rules/kinetic-default.yml | 4 +
ci-templates/auto-rules/melodic-default.yml | 4 +
ci-templates/auto-rules/no-default.yml | 86 ++++++++
ci-templates/auto-rules/noetic-default.yml | 4 +
.../with-rosinstall/kinetic-default.yml | 4 +
.../with-rosinstall/melodic-default.yml | 4 +
.../auto-rules/with-rosinstall/no-default.yml | 6 +
.../with-rosinstall/noetic-default.yml | 4 +
ci-templates/core.yml | 31 +++
ci-templates/examples/build_vars.yml | 8 +
ci-templates/examples/manual_job.yml | 9 +
ci-templates/examples/simple_include.yml | 3 +
ci-templates/industrial-ci-templates.yml | 38 ++++
ci-templates/syntax-check.yml | 83 ++++++++
18 files changed, 351 insertions(+), 403 deletions(-)
create mode 100644 ci-templates/README.adoc
create mode 100644 ci-templates/auto-rules/kinetic-default.yml
create mode 100644 ci-templates/auto-rules/melodic-default.yml
create mode 100644 ci-templates/auto-rules/no-default.yml
create mode 100644 ci-templates/auto-rules/noetic-default.yml
create mode 100644 ci-templates/auto-rules/with-rosinstall/kinetic-default.yml
create mode 100644 ci-templates/auto-rules/with-rosinstall/melodic-default.yml
create mode 100644 ci-templates/auto-rules/with-rosinstall/no-default.yml
create mode 100644 ci-templates/auto-rules/with-rosinstall/noetic-default.yml
create mode 100644 ci-templates/core.yml
create mode 100644 ci-templates/examples/build_vars.yml
create mode 100644 ci-templates/examples/manual_job.yml
create mode 100644 ci-templates/examples/simple_include.yml
create mode 100644 ci-templates/industrial-ci-templates.yml
create mode 100644 ci-templates/syntax-check.yml
diff --git a/.gitlab-industrial-ci-kinetic-no-rosinstall.yml b/.gitlab-industrial-ci-kinetic-no-rosinstall.yml
index d5cc20d..0e7c125 100644
--- a/.gitlab-industrial-ci-kinetic-no-rosinstall.yml
+++ b/.gitlab-industrial-ci-kinetic-no-rosinstall.yml
@@ -1,188 +1,3 @@
-image: tecnalia-docker-dev.artifact.tecnalia.com/docker:git
-
-variables:
- TMPDIR: /builds/${CI_PROJECT_NAMESPACE}
- GIT_LFS_SKIP_SMUDGE: 1 # Avoid fetching the files from LFS when doing git fetch
-
-services:
- - name: tecnalia-docker-dev.artifact.tecnalia.com/docker:dind
- alias: docker
-
-stages:
- - build
- - postcheck
-
-workflow:
- rules:
- - if: $CI_COMMIT_TAG
- - if: $CI_COMMIT_BRANCH
-
-before_script:
- - apk add --update bash coreutils tar wget
- - docker login tecnalia-docker-dev.artifact.tecnalia.com -u ${ARTIFACT_CI_USER} -p ${ARTIFACT_CI_TOKEN}
-
-.industrial_ci:
- stage: build
- before_script:
- - apk add --update bash coreutils tar
- - git clone --quiet --depth 1 https://github.com/ros-industrial/industrial_ci.git .ci_config
- - docker login ${ARTIFACT_DOCKER_URL} -u ${ARTIFACT_CI_USER} -p ${ARTIFACT_CI_TOKEN}
- - docker login tecnalia-docker-dev.artifact.tecnalia.com -u ${ARTIFACT_CI_USER} -p ${ARTIFACT_CI_TOKEN}
- script: .ci_config/gitlab.sh
- variables:
- ADDITIONAL_DEBS: curl
- PARALLEL_BUILDS: 2 # keep this for now, since there seems to be memory limit issues in the ci VM
- AFTER_SETUP_TARGET_WORKSPACE: rosenv && bash <(curl -Ls https://git.code.tecnalia.com/tecnalia_robotics-public/gitlab_templates/raw/master/scripts/ci_run_entry_points.sh) $$current_ws
- CMAKE_ARGS: -DCMAKE_CXX_FLAGS=-Wno-ignored-attributes -Wno-int-in-bool-context
- tags:
- - docker
-
-industrial_ci_kinetic:
- extends: .industrial_ci
- variables:
- ROS_DISTRO: kinetic
- DOCKER_IMAGE: "${ARTIFACT_DOCKER_URL}/flexbotics-base-devel:${ROS_DISTRO}"
- rules:
- - if: $CI_COMMIT_TAG
- when: never
- - if: $CI_COMMIT_REF_NAME =~ /^melodic-.*/
- when: never
- - if: $CI_COMMIT_REF_NAME =~ /^noetic-.*/
- when: never
- - when: always
-
-industrial_ci_melodic:
- extends: .industrial_ci
- variables:
- ROS_DISTRO: melodic
- DOCKER_IMAGE: "${ARTIFACT_DOCKER_URL}/flexbotics-base-devel:${ROS_DISTRO}"
- rules:
- - if: $CI_COMMIT_TAG
- when: never
- - if: $CI_COMMIT_REF_NAME =~ /^melodic-.*/
-
-industrial_ci_noetic:
- extends: .industrial_ci
- variables:
- ROS_DISTRO: noetic
- DOCKER_IMAGE: "${ARTIFACT_DOCKER_URL}/flexbotics-base-devel:${ROS_DISTRO}"
- rules:
- - if: $CI_COMMIT_TAG
- when: never
- - if: $CI_COMMIT_REF_NAME =~ /^noetic-.*/
-
-industrial_ci_kinetic_deploy:
- extends: .industrial_ci
- variables:
- ROS_DISTRO: kinetic
- DOCKER_IMAGE: "${ARTIFACT_DOCKER_URL}/flexbotics-base-devel:${ROS_DISTRO}"
- AFTER_SCRIPT: "wget -q -O - https://git.code.tecnalia.com/tecnalia_robotics-public/gitlab_templates/raw/master/scripts/pre-deploy-cleanup.bash | bash"
- DOCKER_COMMIT: industrial_ci_image
- script:
- - .ci_config/gitlab.sh
- - cd $(mktemp -d)
- - wget https://git.code.tecnalia.com/tecnalia_robotics-public/gitlab_templates/-/archive/master/gitlab_templates-master.zip
- - unzip gitlab_templates-master.zip && cd gitlab_templates-master/deploy
- - cd base
- - chmod a+x ros_entrypoint.bash
- - docker build -t ${ARTIFACT_DOCKER_URL}/${CI_PROJECT_NAME}:${CI_COMMIT_REF_NAME} .
- - docker push ${ARTIFACT_DOCKER_URL}/${CI_PROJECT_NAME}:${CI_COMMIT_REF_NAME}
- - cd ..
- - cd dev
- - docker build --build-arg APPLICATION_IMAGE=${ARTIFACT_DOCKER_URL}/${CI_PROJECT_NAME}:${CI_COMMIT_REF_NAME} -t ${ARTIFACT_DOCKER_URL}/${CI_PROJECT_NAME}:${CI_COMMIT_REF_NAME}-dev .
- - docker push ${ARTIFACT_DOCKER_URL}/${CI_PROJECT_NAME}:${CI_COMMIT_REF_NAME}-dev
- - cd ..
- rules:
- - if: $CI_COMMIT_TAG
-
-clang_format:
- stage: postcheck
- before_script:
- - apk add --update bash coreutils tar wget
- - git clone --quiet --depth 1 https://github.com/ros-industrial/industrial_ci.git .ci_config -b legacy
- - wget https://git.code.tecnalia.com/tecnalia_robotics-public/gitlab_templates/raw/master/clang-format -O .clang-format
- script: .ci_config/gitlab.sh
- variables:
- ROS_DISTRO: kinetic
- CLANG_FORMAT_CHECK: file
- CLANG_FORMAT_VERSION: "3.8"
- when: always
- tags:
- - docker
-
-##########################################################
-## Simple check for basic Python syntax errors; this is much less thorough than
-## other options like pylint, but much faster and less strict
-##########################################################
-python_syntax:
- before_script:
- - apk add --update python2
- stage: build
- script: python -m compileall -q .
- when: always
- tags:
- - docker
-
-##########################################################
-## Check for errors in bash/sh scripts
-## Uses https://github.com/koalaman/shellcheck
-##########################################################
-bash_syntax:
- before_script:
- - apk add --update shellcheck
- stage: postcheck
- script:
- - RET=0
- - find . -type f -name '*.sh' -exec shellcheck -s sh {} + || RET=1
- - find . -type f -name '*.bash' -exec shellcheck -s bash {} + || RET=1
- - exit $RET
- when: always
- tags:
- - docker
-
-##########################################################
-## Check for errors in markdown files
-## Uses https://github.com/cytopia/awesome-ci
-## Disables the line length rule with --custom="-r ..."
-## See other rules that can be configured in
-## https://github.com/markdownlint/markdownlint/blob/master/docs/RULES.md
-##########################################################
-markdown_syntax:
- stage: postcheck
- script: docker run -v ${PWD}:/ac tecnalia-docker-dev.artifact.tecnalia.com/cytopia/awesome-ci syntax-markdown --path=/ac --extension=md --ignore=.ci_config --custom="-r ~MD013,~MD014"
- when: always
- tags:
- - docker
-
-##########################################################
-## Check for various syntax issues in files; currently
-## trailing white spaces
-## utf8 formatting
-## Uses https://github.com/cytopia/awesome-ci
-##########################################################
-file_syntax:
- stage: postcheck
- script:
- - RET=0
- - docker run -v ${PWD}:/ac tecnalia-docker-dev.artifact.tecnalia.com/cytopia/awesome-ci file-trailing-space --path=/ac --extension=py,txt,cpp,h,md,sh,bash,xml,launch --ignore=.ci_config || RET=1
- - docker run -v ${PWD}:/ac tecnalia-docker-dev.artifact.tecnalia.com/cytopia/awesome-ci file-utf8 --path=/ac --extension=py,txt,cpp,h,md,sh,bash,xml,launch --ignore=.ci_config || RET=1
- - exit $RET
- when: always
- tags:
- - docker
-
-##########################################################
-## Check for errors in Asciidoctor files
-## Could not find a proper linter for Asciidoctor format,
-## so this is a custom-made "dirty" script
-##########################################################
-adoc_syntax:
- stage: postcheck
- image: tecnalia-docker-dev.artifact.tecnalia.com/alpine
- before_script:
- - apk add --update curl asciidoctor bash
- script:
- - curl -Ls https://git.code.tecnalia.com/tecnalia_robotics-public/gitlab_templates/raw/master/scripts/asciidoctor_syntax_check.bash | bash -s -- .
- tags:
- - docker
- when: always
+include:
+ - project: tecnalia_robotics-public/gitlab_templates
+ file: ci-templates/auto-rules/kinetic-default.yml
\ No newline at end of file
diff --git a/.gitlab-industrial-ci-kinetic-with-rosinstall.yml b/.gitlab-industrial-ci-kinetic-with-rosinstall.yml
index 643b46b..2413c17 100644
--- a/.gitlab-industrial-ci-kinetic-with-rosinstall.yml
+++ b/.gitlab-industrial-ci-kinetic-with-rosinstall.yml
@@ -1,190 +1,3 @@
-image: tecnalia-docker-dev.artifact.tecnalia.com/docker:git
-
-variables:
- TMPDIR: /builds/${CI_PROJECT_NAMESPACE}
- GIT_LFS_SKIP_SMUDGE: 1 # Avoid fetching the files from LFS when doing git fetch
-
-services:
- - name: tecnalia-docker-dev.artifact.tecnalia.com/docker:dind
- alias: docker
-
-stages:
- - build
- - postcheck
-
-workflow:
- rules:
- - if: $CI_COMMIT_TAG
- - if: $CI_COMMIT_BRANCH
-
-before_script:
- - apk add --update bash coreutils tar wget
- - docker login tecnalia-docker-dev.artifact.tecnalia.com -u ${ARTIFACT_CI_USER} -p ${ARTIFACT_CI_TOKEN}
-
-.industrial_ci:
- stage: build
- before_script:
- - apk add --update bash coreutils tar
- - git clone --quiet --depth 1 https://github.com/ros-industrial/industrial_ci.git .ci_config
- - docker login ${ARTIFACT_DOCKER_URL} -u ${ARTIFACT_CI_USER} -p ${ARTIFACT_CI_TOKEN}
- - docker login tecnalia-docker-dev.artifact.tecnalia.com -u ${ARTIFACT_CI_USER} -p ${ARTIFACT_CI_TOKEN}
- script: .ci_config/gitlab.sh
- variables:
- ADDITIONAL_DEBS: curl
- PARALLEL_BUILDS: 2 # keep this for now, since there seems to be memory limit issues in the ci VM
- UPSTREAM_WORKSPACE: .rosinstall
- AFTER_SETUP_UPSTREAM_WORKSPACE: "rosenv && bash <(curl -Ls https://git.code.tecnalia.com/tecnalia_robotics-public/gitlab_templates/raw/master/scripts/ci_run_entry_points.sh) $$current_ws"
- AFTER_SETUP_TARGET_WORKSPACE: "rosenv && bash <(curl -Ls https://git.code.tecnalia.com/tecnalia_robotics-public/gitlab_templates/raw/master/scripts/ci_run_entry_points.sh) $$current_ws"
- CMAKE_ARGS: '-DCMAKE_CXX_FLAGS=-Wno-ignored-attributes -Wno-int-in-bool-context'
- tags:
- - docker
-
-industrial_ci_kinetic:
- extends: .industrial_ci
- variables:
- ROS_DISTRO: kinetic
- DOCKER_IMAGE: "${ARTIFACT_DOCKER_URL}/flexbotics-base-devel:${ROS_DISTRO}"
- rules:
- - if: $CI_COMMIT_TAG
- when: never
- - if: $CI_COMMIT_REF_NAME =~ /^melodic-.*/
- when: never
- - if: $CI_COMMIT_REF_NAME =~ /^noetic-.*/
- when: never
- - when: always
-
-industrial_ci_melodic:
- extends: .industrial_ci
- variables:
- ROS_DISTRO: melodic
- DOCKER_IMAGE: "${ARTIFACT_DOCKER_URL}/flexbotics-base-devel:${ROS_DISTRO}"
- rules:
- - if: $CI_COMMIT_TAG
- when: never
- - if: $CI_COMMIT_REF_NAME =~ /^melodic-.*/
-
-industrial_ci_noetic:
- extends: .industrial_ci
- variables:
- ROS_DISTRO: noetic
- DOCKER_IMAGE: "${ARTIFACT_DOCKER_URL}/flexbotics-base-devel:${ROS_DISTRO}"
- rules:
- - if: $CI_COMMIT_TAG
- when: never
- - if: $CI_COMMIT_REF_NAME =~ /^noetic-.*/
-
-industrial_ci_kinetic_deploy:
- extends: .industrial_ci
- variables:
- ROS_DISTRO: kinetic
- DOCKER_IMAGE: "${ARTIFACT_DOCKER_URL}/flexbotics-base-devel:${ROS_DISTRO}"
- AFTER_SCRIPT: "wget -q -O - https://git.code.tecnalia.com/tecnalia_robotics-public/gitlab_templates/raw/master/scripts/pre-deploy-cleanup.bash | bash"
- DOCKER_COMMIT: industrial_ci_image
- script:
- - .ci_config/gitlab.sh
- - cd $(mktemp -d)
- - wget https://git.code.tecnalia.com/tecnalia_robotics-public/gitlab_templates/-/archive/master/gitlab_templates-master.zip
- - unzip gitlab_templates-master.zip && cd gitlab_templates-master/deploy
- - cd base
- - chmod a+x ros_entrypoint.bash
- - docker build -t ${ARTIFACT_DOCKER_URL}/${CI_PROJECT_NAME}:${CI_COMMIT_REF_NAME} .
- - docker push ${ARTIFACT_DOCKER_URL}/${CI_PROJECT_NAME}:${CI_COMMIT_REF_NAME}
- - cd ..
- - cd dev
- - docker build --build-arg APPLICATION_IMAGE=${ARTIFACT_DOCKER_URL}/${CI_PROJECT_NAME}:${CI_COMMIT_REF_NAME} -t ${ARTIFACT_DOCKER_URL}/${CI_PROJECT_NAME}:${CI_COMMIT_REF_NAME}-dev .
- - docker push ${ARTIFACT_DOCKER_URL}/${CI_PROJECT_NAME}:${CI_COMMIT_REF_NAME}-dev
- - cd ..
- rules:
- - if: $CI_COMMIT_TAG
-
-clang_format:
- stage: postcheck
- before_script:
- - apk add --update bash coreutils tar wget
- - git clone --quiet --depth 1 https://github.com/ros-industrial/industrial_ci.git .ci_config -b legacy
- - wget https://git.code.tecnalia.com/tecnalia_robotics-public/gitlab_templates/raw/master/clang-format -O .clang-format
- script: .ci_config/gitlab.sh
- variables:
- ROS_DISTRO: kinetic
- CLANG_FORMAT_CHECK: file
- CLANG_FORMAT_VERSION: "3.8"
- when: always
- tags:
- - docker
-
-##########################################################
-## Simple check for basic Python syntax errors; this is much less thorough than
-## other options like pylint, but much faster and less strict
-##########################################################
-python_syntax:
- before_script:
- - apk add --update python2
- stage: build
- script: python -m compileall -q .
- when: always
- tags:
- - docker
-
-##########################################################
-## Check for errors in bash/sh scripts
-## Uses https://github.com/koalaman/shellcheck
-##########################################################
-bash_syntax:
- before_script:
- - apk add --update shellcheck
- stage: postcheck
- script:
- - RET=0
- - find . -type f -name '*.sh' -exec shellcheck -s sh {} + || RET=1
- - find . -type f -name '*.bash' -exec shellcheck -s bash {} + || RET=1
- - exit $RET
- when: always
- tags:
- - docker
-
-##########################################################
-## Check for errors in markdown files
-## Uses https://github.com/cytopia/awesome-ci
-## Disables the line length rule with --custom="-r ..."
-## See other rules that can be configured in
-## https://github.com/markdownlint/markdownlint/blob/master/docs/RULES.md
-##########################################################
-markdown_syntax:
- stage: postcheck
- script: docker run -v ${PWD}:/ac tecnalia-docker-dev.artifact.tecnalia.com/cytopia/awesome-ci syntax-markdown --path=/ac --extension=md --ignore=.ci_config --custom="-r ~MD013,~MD014"
- when: always
- tags:
- - docker
-
-##########################################################
-## Check for various syntax issues in files; currently
-## trailing white spaces
-## utf8 formatting
-## Uses https://github.com/cytopia/awesome-ci
-##########################################################
-file_syntax:
- stage: postcheck
- script:
- - RET=0
- - docker run -v ${PWD}:/ac tecnalia-docker-dev.artifact.tecnalia.com/cytopia/awesome-ci file-trailing-space --path=/ac --extension=py,txt,cpp,h,md,sh,bash,xml,launch --ignore=.ci_config || RET=1
- - docker run -v ${PWD}:/ac tecnalia-docker-dev.artifact.tecnalia.com/cytopia/awesome-ci file-utf8 --path=/ac --extension=py,txt,cpp,h,md,sh,bash,xml,launch --ignore=.ci_config || RET=1
- - exit $RET
- when: always
- tags:
- - docker
-
-##########################################################
-## Check for errors in Asciidoctor files
-## Could not find a proper linter for Asciidoctor format,
-## so this is a custom-made "dirty" script
-##########################################################
-adoc_syntax:
- stage: postcheck
- image: tecnalia-docker-dev.artifact.tecnalia.com/alpine
- before_script:
- - apk add --update curl asciidoctor bash
- script:
- - curl -Ls https://git.code.tecnalia.com/tecnalia_robotics-public/gitlab_templates/raw/master/scripts/asciidoctor_syntax_check.bash | bash -s -- .
- tags:
- - docker
- when: always
+include:
+ - project: tecnalia_robotics-public/gitlab_templates
+ file: ci-templates/auto-rules/with-rosinstall/kinetic-default.yml
\ No newline at end of file
diff --git a/README.adoc b/README.adoc
index f50fb4d..eb4ff72 100644
--- a/README.adoc
+++ b/README.adoc
@@ -1,32 +1,10 @@
= GitLab Templates
+:toc:
This repository contains miscellaneous resources that need to be accessible without authentication.
-== GitLab CI templates
-Two templates with default CI pipeline configurations are provided, for any repo to make use of:
-
-- `.gitlab-industrial-ci-kinetic-no-rosinstall.yml`
-- `.gitlab-industrial-ci-kinetic-with-rosinstall.yml`
-
-Both define basically the same pipeline, consisting of the following stages and jobs:
-
-- Stage: `precheck`
- * Check python syntax
-- Stage: `build`
- * Run `industrial_ci`
- ** Uses `flexbotics-base-devel` as a starting image
- ** Runs link:scripts/ci_run_entry_points.sh[] in the `BEFORE_SCRIPT` stage, which in turn finds and runs every `ci_entry_script.bash` script found in the source tree (target repo + repos from `.rosinstall`)
- * Only when tags are pushed:
- ** Run some cleanup in the `industrial_ci` context (e.g. delete sources)
- ** Commit image used by `industrial_ci`
- ** Build deployment image on top of it and push it to our registry
-- Stage: `postcheck` (jobs in this stage are allowed to fail)
- * Check `bash`/`sh` script syntax
- * Check Markdown syntax
- * Check files have no trailing whitespace and are UTF-8 encoded
-
-It is important to note that both images set the variable `GIT_LFS_SKIP_SMUDGE`, which means that when the repositories are cloned/fetched, the files hosted using Git LFS will not be downloaded. An example on how to explicitely fetch the files from LFS can be found in the https://git.code.tecnalia.com/tecnalia_robotics/common_config/blob/e93dbe6379d94e6a895a33c59bf17ccab26eacf9/.gitlab-ci.yml#L132[common_config repository].
+include::ci-templates/README.adoc[leveloffset=+1]
== Custom `rosdep` definitions
@@ -37,5 +15,5 @@ Custom `rosdep` definitions for the packages hosted in our internal debian repos
Resources required for building the deployment docker images are also hosted in this repository.
This basically boils down to:
-- link:scripts/pre-deploy-cleanup.bash[] is run in the `industrial_ci` container before commiting to an image
+- link:scripts/pre-deploy-cleanup.bash[] is run in the `.industrial_ci_deploy` template before commiting to an image
- link:deploy/Dockerfile[] is used to perform some additional cleanup and install link:deploy/ros_entrypoint.bash[] as entrypoint script on top of the `industrial_ci` image
diff --git a/ci-templates/README.adoc b/ci-templates/README.adoc
new file mode 100644
index 0000000..d4f1a0d
--- /dev/null
+++ b/ci-templates/README.adoc
@@ -0,0 +1,54 @@
+
+= GitLab CI templates
+
+Templates for GitLab Continuous Integration are provided in the folder link:ci-templates[]. These can be included from any repo by using `include:project`:
+
+```yml
+include::examples/simple_include.yml[]
+```
+
+== Core Pipeline
+The Core pipeline, defined in link:ci-templates/core.yml[], has the following structure:
+
+- Stage: `build`
+ * Check python syntax
+ * `.industrial_ci` template (see https://github.com/ros-industrial/industrial_ci[the industrial_ci repo]), which can be used to define jobs that build `ROS` packages
+ ** Needs the `ROS_DISTRO` variable to be defined
+ ** If the package has `.rosinstall` dependencies, they can be installed using the variable `UPSTREAM_WORKSPACE`
+ ** Uses `flexbotics-base-devel:${ROS_DISTRO}` as a starting image
+ ** Runs link:scripts/ci_run_entry_points.sh[] in the `AFTER_SETUP_TARGET_WORKSPACE` stage, which in turn finds and runs every `ci_entry_script.bash` script found in the target repo.
+ * `.industrial_ci_deploy` template, which:
+ ** Runs `industrial_ci` by extending `.industrial_ci`
+ ** Runs some cleanup in the `industrial_ci` context (e.g. delete sources)
+ ** Commits image used by `industrial_ci`
+ ** Builds deployment image on top of it and push it to our registry
+- Stage: `.post`
+ * Check `bash`/`sh` script syntax
+ * Check Markdown and AsciiDoc syntax
+ * Check files have no trailing whitespace and are `UTF-8` encoded
+ * Check that `C++` files follow this link:clang-format[]
+
+It is important to note that both templates set the variable `GIT_LFS_SKIP_SMUDGE`, which means that when the repositories are cloned/fetched, the files hosted using Git LFS will not be downloaded. An example on how to explicitely fetch the files from LFS can be found in the https://git.code.tecnalia.com/tecnalia_robotics/common_config/blob/e93dbe6379d94e6a895a33c59bf17ccab26eacf9/.gitlab-ci.yml#L132[common_config repository].
+
+== Manual Jobs
+
+Using the templates defined by the core pipeline, one can easily define jobs to build the packages in the repo.
+
+```yml
+include::examples/manual_job.yml[]
+```
+
+More examples can be found in link:ci-templates/examples[].
+
+== Auto Rules
+In the link:ci-templates/auto-rules[] folder, automatic pipelines are defined that will try to detect the ROS distro from the branch name and create jobs.
+For example, if the branch starts with `noetic-` it will build for noetic.
+If the branch doesn't start with any ROS distro, it will build the default distro, which can be specified with the variable `DEFAULT_DISTRO`.
+For convenience, some templates are provided that already set this variable.
+If your repo needs to uses `.rosinstall`, you can use the templates in the link:ci-templates/auto-rules/with-rosinstall[] folder.
+
+The auto rules also allow for using variables to explicitely activate each job:
+
+```yml
+include::examples/build_vars.yml[]
+```
diff --git a/ci-templates/auto-rules/kinetic-default.yml b/ci-templates/auto-rules/kinetic-default.yml
new file mode 100644
index 0000000..7bc72b9
--- /dev/null
+++ b/ci-templates/auto-rules/kinetic-default.yml
@@ -0,0 +1,4 @@
+include : ci-templates/auto-rules/no-default.yml
+
+variables:
+ DEFAULT_DISTRO: kinetic
\ No newline at end of file
diff --git a/ci-templates/auto-rules/melodic-default.yml b/ci-templates/auto-rules/melodic-default.yml
new file mode 100644
index 0000000..62379b9
--- /dev/null
+++ b/ci-templates/auto-rules/melodic-default.yml
@@ -0,0 +1,4 @@
+include : ci-templates/auto-rules/no-default.yml
+
+variables:
+ DEFAULT_DISTRO: melodic
\ No newline at end of file
diff --git a/ci-templates/auto-rules/no-default.yml b/ci-templates/auto-rules/no-default.yml
new file mode 100644
index 0000000..a2f6a98
--- /dev/null
+++ b/ci-templates/auto-rules/no-default.yml
@@ -0,0 +1,86 @@
+include : ci-templates/core.yml
+
+workflow:
+ rules:
+ - if: $CI_COMMIT_BRANCH && $CI_COMMIT_REF_NAME =~ /^kinetic-.*/
+ variables:
+ DEFAULT_DISTRO: "kinetic"
+ - if: $CI_COMMIT_BRANCH && $CI_COMMIT_REF_NAME =~ /^melodic-.*/
+ variables:
+ DEFAULT_DISTRO: "melodic"
+ - if: $CI_COMMIT_BRANCH && $CI_COMMIT_REF_NAME =~ /^noetic-.*/
+ variables:
+ DEFAULT_DISTRO: "noetic"
+ - if: $CI_COMMIT_TAG && $CI_COMMIT_REF_NAME =~ /^kinetic-.*/
+ variables:
+ DEFAULT_DISTRO: "kinetic"
+ DEPLOY: "true"
+ - if: $CI_COMMIT_TAG && $CI_COMMIT_REF_NAME =~ /^melodic-.*/
+ variables:
+ DEFAULT_DISTRO: "melodic"
+ DEPLOY: "true"
+ - if: $CI_COMMIT_TAG && $CI_COMMIT_REF_NAME =~ /^noetic-.*/
+ variables:
+ DEFAULT_DISTRO: "noetic"
+ DEPLOY: "true"
+ - if: $CI_COMMIT_TAG
+ variables:
+ DEPLOY: "true"
+ - if: $CI_COMMIT_BRANCH
+
+
+industrial_ci_kinetic:
+ extends: .industrial_ci
+ variables:
+ ROS_DISTRO: kinetic
+ rules:
+ - if: $DEPLOY
+ when: never
+ - if: $DEFAULT_DISTRO == "kinetic"
+ - if: $BUILD_KINETIC
+
+industrial_ci_melodic:
+ extends: .industrial_ci
+ variables:
+ ROS_DISTRO: melodic
+ rules:
+ - if: $DEPLOY
+ when: never
+ - if: $DEFAULT_DISTRO == "melodic"
+ - if: $BUILD_MELODIC
+
+industrial_ci_noetic:
+ extends: .industrial_ci
+ variables:
+ ROS_DISTRO: noetic
+ rules:
+ - if: $DEPLOY
+ when: never
+ - if: $DEFAULT_DISTRO == "noetic"
+ - if: $BUILD_NOETIC
+
+
+
+industrial_kinetic_deploy:
+ extends:
+ - industrial_ci_kinetic
+ - .industrial_ci_deploy
+ rules:
+ - if: $DEPLOY && $DEFAULT_DISTRO == "kinetic"
+ - if: $DEPLOY && $BUILD_KINETIC
+
+industrial_melodic_deploy:
+ extends:
+ - industrial_ci_melodic
+ - .industrial_ci_deploy
+ rules:
+ - if: $DEPLOY && $DEFAULT_DISTRO == "melodic"
+ - if: $DEPLOY && $BUILD_MELODIC
+
+industrial_noetic_deploy:
+ extends:
+ - industrial_ci_noetic
+ - .industrial_ci_deploy
+ rules:
+ - if: $DEPLOY && $DEFAULT_DISTRO == "noetic"
+ - if: $DEPLOY && $BUILD_NOETIC
diff --git a/ci-templates/auto-rules/noetic-default.yml b/ci-templates/auto-rules/noetic-default.yml
new file mode 100644
index 0000000..f35658b
--- /dev/null
+++ b/ci-templates/auto-rules/noetic-default.yml
@@ -0,0 +1,4 @@
+include : ci-templates/auto-rules/no-default.yml
+
+variables:
+ DEFAULT_DISTRO: noetic
\ No newline at end of file
diff --git a/ci-templates/auto-rules/with-rosinstall/kinetic-default.yml b/ci-templates/auto-rules/with-rosinstall/kinetic-default.yml
new file mode 100644
index 0000000..fe99b53
--- /dev/null
+++ b/ci-templates/auto-rules/with-rosinstall/kinetic-default.yml
@@ -0,0 +1,4 @@
+include : ci-templates/auto-rules/with-rosinstall/no-default.yml
+
+variables:
+ DEFAULT_DISTRO: kinetic
\ No newline at end of file
diff --git a/ci-templates/auto-rules/with-rosinstall/melodic-default.yml b/ci-templates/auto-rules/with-rosinstall/melodic-default.yml
new file mode 100644
index 0000000..47e501b
--- /dev/null
+++ b/ci-templates/auto-rules/with-rosinstall/melodic-default.yml
@@ -0,0 +1,4 @@
+include : ci-templates/auto-rules/with-rosinstall/no-default.yml
+
+variables:
+ DEFAULT_DISTRO: melodic
\ No newline at end of file
diff --git a/ci-templates/auto-rules/with-rosinstall/no-default.yml b/ci-templates/auto-rules/with-rosinstall/no-default.yml
new file mode 100644
index 0000000..f4735f5
--- /dev/null
+++ b/ci-templates/auto-rules/with-rosinstall/no-default.yml
@@ -0,0 +1,6 @@
+include : ci-templates/auto-rules/no-default.yml
+
+
+.industrial_ci:
+ variables:
+ UPSTREAM_WORKSPACE: .rosinstall
\ No newline at end of file
diff --git a/ci-templates/auto-rules/with-rosinstall/noetic-default.yml b/ci-templates/auto-rules/with-rosinstall/noetic-default.yml
new file mode 100644
index 0000000..0d83acd
--- /dev/null
+++ b/ci-templates/auto-rules/with-rosinstall/noetic-default.yml
@@ -0,0 +1,4 @@
+include : ci-templates/auto-rules/with-rosinstall/no-default.yml
+
+variables:
+ DEFAULT_DISTRO: noetic
\ No newline at end of file
diff --git a/ci-templates/core.yml b/ci-templates/core.yml
new file mode 100644
index 0000000..859514f
--- /dev/null
+++ b/ci-templates/core.yml
@@ -0,0 +1,31 @@
+
+image: tecnalia-docker-dev.artifact.tecnalia.com/docker:git
+
+
+workflow:
+ rules:
+ - if: $CI_COMMIT_TAG
+ - if: $CI_COMMIT_BRANCH
+
+variables:
+ TMPDIR: /builds/${CI_PROJECT_NAMESPACE}
+ GIT_LFS_SKIP_SMUDGE: 1 # Avoid fetching the files from LFS when doing git fetch
+
+services:
+ - name: tecnalia-docker-dev.artifact.tecnalia.com/docker:dind
+ alias: docker
+
+stages:
+ - build
+
+before_script:
+ - apk add --update bash coreutils tar wget
+ - docker login tecnalia-docker-dev.artifact.tecnalia.com -u ${ARTIFACT_CI_USER} -p ${ARTIFACT_CI_TOKEN}
+
+include :
+ - ci-templates/syntax-check.yml
+ - ci-templates/industrial-ci-templates.yml
+
+default:
+ tags:
+ - docker
\ No newline at end of file
diff --git a/ci-templates/examples/build_vars.yml b/ci-templates/examples/build_vars.yml
new file mode 100644
index 0000000..276246b
--- /dev/null
+++ b/ci-templates/examples/build_vars.yml
@@ -0,0 +1,8 @@
+include:
+ - project: tecnalia_robotics-public/gitlab_templates
+ file: ci-templates/auto-rules/no-default.yml
+
+variables:
+ BUILD_KINETIC: 1
+ BUILD_MELODIC: 1
+ BUILD_NOETIC: 1
diff --git a/ci-templates/examples/manual_job.yml b/ci-templates/examples/manual_job.yml
new file mode 100644
index 0000000..8cf4e1d
--- /dev/null
+++ b/ci-templates/examples/manual_job.yml
@@ -0,0 +1,9 @@
+include:
+ - project: tecnalia_robotics-public/gitlab_templates
+ file: ci-templates/core.yml
+
+industrial_ci_kinetic:
+ extends: .industrial-ci
+ variables:
+ ROS_DISTRO: kinetic
+ UPSTREAM_WORKSPACE: .rosinstall
\ No newline at end of file
diff --git a/ci-templates/examples/simple_include.yml b/ci-templates/examples/simple_include.yml
new file mode 100644
index 0000000..3b1a85d
--- /dev/null
+++ b/ci-templates/examples/simple_include.yml
@@ -0,0 +1,3 @@
+include:
+ - project: tecnalia_robotics-public/gitlab_templates
+ file: ci-templates/core.yml
\ No newline at end of file
diff --git a/ci-templates/industrial-ci-templates.yml b/ci-templates/industrial-ci-templates.yml
new file mode 100644
index 0000000..9f2516f
--- /dev/null
+++ b/ci-templates/industrial-ci-templates.yml
@@ -0,0 +1,38 @@
+
+.industrial_ci:
+ stage: build
+ before_script:
+ - apk add --update bash coreutils tar
+ - git clone --quiet --depth 1 https://github.com/ros-industrial/industrial_ci.git .ci_config
+ - docker login ${ARTIFACT_DOCKER_URL} -u ${ARTIFACT_CI_USER} -p ${ARTIFACT_CI_TOKEN}
+ - docker login tecnalia-docker-dev.artifact.tecnalia.com -u ${ARTIFACT_CI_USER} -p ${ARTIFACT_CI_TOKEN}
+ script: .ci_config/gitlab.sh
+ variables:
+ DOCKER_IMAGE: "${ARTIFACT_DOCKER_URL}/flexbotics-base-devel:${ROS_DISTRO}"
+ ADDITIONAL_DEBS: curl
+ PARALLEL_BUILDS: 2 # keep this for now, since there seems to be memory limit issues in the ci VM
+ AFTER_SETUP_TARGET_WORKSPACE: "rosenv && bash <(curl -Ls https://git.code.tecnalia.com/tecnalia_robotics-public/gitlab_templates/raw/master/scripts/ci_run_entry_points.sh) $$current_ws"
+ CMAKE_ARGS: -DCMAKE_CXX_FLAGS=-Wno-ignored-attributes -Wno-int-in-bool-context
+
+
+.industrial_ci_deploy:
+ extends: .industrial_ci
+ variables:
+ AFTER_SCRIPT: "wget -q -O - https://git.code.tecnalia.com/tecnalia_robotics-public/gitlab_templates/raw/master/scripts/pre-deploy-cleanup.bash | bash"
+ DOCKER_COMMIT: industrial_ci_image
+ script:
+ - .ci_config/gitlab.sh
+ - cd $(mktemp -d)
+ - wget https://git.code.tecnalia.com/tecnalia_robotics-public/gitlab_templates/-/archive/master/gitlab_templates-master.zip
+ - unzip gitlab_templates-master.zip && cd gitlab_templates-master/deploy
+ - cd base
+ - chmod a+x ros_entrypoint.bash
+ - docker build -t ${ARTIFACT_DOCKER_URL}/${CI_PROJECT_NAME}:${CI_COMMIT_REF_NAME} .
+ - docker push ${ARTIFACT_DOCKER_URL}/${CI_PROJECT_NAME}:${CI_COMMIT_REF_NAME}
+ - cd ..
+ - cd dev
+ - docker build --build-arg APPLICATION_IMAGE=${ARTIFACT_DOCKER_URL}/${CI_PROJECT_NAME}:${CI_COMMIT_REF_NAME} -t ${ARTIFACT_DOCKER_URL}/${CI_PROJECT_NAME}:${CI_COMMIT_REF_NAME}-dev .
+ - docker push ${ARTIFACT_DOCKER_URL}/${CI_PROJECT_NAME}:${CI_COMMIT_REF_NAME}-dev
+ - cd ..
+ rules:
+ - if: $DEPLOY
diff --git a/ci-templates/syntax-check.yml b/ci-templates/syntax-check.yml
new file mode 100644
index 0000000..ccf1df2
--- /dev/null
+++ b/ci-templates/syntax-check.yml
@@ -0,0 +1,83 @@
+##########################################################
+## Check formating of C and C++ files using clang-format
+##########################################################
+
+clang_format:
+ stage: .post
+ before_script:
+ - apk add --update bash coreutils tar wget
+ - git clone --quiet --depth 1 https://github.com/ros-industrial/industrial_ci.git .ci_config -b legacy
+ - wget https://git.code.tecnalia.com/tecnalia_robotics-public/gitlab_templates/raw/master/clang-format -O .clang-format
+ script: .ci_config/gitlab.sh
+ variables:
+ ROS_DISTRO: kinetic
+ CLANG_FORMAT_CHECK: file
+ CLANG_FORMAT_VERSION: "3.8"
+ when: always
+
+##########################################################
+## Simple check for basic Python syntax errors; this is much less thorough than
+## other options like pylint, but much faster and less strict
+##########################################################
+python_syntax:
+ before_script:
+ - apk add --update python2
+ stage: build
+ script: python -m compileall -q .
+ when: always
+
+##########################################################
+## Check for errors in bash/sh scripts
+## Uses https://github.com/koalaman/shellcheck
+##########################################################
+bash_syntax:
+ before_script:
+ - apk add --update shellcheck
+ stage: .post
+ script:
+ - RET=0
+ - find . -type f -name '*.sh' -exec shellcheck -s sh {} + || RET=1
+ - find . -type f -name '*.bash' -exec shellcheck -s bash {} + || RET=1
+ - exit $RET
+ when: always
+
+##########################################################
+## Check for errors in markdown files
+## Uses https://github.com/cytopia/awesome-ci
+## Disables the line length rule with --custom="-r ..."
+## See other rules that can be configured in
+## https://github.com/markdownlint/markdownlint/blob/master/docs/RULES.md
+##########################################################
+markdown_syntax:
+ stage: .post
+ script: docker run -v ${PWD}:/ac tecnalia-docker-dev.artifact.tecnalia.com/cytopia/awesome-ci syntax-markdown --path=/ac --extension=md --ignore=.ci_config --custom="-r ~MD013,~MD014"
+ when: always
+
+##########################################################
+## Check for various syntax issues in files; currently
+## trailing white spaces
+## utf8 formatting
+## Uses https://github.com/cytopia/awesome-ci
+##########################################################
+file_syntax:
+ stage: .post
+ script:
+ - RET=0
+ - docker run -v ${PWD}:/ac tecnalia-docker-dev.artifact.tecnalia.com/cytopia/awesome-ci file-trailing-space --path=/ac --extension=py,txt,cpp,h,md,sh,bash,xml,launch --ignore=.ci_config || RET=1
+ - docker run -v ${PWD}:/ac tecnalia-docker-dev.artifact.tecnalia.com/cytopia/awesome-ci file-utf8 --path=/ac --extension=py,txt,cpp,h,md,sh,bash,xml,launch --ignore=.ci_config || RET=1
+ - exit $RET
+ when: always
+
+##########################################################
+## Check for errors in Asciidoctor files
+## Could not find a proper linter for Asciidoctor format,
+## so this is a custom-made "dirty" script
+##########################################################
+adoc_syntax:
+ stage: .post
+ image: tecnalia-docker-dev.artifact.tecnalia.com/alpine
+ before_script:
+ - apk add --update curl asciidoctor bash
+ script:
+ - curl -Ls https://git.code.tecnalia.com/tecnalia_robotics-public/gitlab_templates/raw/master/scripts/asciidoctor_syntax_check.bash | bash -s -- .
+ when: always
--
GitLab