Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • miguel.prada/gitlab_templates
  • tecnalia_robotics-public/gitlab_templates
2 results
Show changes
Commits on Source (86)
Showing
with 224 additions and 391 deletions
shellcheck:
image: koalaman/shellcheck-alpine
script: shellcheck deploy/base/ros_entrypoint.bash scripts/*sh
image: tecnalia-docker-dev.artifact.tecnalia.com/koalaman/shellcheck-alpine
script: shellcheck scripts/*sh
tags:
- docker
# The ignored rules in hadolint checks are
# DL3006 Always tag the version of an image explicitly
# One of the Dockerfiles uses a temporary image which isn't important to tag
# and the other uses a tagged image through an ARG, which apparently is not
# properly recognized.
# DL3008 Pin versions in apt-get install
# A bit overkill
dockerfiles:
image: hadolint/hadolint:latest-debian
script:
- RET=0
- hadolint --ignore DL3006 --ignore DL3008 deploy/base/Dockerfile || RET=1
- hadolint --ignore DL3006 --ignore DL3008 deploy/dev/Dockerfile || RET=1
- exit $RET
tags:
- docker
yamllint:
image: sdesbure/yamllint
image: tecnalia-docker-dev.artifact.tecnalia.com/sdesbure/yamllint
script: yamllint $(find . -name '*.yml' -o -name '*.yaml')
tags:
- docker
......
image: docker:git
variables:
TMPDIR: /builds/${CI_PROJECT_NAMESPACE}
GIT_LFS_SKIP_SMUDGE: 1 # Avoid fetching the files from LFS when doing git fetch
services:
- docker:dind
stages:
- precheck
- build
- postcheck
before_script:
- apk add --update bash coreutils tar wget
- docker login tecnalia-docker-dev.artifact.tecnalia.com -u ${ARTIFACT_CI_USER} -p ${ARTIFACT_CI_TOKEN}
.industrial_ci:
stage: build
before_script:
- apk add --update bash coreutils tar
- git clone --quiet --depth 1 https://github.com/ros-industrial/industrial_ci.git .ci_config -b legacy
- docker login ${ARTIFACT_DOCKER_URL} -u ${ARTIFACT_CI_USER} -p ${ARTIFACT_CI_TOKEN}
- docker login tecnalia-docker-dev.artifact.tecnalia.com -u ${ARTIFACT_CI_USER} -p ${ARTIFACT_CI_TOKEN}
script: .ci_config/gitlab.sh
variables:
ADDITIONAL_DEBS: curl
ROS_PARALLEL_JOBS: -j2 # keep this for now, since there seems to be memory limit issues in the ci VM
BEFORE_SCRIPT: bash <(curl -Ls https://git.code.tecnalia.com/tecnalia_robotics-public/gitlab_templates/raw/master/scripts/ci_run_entry_points.sh) $$CATKIN_WORKSPACE
CMAKE_ARGS: -DCMAKE_CXX_FLAGS=-Wno-ignored-attributes -Wno-int-in-bool-context
when: always
tags:
- docker
industrial_ci_kinetic:
extends: .industrial_ci
variables:
ROS_DISTRO: kinetic
DOCKER_IMAGE: "${ARTIFACT_DOCKER_URL}/flexbotics-base-devel:${ROS_DISTRO}"
except:
- tags
- melodic-devel
industrial_ci_melodic:
extends: .industrial_ci
variables:
ROS_DISTRO: melodic
DOCKER_IMAGE: "${ARTIFACT_DOCKER_URL}/flexbotics-base-devel:${ROS_DISTRO}"
only:
- melodic-devel
except:
- tags
industrial_ci_kinetic_deploy:
extends: .industrial_ci
variables:
ROS_DISTRO: kinetic
DOCKER_IMAGE: "${ARTIFACT_DOCKER_URL}/flexbotics-base-devel:${ROS_DISTRO}"
AFTER_SCRIPT: "wget -q -O - https://git.code.tecnalia.com/tecnalia_robotics-public/gitlab_templates/raw/master/scripts/pre-deploy-cleanup.bash | bash"
DOCKER_COMMIT: industrial_ci_image
script:
- .ci_config/gitlab.sh
- cd $(mktemp -d)
- wget https://git.code.tecnalia.com/tecnalia_robotics-public/gitlab_templates/-/archive/master/gitlab_templates-master.zip
- unzip gitlab_templates-master.zip && cd gitlab_templates-master/deploy
- cd base
- chmod a+x ros_entrypoint.bash
- docker build -t ${ARTIFACT_DOCKER_URL}/${CI_PROJECT_NAME}:${CI_COMMIT_REF_NAME} .
- docker push ${ARTIFACT_DOCKER_URL}/${CI_PROJECT_NAME}:${CI_COMMIT_REF_NAME}
- cd ..
- cd dev
- docker build --build-arg APPLICATION_IMAGE=${ARTIFACT_DOCKER_URL}/${CI_PROJECT_NAME}:${CI_COMMIT_REF_NAME} -t ${ARTIFACT_DOCKER_URL}/${CI_PROJECT_NAME}:${CI_COMMIT_REF_NAME}-dev .
- docker push ${ARTIFACT_DOCKER_URL}/${CI_PROJECT_NAME}:${CI_COMMIT_REF_NAME}-dev
- cd ..
only:
- tags
clang_format:
stage: postcheck
before_script:
- apk add --update bash coreutils tar wget
- git clone --quiet --depth 1 https://github.com/ros-industrial/industrial_ci.git .ci_config -b legacy
- wget https://git.code.tecnalia.com/tecnalia_robotics-public/gitlab_templates/raw/master/clang-format -O .clang-format
script: .ci_config/gitlab.sh
variables:
ROS_DISTRO: kinetic
CLANG_FORMAT_CHECK: file
CLANG_FORMAT_VERSION: "3.8"
when: always
tags:
- docker
##########################################################
## Simple check for basic Python syntax errors; this is much less thorough than
## other options like pylint, but much faster and less strict
##########################################################
python_syntax:
before_script:
- apk add --update python2
stage: precheck
script: python -m compileall -q .
when: always
tags:
- docker
##########################################################
## Check for errors in bash/sh scripts
## Uses https://github.com/koalaman/shellcheck
##########################################################
bash_syntax:
before_script:
- apk add --update shellcheck
stage: postcheck
script:
- RET=0
- find . -type f -name '*.sh' -exec shellcheck -s sh {} + || RET=1
- find . -type f -name '*.bash' -exec shellcheck -s bash {} + || RET=1
- exit $RET
when: always
tags:
- docker
##########################################################
## Check for errors in markdown files
## Uses https://github.com/cytopia/awesome-ci
## Disables the line length rule with --custom="-r ..."
## See other rules that can be configured in
## https://github.com/markdownlint/markdownlint/blob/master/docs/RULES.md
##########################################################
markdown_syntax:
stage: postcheck
script: docker run -v ${PWD}:/ac tecnalia-docker-dev.artifact.tecnalia.com/cytopia/awesome-ci syntax-markdown --path=/ac --extension=md --ignore=.ci_config --custom="-r ~MD013,~MD014"
when: always
tags:
- docker
##########################################################
## Check for various syntax issues in files; currently
## trailing white spaces
## utf8 formatting
## Uses https://github.com/cytopia/awesome-ci
##########################################################
file_syntax:
stage: postcheck
script:
- RET=0
- docker run -v ${PWD}:/ac tecnalia-docker-dev.artifact.tecnalia.com/cytopia/awesome-ci file-trailing-space --path=/ac --extension=py,txt,cpp,h,md,sh,bash,xml,launch --ignore=.ci_config || RET=1
- docker run -v ${PWD}:/ac tecnalia-docker-dev.artifact.tecnalia.com/cytopia/awesome-ci file-utf8 --path=/ac --extension=py,txt,cpp,h,md,sh,bash,xml,launch --ignore=.ci_config || RET=1
- exit $RET
when: always
tags:
- docker
##########################################################
## Check for errors in Asciidoctor files
## Could not find a proper linter for Asciidoctor format,
## so this is a custom-made "dirty" script
##########################################################
adoc_syntax:
stage: postcheck
image: alpine
before_script:
- apk add --update curl asciidoctor bash
script:
- curl -Ls https://git.code.tecnalia.com/tecnalia_robotics-public/gitlab_templates/raw/master/scripts/asciidoctor_syntax_check.bash | bash -s -- .
tags:
- docker
when: always
include:
- project: tecnalia_robotics-public/gitlab_templates
file: ci-templates/auto-rules/kinetic-default.yml
image: docker:git
variables:
TMPDIR: /builds/${CI_PROJECT_NAMESPACE}
GIT_LFS_SKIP_SMUDGE: 1 # Avoid fetching the files from LFS when doing git fetch
services:
- docker:dind
stages:
- precheck
- build
- postcheck
before_script:
- apk add --update bash coreutils tar wget
- docker login tecnalia-docker-dev.artifact.tecnalia.com -u ${ARTIFACT_CI_USER} -p ${ARTIFACT_CI_TOKEN}
.industrial_ci:
stage: build
before_script:
- apk add --update bash coreutils tar
- git clone --quiet --depth 1 https://github.com/ros-industrial/industrial_ci.git .ci_config -b legacy
- docker login ${ARTIFACT_DOCKER_URL} -u ${ARTIFACT_CI_USER} -p ${ARTIFACT_CI_TOKEN}
- docker login tecnalia-docker-dev.artifact.tecnalia.com -u ${ARTIFACT_CI_USER} -p ${ARTIFACT_CI_TOKEN}
script: .ci_config/gitlab.sh
variables:
ADDITIONAL_DEBS: curl
ROS_PARALLEL_JOBS: "-j2" # keep this for now, since there seems to be memory limit issues in the ci VM
UPSTREAM_WORKSPACE: file
ROSINSTALL_FILENAME: ".rosinstall"
BEFORE_SCRIPT: "bash <(curl -Ls https://git.code.tecnalia.com/tecnalia_robotics-public/gitlab_templates/raw/master/scripts/ci_run_entry_points.sh) $$CATKIN_WORKSPACE"
CATKIN_CONFIG: '--cmake-args -DCMAKE_CXX_FLAGS=-Wno-ignored-attributes -Wno-int-in-bool-context --'
when: always
tags:
- docker
industrial_ci_kinetic:
extends: .industrial_ci
variables:
ROS_DISTRO: kinetic
DOCKER_IMAGE: "${ARTIFACT_DOCKER_URL}/flexbotics-base-devel:${ROS_DISTRO}"
except:
- tags
- melodic-devel
industrial_ci_melodic:
extends: .industrial_ci
variables:
ROS_DISTRO: melodic
DOCKER_IMAGE: "${ARTIFACT_DOCKER_URL}/flexbotics-base-devel:${ROS_DISTRO}"
only:
- melodic-devel
except:
- tags
industrial_ci_kinetic_deploy:
extends: .industrial_ci
variables:
ROS_DISTRO: kinetic
DOCKER_IMAGE: "${ARTIFACT_DOCKER_URL}/flexbotics-base-devel:${ROS_DISTRO}"
AFTER_SCRIPT: "wget -q -O - https://git.code.tecnalia.com/tecnalia_robotics-public/gitlab_templates/raw/master/scripts/pre-deploy-cleanup.bash | bash"
DOCKER_COMMIT: industrial_ci_image
script:
- .ci_config/gitlab.sh
- cd $(mktemp -d)
- wget https://git.code.tecnalia.com/tecnalia_robotics-public/gitlab_templates/-/archive/master/gitlab_templates-master.zip
- unzip gitlab_templates-master.zip && cd gitlab_templates-master/deploy
- cd base
- chmod a+x ros_entrypoint.bash
- docker build -t ${ARTIFACT_DOCKER_URL}/${CI_PROJECT_NAME}:${CI_COMMIT_REF_NAME} .
- docker push ${ARTIFACT_DOCKER_URL}/${CI_PROJECT_NAME}:${CI_COMMIT_REF_NAME}
- cd ..
- cd dev
- docker build --build-arg APPLICATION_IMAGE=${ARTIFACT_DOCKER_URL}/${CI_PROJECT_NAME}:${CI_COMMIT_REF_NAME} -t ${ARTIFACT_DOCKER_URL}/${CI_PROJECT_NAME}:${CI_COMMIT_REF_NAME}-dev .
- docker push ${ARTIFACT_DOCKER_URL}/${CI_PROJECT_NAME}:${CI_COMMIT_REF_NAME}-dev
- cd ..
only:
- tags
clang_format:
stage: postcheck
before_script:
- apk add --update bash coreutils tar wget
- git clone --quiet --depth 1 https://github.com/ros-industrial/industrial_ci.git .ci_config -b legacy
- wget https://git.code.tecnalia.com/tecnalia_robotics-public/gitlab_templates/raw/master/clang-format -O .clang-format
script: .ci_config/gitlab.sh
variables:
ROS_DISTRO: kinetic
CLANG_FORMAT_CHECK: file
CLANG_FORMAT_VERSION: "3.8"
when: always
tags:
- docker
##########################################################
## Simple check for basic Python syntax errors; this is much less thorough than
## other options like pylint, but much faster and less strict
##########################################################
python_syntax:
before_script:
- apk add --update python2
stage: precheck
script: python -m compileall -q .
when: always
tags:
- docker
##########################################################
## Check for errors in bash/sh scripts
## Uses https://github.com/koalaman/shellcheck
##########################################################
bash_syntax:
before_script:
- apk add --update shellcheck
stage: postcheck
script:
- RET=0
- find . -type f -name '*.sh' -exec shellcheck -s sh {} + || RET=1
- find . -type f -name '*.bash' -exec shellcheck -s bash {} + || RET=1
- exit $RET
when: always
tags:
- docker
##########################################################
## Check for errors in markdown files
## Uses https://github.com/cytopia/awesome-ci
## Disables the line length rule with --custom="-r ..."
## See other rules that can be configured in
## https://github.com/markdownlint/markdownlint/blob/master/docs/RULES.md
##########################################################
markdown_syntax:
stage: postcheck
script: docker run -v ${PWD}:/ac tecnalia-docker-dev.artifact.tecnalia.com/cytopia/awesome-ci syntax-markdown --path=/ac --extension=md --ignore=.ci_config --custom="-r ~MD013,~MD014"
when: always
tags:
- docker
##########################################################
## Check for various syntax issues in files; currently
## trailing white spaces
## utf8 formatting
## Uses https://github.com/cytopia/awesome-ci
##########################################################
file_syntax:
stage: postcheck
script:
- RET=0
- docker run -v ${PWD}:/ac tecnalia-docker-dev.artifact.tecnalia.com/cytopia/awesome-ci file-trailing-space --path=/ac --extension=py,txt,cpp,h,md,sh,bash,xml,launch --ignore=.ci_config || RET=1
- docker run -v ${PWD}:/ac tecnalia-docker-dev.artifact.tecnalia.com/cytopia/awesome-ci file-utf8 --path=/ac --extension=py,txt,cpp,h,md,sh,bash,xml,launch --ignore=.ci_config || RET=1
- exit $RET
when: always
tags:
- docker
##########################################################
## Check for errors in Asciidoctor files
## Could not find a proper linter for Asciidoctor format,
## so this is a custom-made "dirty" script
##########################################################
adoc_syntax:
stage: postcheck
image: alpine
before_script:
- apk add --update curl asciidoctor bash
script:
- curl -Ls https://git.code.tecnalia.com/tecnalia_robotics-public/gitlab_templates/raw/master/scripts/asciidoctor_syntax_check.bash | bash -s -- .
tags:
- docker
when: always
include:
- project: tecnalia_robotics-public/gitlab_templates
file: ci-templates/auto-rules/with-rosinstall/kinetic-default.yml
= GitLab Templates
:toc:
This repository contains miscellaneous resources that need to be accessible without authentication.
== GitLab CI templates
Two templates with default CI pipeline configurations are provided, for any repo to make use of:
- `.gitlab-industrial-ci-kinetic-no-rosinstall.yml`
- `.gitlab-industrial-ci-kinetic-with-rosinstall.yml`
Both define basically the same pipeline, consisting of the following stages and jobs:
- Stage: `precheck`
* Check python syntax
- Stage: `build`
* Run `industrial_ci`
** Uses `flexbotics-base-devel` as a starting image
** Runs link:scripts/ci_run_entry_points.sh[] in the `BEFORE_SCRIPT` stage, which in turn finds and runs every `ci_entry_script.bash` script found in the source tree (target repo + repos from `.rosinstall`)
* Only when tags are pushed:
** Run some cleanup in the `industrial_ci` context (e.g. delete sources)
** Commit image used by `industrial_ci`
** Build deployment image on top of it and push it to our registry
- Stage: `postcheck` (jobs in this stage are allowed to fail)
* Check `bash`/`sh` script syntax
* Check Markdown syntax
* Check files have no trailing whitespace and are UTF-8 encoded
It is important to note that both images set the variable `GIT_LFS_SKIP_SMUDGE`, which means that when the repositories are cloned/fetched, the files hosted using Git LFS will not be downloaded. An example on how to explicitely fetch the files from LFS can be found in the https://git.code.tecnalia.com/tecnalia_robotics/common_config/blob/e93dbe6379d94e6a895a33c59bf17ccab26eacf9/.gitlab-ci.yml#L132[common_config repository].
include::ci-templates/README.adoc[leveloffset=+1]
== Custom `rosdep` definitions
Custom `rosdep` definitions for the packages hosted in our internal debian repository are declared in link:rosdistro/rosdep_tecnalia.yaml[].
== Deployment image build resources
Resources required for building the deployment docker images are also hosted in this repository.
This basically boils down to:
- link:scripts/pre-deploy-cleanup.bash[] is run in the `industrial_ci` container before commiting to an image
- link:deploy/Dockerfile[] is used to perform some additional cleanup and install link:deploy/ros_entrypoint.bash[] as entrypoint script on top of the `industrial_ci` image
= GitLab CI templates
Templates for GitLab Continuous Integration are provided in the folder link:ci-templates[]. These can be included from any repo by using `include:project`:
```yml
include::examples/simple_include.yml[]
```
== Core Pipeline
The Core pipeline, defined in link:ci-templates/core.yml[], has the following structure:
- Stage: `build`
* Check python syntax
* `.industrial_ci` template (see https://github.com/ros-industrial/industrial_ci[the industrial_ci repo]), which can be used to define jobs that build `ROS` packages
** Needs the `ROS_DISTRO` variable to be defined
** If the package has `.rosinstall` dependencies, they can be installed using the variable `UPSTREAM_WORKSPACE`
** Uses `flexbotics-base-devel:${ROS_DISTRO}` as a starting image
** Runs link:scripts/ci_run_entry_points.sh[] in the `AFTER_SETUP_TARGET_WORKSPACE` stage, which in turn finds and runs every `ci_entry_script.bash` script found in the target repo.
- Stage: `deploy`
* `.ddeploy` template (see https://git.code.tecnalia.com/tecnalia_robotics/flexbotics/flexbotics_utils/ddeploy[the ddeploy repo]), which:
** Is run automatically for tags, but can also be run manually on normal commits
** Calls `ddeploy` to generate the docker image
** Renames the image created by ddeploy with different tags and pushes them to the registry.
*** Tag `${COMMIT_SHA}` (hash of commit)
*** Tag `${CI_COMMIT_REF_NAME}` (branch or tag name)
*** Tag `latest` only on the default branch
- Stage: `.post`
* Check `bash`/`sh` script syntax
* Check Markdown and AsciiDoc syntax
* Check files have no trailing whitespace and are `UTF-8` encoded
* Check that `C++` files follow this link:clang-format[]
== Manual Jobs
Using the templates defined by the core pipeline, one can easily define jobs to build the packages in the repo.
```yml
include::examples/manual_job.yml[]
```
More examples can be found in link:ci-templates/examples[].
== Auto Rules
In the link:ci-templates/auto-rules[] folder, automatic pipelines are defined that will try to detect the ROS distro from the branch name and create jobs.
For example, if the branch starts with `noetic-` it will build for noetic.
If the branch doesn't start with any ROS distro, it will build the default distro, which can be specified with the variable `DEFAULT_DISTRO`.
For convenience, some templates are provided that already set this variable.
If your repo needs to uses `.rosinstall`, you can use the templates in the link:ci-templates/auto-rules/with-rosinstall[] folder.
The auto rules also allow for using variables to explicitely activate each job:
```yml
include::examples/build_vars.yml[]
```
If a `ddeploy.yaml` file exists, the auto-rules will add a job extending the `.ddeploy` template defined
== LFS
By default, lfs files are not downloaded. If you need to add LFS, include the link:ci-templates/lfs-pull.yml[] along with anything else:
```yml
include::examples/include_lfs.yml[]
```
\ No newline at end of file
include: ci-templates/auto-rules/no-default.yml
variables:
DEFAULT_DISTRO: humble
include: ci-templates/auto-rules/no-default.yml
variables:
DEFAULT_DISTRO: kinetic
include: ci-templates/auto-rules/no-default.yml
variables:
DEFAULT_DISTRO: melodic
include: ci-templates/core.yml
workflow:
rules:
- if: $CI_COMMIT_REF_NAME =~ /^kinetic-.*/
variables:
DEFAULT_DISTRO: "kinetic"
- if: $CI_COMMIT_REF_NAME =~ /^melodic-.*/
variables:
DEFAULT_DISTRO: "melodic"
- if: $CI_COMMIT_REF_NAME =~ /^noetic-.*/
variables:
DEFAULT_DISTRO: "noetic"
- if: $CI_COMMIT_REF_NAME =~ /^humble-.*/
variables:
DEFAULT_DISTRO: "humble"
- if: $CI_COMMIT_TAG
- if: $CI_COMMIT_BRANCH
industrial_ci_kinetic:
extends: .industrial_ci
variables:
ROS_DISTRO: kinetic
rules:
- if: $DEFAULT_DISTRO == "kinetic"
- if: $BUILD_KINETIC
industrial_ci_melodic:
extends: .industrial_ci
variables:
ROS_DISTRO: melodic
rules:
- if: $DEFAULT_DISTRO == "melodic"
- if: $BUILD_MELODIC
industrial_ci_noetic:
extends: .industrial_ci
variables:
ROS_DISTRO: noetic
rules:
- if: $DEFAULT_DISTRO == "noetic"
- if: $BUILD_NOETIC
industrial_ci_humble:
extends: .industrial_ci
variables:
ROS_DISTRO: humble
rules:
- if: $DEFAULT_DISTRO == "humble"
- if: $BUILD_HUMBLE
ddeploy:
extends: .ddeploy
rules:
- if: $CI_COMMIT_TAG
exists:
- ddeploy.yaml
when: always
- exists:
- ddeploy.yaml
when: manual
allow_failure: true
include: ci-templates/auto-rules/no-default.yml
variables:
DEFAULT_DISTRO: noetic
include: ci-templates/auto-rules/with-rosinstall/no-default.yml
variables:
DEFAULT_DISTRO: humble
include: ci-templates/auto-rules/with-rosinstall/no-default.yml
variables:
DEFAULT_DISTRO: kinetic
include: ci-templates/auto-rules/with-rosinstall/no-default.yml
variables:
DEFAULT_DISTRO: melodic
include: ci-templates/auto-rules/no-default.yml
.industrial_ci:
variables:
UPSTREAM_WORKSPACE: .rosinstall
include: ci-templates/auto-rules/with-rosinstall/no-default.yml
variables:
DEFAULT_DISTRO: noetic
image: tecnalia-docker-dev.artifact.tecnalia.com/docker:git
workflow:
rules:
- if: $CI_COMMIT_TAG
- if: $CI_COMMIT_BRANCH
variables:
TMPDIR: /builds/${CI_PROJECT_NAMESPACE}
GIT_LFS_SKIP_SMUDGE: 1 # Avoid fetching the files from LFS when doing git fetch
stages:
- build
- deploy
include:
- ci-templates/syntax-check.yml
- ci-templates/industrial-ci-templates.yml
default:
tags:
- docker
include:
- project: tecnalia_robotics-public/gitlab_templates
file: ci-templates/auto-rules/no-default.yml
variables:
BUILD_KINETIC: 1
BUILD_MELODIC: 1
BUILD_NOETIC: 1
include:
- project: tecnalia_robotics-public/gitlab_templates
file:
- ci-templates/auto-rules/kinetic-default.yml
- ci-templates/lfs-pull.yml
include:
- project: tecnalia_robotics-public/gitlab_templates
file: ci-templates/core.yml
industrial_ci_kinetic:
extends: .industrial_ci
variables:
ROS_DISTRO: kinetic
UPSTREAM_WORKSPACE: .rosinstall
include:
- project: tecnalia_robotics-public/gitlab_templates
file: ci-templates/core.yml