Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • miguel.prada/gitlab_templates
  • tecnalia_robotics-public/gitlab_templates
2 results
Show changes
Commits on Source (203)
Showing
with 315 additions and 323 deletions
shellcheck:
image: koalaman/shellcheck-alpine
script: shellcheck deploy/base/ros_entrypoint.bash scripts/*sh
image: tecnalia-docker-dev.artifact.tecnalia.dev/koalaman/shellcheck-alpine
script: shellcheck scripts/*sh
tags:
- docker
# The ignored rules in hadolint checks are
# DL3006 Always tag the version of an image explicitly
# One of the Dockerfiles uses a temporary image which isn't important to tag
# and the other uses a tagged image through an ARG, which apparently is not
# properly recognized.
# DL3008 Pin versions in apt-get install
# A bit overkill
dockerfiles:
image: hadolint/hadolint:latest-debian
script:
- RET=0
- hadolint --ignore DL3006 --ignore DL3008 deploy/base/Dockerfile || RET=1
- hadolint --ignore DL3006 --ignore DL3008 deploy/dev/Dockerfile || RET=1
- exit $RET
tags:
- docker
yamllint:
image: sdesbure/yamllint
image: tecnalia-docker-dev.artifact.tecnalia.dev/sdesbure/yamllint
script: yamllint $(find . -name '*.yml' -o -name '*.yaml')
tags:
- docker
deploy_devcontainer:
script:
- tar cvfz dot_devcontainer.tar.gz -C vscode_devcontainer .devcontainer/
artifacts:
paths:
- dot_devcontainer.tar.gz
expire_in: 1 year
tags:
- docker
.simple_ros_test:
variables:
TEMPLATES_PIPELINE_REF: $CI_COMMIT_REF_NAME
trigger:
project: tecnalia_robotics/ci_test_repos/simple_ros_test
strategy: depend
simple_ros_melodic_test:
extends: .simple_ros_test
variables:
TEMPLATE_TO_USE: ci-templates/auto-rules/melodic-default.yml
simple_ros_noetic_test:
extends: .simple_ros_test
variables:
TEMPLATE_TO_USE: ci-templates/auto-rules/noetic-default.yml
simple_ros_core_with_noetic_test:
extends: .simple_ros_test
variables:
TEMPLATE_TO_USE: test/core_with_noetic.yml
simple_ros_auto_rules_melodic_noetic_test:
extends: .simple_ros_test
variables:
TEMPLATE_TO_USE: test/auto_rules_melodic_noetic.yml
image: docker:git
variables:
TMPDIR: /builds/${CI_PROJECT_NAMESPACE}
GIT_LFS_SKIP_SMUDGE: 1 # Avoid fetching the files from LFS when doing git fetch
services:
- docker:dind
stages:
- precheck
- build
- postcheck
before_script:
- apk add --update bash coreutils tar wget python
.industrial_ci:
stage: build
before_script:
- apk add --update bash coreutils tar
- git clone https://github.com/ros-industrial/industrial_ci.git .ci_config
- docker login ${ARTIFACT_DOCKER_URL} -u ${ARTIFACT_CI_USER} -p ${ARTIFACT_CI_TOKEN}
script: .ci_config/gitlab.sh
variables:
ADDITIONAL_DEBS: curl
ROS_PARALLEL_JOBS: -j2 # keep this for now, since there seems to be memory limit issues in the ci VM
BEFORE_SCRIPT: bash <(curl -Ls https://git.code.tecnalia.com/tecnalia_robotics-public/gitlab_templates/raw/master/scripts/ci_run_entry_points.sh) $$CATKIN_WORKSPACE
CMAKE_ARGS: -DCMAKE_CXX_FLAGS=-Wno-ignored-attributes -Wno-int-in-bool-context
when: always
tags:
- docker
industrial_ci_kinetic:
extends: .industrial_ci
variables:
ROS_DISTRO: kinetic
DOCKER_IMAGE: "${ARTIFACT_DOCKER_URL}/flexbotics-base-devel:${ROS_DISTRO}"
except:
- tags
- melodic-devel
industrial_ci_melodic:
extends: .industrial_ci
variables:
ROS_DISTRO: melodic
DOCKER_IMAGE: "${ARTIFACT_DOCKER_URL}/flexbotics-base-devel:${ROS_DISTRO}"
only:
- melodic-devel
except:
- tags
allow_failure: yes
industrial_ci_kinetic_deploy:
extends: .industrial_ci
variables:
ROS_DISTRO: kinetic
DOCKER_IMAGE: "${ARTIFACT_DOCKER_URL}/flexbotics-base-devel:${ROS_DISTRO}"
AFTER_SCRIPT: "wget -q -O - https://git.code.tecnalia.com/tecnalia_robotics-public/gitlab_templates/raw/master/scripts/pre-deploy-cleanup.bash | bash"
DOCKER_COMMIT: industrial_ci_image
script:
- .ci_config/gitlab.sh
- cd $(mktemp -d)
- wget https://git.code.tecnalia.com/tecnalia_robotics-public/gitlab_templates/-/archive/master/gitlab_templates-master.zip
- unzip gitlab_templates-master.zip && cd gitlab_templates-master/deploy
- cd base
- chmod a+x ros_entrypoint.bash
- docker build -t ${ARTIFACT_DOCKER_URL}/${CI_PROJECT_NAME}:${CI_COMMIT_REF_NAME} .
- docker push ${ARTIFACT_DOCKER_URL}/${CI_PROJECT_NAME}:${CI_COMMIT_REF_NAME}
- cd ..
- cd dev
- docker build --build-arg APPLICATION_IMAGE=${ARTIFACT_DOCKER_URL}/${CI_PROJECT_NAME}:${CI_COMMIT_REF_NAME} -t ${ARTIFACT_DOCKER_URL}/${CI_PROJECT_NAME}:${CI_COMMIT_REF_NAME}-dev .
- docker push ${ARTIFACT_DOCKER_URL}/${CI_PROJECT_NAME}:${CI_COMMIT_REF_NAME}-dev
- cd ..
only:
- tags
##########################################################
## Simple check for basic Python syntax errors; this is much less thorough than
## other options like pylint, but much faster and less strict
##########################################################
python_syntax:
stage: precheck
script: python -m compileall -q .
when: always
tags:
- docker
##########################################################
## Check for errors in bash/sh scripts
## Uses https://github.com/cytopia/awesome-ci
##########################################################
bash_syntax:
stage: postcheck
script:
- RET=0
- docker run -v ${PWD}:/ac cytopia/awesome-ci syntax-sh --path=/ac --extension=sh --ignore=.ci_config || RET=1
- docker run -v ${PWD}:/ac cytopia/awesome-ci syntax-bash --path=/ac --extension=bash --ignore=.ci_config || RET=1
- exit $RET
allow_failure: yes
when: always
tags:
- docker
##########################################################
## Check for errors in markdown files
## Uses https://github.com/cytopia/awesome-ci
## Disables the line length rule with --custom="-r ..."
## See other rules that can be configured in
## https://github.com/markdownlint/markdownlint/blob/master/docs/RULES.md
##########################################################
markdown_syntax:
stage: postcheck
script: docker run -v ${PWD}:/ac cytopia/awesome-ci syntax-markdown --path=/ac --extension=md --ignore=.ci_config --custom="-r ~MD013"
allow_failure: yes
when: always
tags:
- docker
##########################################################
## Check for various syntax issues in files; currently
## trailing white spaces
## utf8 formatting
## Uses https://github.com/cytopia/awesome-ci
##########################################################
file_syntax:
stage: postcheck
script:
- RET=0
- docker run -v ${PWD}:/ac cytopia/awesome-ci file-trailing-space --path=/ac --extension=py,txt,cpp,h,md,sh,bash,xml,launch --ignore=.ci_config || RET=1
- docker run -v ${PWD}:/ac cytopia/awesome-ci file-utf8 --path=/ac --extension=py,txt,cpp,h,md,sh,bash,xml,launch --ignore=.ci_config || RET=1
- exit $RET
when: always
tags:
- docker
include:
- project: tecnalia_robotics-public/gitlab_templates
file: ci-templates/auto-rules/kinetic-default.yml
image: docker:git
variables:
TMPDIR: /builds/${CI_PROJECT_NAMESPACE}
GIT_LFS_SKIP_SMUDGE: 1 # Avoid fetching the files from LFS when doing git fetch
services:
- docker:dind
stages:
- precheck
- build
- postcheck
before_script:
- apk add --update bash coreutils tar wget python
.industrial_ci:
stage: build
before_script:
- apk add --update bash coreutils tar
- git clone https://github.com/ros-industrial/industrial_ci.git .ci_config
- docker login ${ARTIFACT_DOCKER_URL} -u ${ARTIFACT_CI_USER} -p ${ARTIFACT_CI_TOKEN}
script: .ci_config/gitlab.sh
variables:
ADDITIONAL_DEBS: curl
ROS_PARALLEL_JOBS: "-j2" # keep this for now, since there seems to be memory limit issues in the ci VM
UPSTREAM_WORKSPACE: file
ROSINSTALL_FILENAME: ".rosinstall"
BEFORE_SCRIPT: "bash <(curl -Ls https://git.code.tecnalia.com/tecnalia_robotics-public/gitlab_templates/raw/master/scripts/ci_run_entry_points.sh) $$CATKIN_WORKSPACE"
CATKIN_CONFIG: '--cmake-args -DCMAKE_CXX_FLAGS=-Wno-ignored-attributes -Wno-int-in-bool-context --'
when: always
tags:
- docker
industrial_ci_kinetic:
extends: .industrial_ci
variables:
ROS_DISTRO: kinetic
DOCKER_IMAGE: "${ARTIFACT_DOCKER_URL}/flexbotics-base-devel:${ROS_DISTRO}"
except:
- tags
- melodic-devel
industrial_ci_melodic:
extends: .industrial_ci
variables:
ROS_DISTRO: melodic
DOCKER_IMAGE: "${ARTIFACT_DOCKER_URL}/flexbotics-base-devel:${ROS_DISTRO}"
only:
- melodic-devel
except:
- tags
allow_failure: yes
industrial_ci_kinetic_deploy:
extends: .industrial_ci
variables:
ROS_DISTRO: kinetic
DOCKER_IMAGE: "${ARTIFACT_DOCKER_URL}/flexbotics-base-devel:${ROS_DISTRO}"
AFTER_SCRIPT: "wget -q -O - https://git.code.tecnalia.com/tecnalia_robotics-public/gitlab_templates/raw/master/scripts/pre-deploy-cleanup.bash | bash"
DOCKER_COMMIT: industrial_ci_image
script:
- .ci_config/gitlab.sh
- cd $(mktemp -d)
- wget https://git.code.tecnalia.com/tecnalia_robotics-public/gitlab_templates/-/archive/master/gitlab_templates-master.zip
- unzip gitlab_templates-master.zip && cd gitlab_templates-master/deploy
- cd base
- chmod a+x ros_entrypoint.bash
- docker build -t ${ARTIFACT_DOCKER_URL}/${CI_PROJECT_NAME}:${CI_COMMIT_REF_NAME} .
- docker push ${ARTIFACT_DOCKER_URL}/${CI_PROJECT_NAME}:${CI_COMMIT_REF_NAME}
- cd ..
- cd dev
- docker build --build-arg APPLICATION_IMAGE=${ARTIFACT_DOCKER_URL}/${CI_PROJECT_NAME}:${CI_COMMIT_REF_NAME} -t ${ARTIFACT_DOCKER_URL}/${CI_PROJECT_NAME}:${CI_COMMIT_REF_NAME}-dev .
- docker push ${ARTIFACT_DOCKER_URL}/${CI_PROJECT_NAME}:${CI_COMMIT_REF_NAME}-dev
- cd ..
only:
- tags
##########################################################
## Simple check for basic Python syntax errors; this is much less thorough than
## other options like pylint, but much faster and less strict
##########################################################
python_syntax:
stage: precheck
script: python -m compileall -q .
when: always
tags:
- docker
##########################################################
## Check for errors in bash/sh scripts
## Uses https://github.com/cytopia/awesome-ci
##########################################################
bash_syntax:
stage: postcheck
script:
- RET=0
- docker run -v ${PWD}:/ac cytopia/awesome-ci syntax-sh --path=/ac --extension=sh --ignore=.ci_config || RET=1
- docker run -v ${PWD}:/ac cytopia/awesome-ci syntax-bash --path=/ac --extension=bash --ignore=.ci_config || RET=1
- exit $RET
allow_failure: yes
when: always
tags:
- docker
##########################################################
## Check for errors in markdown files
## Uses https://github.com/cytopia/awesome-ci
## Disables the line length rule with --custom="-r ..."
## See other rules that can be configured in
## https://github.com/markdownlint/markdownlint/blob/master/docs/RULES.md
##########################################################
markdown_syntax:
stage: postcheck
script: docker run -v ${PWD}:/ac cytopia/awesome-ci syntax-markdown --path=/ac --extension=md --ignore=.ci_config --custom="-r ~MD013"
allow_failure: yes
when: always
tags:
- docker
##########################################################
## Check for various syntax issues in files; currently
## trailing white spaces
## utf8 formatting
## Uses https://github.com/cytopia/awesome-ci
##########################################################
file_syntax:
stage: postcheck
script:
- RET=0
- docker run -v ${PWD}:/ac cytopia/awesome-ci file-trailing-space --path=/ac --extension=py,txt,cpp,h,md,sh,bash,xml,launch --ignore=.ci_config || RET=1
- docker run -v ${PWD}:/ac cytopia/awesome-ci file-utf8 --path=/ac --extension=py,txt,cpp,h,md,sh,bash,xml,launch --ignore=.ci_config || RET=1
- exit $RET
when: always
tags:
- docker
include:
- project: tecnalia_robotics-public/gitlab_templates
file: ci-templates/auto-rules/with-rosinstall/kinetic-default.yml
= GitLab Templates
:toc:
This repository contains miscellaneous resources that need to be accessible without authentication.
== GitLab CI templates
Two templates with default CI pipeline configurations are provided, for any repo to make use of:
- `.gitlab-industrial-ci-kinetic-no-rosinstall.yml`
- `.gitlab-industrial-ci-kinetic-with-rosinstall.yml`
Both define basically the same pipeline, consisting of the following stages and jobs:
- Stage: `precheck`
* Check python syntax
- Stage: `build`
* Run `industrial_ci`
** Uses `flexbotics-base-devel` as a starting image
** Runs link:scripts/ci_run_entry_points.sh[] in the `BEFORE_SCRIPT` stage, which in turn finds and runs every `ci_entry_script.bash` script found in the source tree (target repo + repos from `.rosinstall`)
* Only when tags are pushed:
** Run some cleanup in the `industrial_ci` context (e.g. delete sources)
** Commit image used by `industrial_ci`
** Build deployment image on top of it and push it to our registry
- Stage: `postcheck` (jobs in this stage are allowed to fail)
* Check `bash`/`sh` script syntax
* Check Markdown syntax
* Check files have no trailing whitespace and are UTF-8 encoded
It is important to note that both images set the variable `GIT_LFS_SKIP_SMUDGE`, which means that when the repositories are cloned/fetched, the files hosted using Git LFS will not be downloaded. An example on how to explicitely fetch the files from LFS can be found in the https://git.code.tecnalia.com/tecnalia_robotics/common_config/blob/e93dbe6379d94e6a895a33c59bf17ccab26eacf9/.gitlab-ci.yml#L132[common_config repository].
include::ci-templates/README.adoc[leveloffset=+1]
== Custom `rosdep` definitions
Custom `rosdep` definitions for the packages hosted in our internal debian repository are declared in link:rosdistro/rosdep_tecnalia.yaml[].
== Deployment image build resources
Resources required for building the deployment docker images are also hosted in this repository.
This basically boils down to:
- link:scripts/pre-deploy-cleanup.bash[] is run in the `industrial_ci` container before commiting to an image
- link:deploy/Dockerfile[] is used to perform some additional cleanup and install link:deploy/ros_entrypoint.bash[] as entrypoint script on top of the `industrial_ci` image
= GitLab CI templates
Templates for GitLab Continuous Integration are provided in the folder link:ci-templates[]. These can be included from any repo by using `include:project`:
```yml
include::examples/simple_include.yml[]
```
== Core Pipeline
The Core pipeline, defined in link:ci-templates/core.yml[], has the following structure:
- Stage: `build`
* Check python syntax
* `.industrial_ci` template (see https://github.com/ros-industrial/industrial_ci[the industrial_ci repo]), which can be used to define jobs that build `ROS` packages
** Needs the `ROS_DISTRO` variable to be defined
** If the package has `.rosinstall` dependencies, they can be installed using the variable `UPSTREAM_WORKSPACE`
** Uses `flexbotics-base-devel:${ROS_DISTRO}` as a starting image
** Runs link:scripts/ci_run_entry_points.sh[] in the `AFTER_SETUP_TARGET_WORKSPACE` stage, which in turn finds and runs every `ci_entry_script.bash` script found in the target repo.
- Stage: `deploy`
* `.ddeploy` template (see https://git.code.tecnalia.dev/tecnalia_robotics/flexbotics/flexbotics_utils/ddeploy[the ddeploy repo]), which:
** Is run automatically for tags, but can also be run manually on normal commits
** Calls `ddeploy` to generate the docker image
** Renames the image created by ddeploy with different tags and pushes them to the registry.
*** Tag `${COMMIT_SHA}` (hash of commit)
*** Tag `${CI_COMMIT_REF_NAME}` (branch or tag name)
*** Tag `latest` only on the default branch
- Stage: `.post`
* Check `bash`/`sh` script syntax
* Check Markdown and AsciiDoc syntax
* Check files have no trailing whitespace and are `UTF-8` encoded
* Check that `C++` files follow this link:clang-format[]
== Manual Jobs
Using the templates defined by the core pipeline, one can easily define jobs to build the packages in the repo.
```yml
include::examples/manual_job.yml[]
```
More examples can be found in link:ci-templates/examples[].
== Auto Rules
In the link:ci-templates/auto-rules[] folder, automatic pipelines are defined that will try to detect the ROS distro from the branch name and create jobs.
For example, if the branch starts with `noetic-` it will build for noetic.
If the branch doesn't start with any ROS distro, it will build the default distro, which can be specified with the variable `DEFAULT_DISTRO`.
For convenience, some templates are provided that already set this variable.
If your repo needs to uses `.rosinstall`, you can use the templates in the link:ci-templates/auto-rules/with-rosinstall[] folder.
The auto rules also allow for using variables to explicitely activate each job:
```yml
include::examples/build_vars.yml[]
```
If a `ddeploy.yaml` file exists, the auto-rules will add a job extending the `.ddeploy` template defined
== LFS
By default, lfs files are not downloaded. If you need to add LFS, include the link:ci-templates/lfs-pull.yml[] along with anything else:
```yml
include::examples/include_lfs.yml[]
```
\ No newline at end of file
include: ci-templates/auto-rules/no-default.yml
variables:
DEFAULT_DISTRO: humble
include: ci-templates/auto-rules/no-default.yml
variables:
DEFAULT_DISTRO: jazzy
include: ci-templates/auto-rules/no-default.yml
variables:
DEFAULT_DISTRO: kinetic
include: ci-templates/auto-rules/no-default.yml
variables:
DEFAULT_DISTRO: melodic
include: ci-templates/core.yml
workflow:
rules:
- if: $CI_COMMIT_REF_NAME =~ /^kinetic-.*/
variables:
DEFAULT_DISTRO: "kinetic"
- if: $CI_COMMIT_REF_NAME =~ /^melodic-.*/
variables:
DEFAULT_DISTRO: "melodic"
- if: $CI_COMMIT_REF_NAME =~ /^noetic-.*/
variables:
DEFAULT_DISTRO: "noetic"
- if: $CI_COMMIT_REF_NAME =~ /^humble-.*/
variables:
DEFAULT_DISTRO: "humble"
- if: $CI_COMMIT_REF_NAME =~ /^jazzy-.*/
variables:
DEFAULT_DISTRO: "jazzy"
- if: $CI_COMMIT_TAG
- if: $CI_COMMIT_BRANCH
industrial_ci_kinetic:
extends: .industrial_ci
variables:
ROS_DISTRO: kinetic
rules:
- if: $DEFAULT_DISTRO == "kinetic"
- if: $BUILD_KINETIC
industrial_ci_melodic:
extends: .industrial_ci
variables:
ROS_DISTRO: melodic
rules:
- if: $DEFAULT_DISTRO == "melodic"
- if: $BUILD_MELODIC
industrial_ci_noetic:
extends: .industrial_ci
variables:
ROS_DISTRO: noetic
rules:
- if: $DEFAULT_DISTRO == "noetic"
- if: $BUILD_NOETIC
industrial_ci_humble:
extends: .industrial_ci
variables:
ROS_DISTRO: humble
rules:
- if: $DEFAULT_DISTRO == "humble"
- if: $BUILD_HUMBLE
industrial_ci_jazzy:
extends: .industrial_ci
variables:
ROS_DISTRO: jazzy
rules:
- if: $DEFAULT_DISTRO == "jazzy"
- if: $BUILD_JAZZY
.py3_rules:
rules:
- if: $DEFAULT_DISTRO == "noetic"
when: always
- if: $BUILD_NOETIC
when: always
- if: $DEFAULT_DISTRO == "humble"
when: always
- if: $BUILD_HUMBLE
when: always
- if: $DEFAULT_DISTRO == "jazzy"
when: always
- if: $BUILD_JAZZY
when: always
py3-flake8:
extends:
- .py3_rules
- .py3-flake8
py3-flake8_extended:
extends:
- .py3_rules
- .py3-flake8_extended
py3-security:
extends:
- .py3_rules
- .py3-security
.py2_rules:
rules:
- if: $DEFAULT_DISTRO == "kinetic"
when: always
- if: $BUILD_KINETIC
when: always
- if: $DEFAULT_DISTRO == "melodic"
when: always
- if: $BUILD_MELODIC
when: always
py2-flake8:
extends:
- .py2_rules
- .py2-flake8
ddeploy:
extends: .ddeploy
rules:
- if: $CI_COMMIT_TAG
exists:
- ddeploy.yaml
when: always
- exists:
- ddeploy.yaml
when: manual
allow_failure: true
include: ci-templates/auto-rules/no-default.yml
variables:
DEFAULT_DISTRO: noetic
include: ci-templates/auto-rules/with-rosinstall/no-default.yml
variables:
DEFAULT_DISTRO: humble
include: ci-templates/auto-rules/with-rosinstall/no-default.yml
variables:
DEFAULT_DISTRO: jazzy
include: ci-templates/auto-rules/with-rosinstall/no-default.yml
variables:
DEFAULT_DISTRO: kinetic
include: ci-templates/auto-rules/with-rosinstall/no-default.yml
variables:
DEFAULT_DISTRO: melodic
include: ci-templates/auto-rules/no-default.yml
.industrial_ci:
variables:
UPSTREAM_WORKSPACE: .rosinstall
include: ci-templates/auto-rules/with-rosinstall/no-default.yml
variables:
DEFAULT_DISTRO: noetic
image: tecnalia-docker-dev.artifact.tecnalia.dev/docker:git
workflow:
rules:
- if: $CI_COMMIT_TAG
- if: $CI_COMMIT_BRANCH
variables:
TMPDIR: /builds/${CI_PROJECT_NAMESPACE}
GIT_LFS_SKIP_SMUDGE: 1 # Avoid fetching the files from LFS when doing git fetch
stages:
- build
- deploy
include:
- ci-templates/syntax-check.yml
- ci-templates/python.yml
- ci-templates/industrial-ci-templates.yml
default:
tags:
- docker
include:
- project: tecnalia_robotics-public/gitlab_templates
file: ci-templates/auto-rules/no-default.yml
variables:
BUILD_KINETIC: 1
BUILD_MELODIC: 1
BUILD_NOETIC: 1
include:
- project: tecnalia_robotics-public/gitlab_templates
file:
- ci-templates/auto-rules/kinetic-default.yml
- ci-templates/lfs-pull.yml