From ca5544ce4c7311380278c46859eadeff48c369f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?I=C3=B1igo=20Moreno?= Date: Wed, 22 Sep 2021 11:07:41 +0200 Subject: [PATCH 01/10] Add lfs-pull --- ci-templates/lfs-pull.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 ci-templates/lfs-pull.yml diff --git a/ci-templates/lfs-pull.yml b/ci-templates/lfs-pull.yml new file mode 100644 index 0000000..b657a44 --- /dev/null +++ b/ci-templates/lfs-pull.yml @@ -0,0 +1,11 @@ +.lfs-pull-script: &lfs-pull-script + "find $$current_ws -type f -name .lfsconfig -execdir git lfs pull \\; ;\ + find $$current_ws -type f -name .lfsconfig -execdir git lfs ls-files \\; ;" + +.industrial-ci: + BEFORE_INIT: "bash <(curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh);\ + apt install -qq -y git-lfs; \ + git config --global credential.helper store; \ + echo https://$ARTIFACT_CI_USER:$ARTIFACT_CI_TOKEN@artifact.tecnalia.com > ~/.git-credentials ;" + BEFORE_BUILD_TARGET_WORKSPACE: *lfs-pull-script + BEFORE_BUILD_UPSTREAM_WORKSPACE: *lfs-pull-script -- GitLab From b52a25f3b88b7bfeb51433bdc2dd79ccd561741e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?I=C3=B1igo=20Moreno?= Date: Wed, 22 Sep 2021 11:22:24 +0200 Subject: [PATCH 02/10] Fix naming --- ci-templates/lfs-pull.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci-templates/lfs-pull.yml b/ci-templates/lfs-pull.yml index b657a44..8e90299 100644 --- a/ci-templates/lfs-pull.yml +++ b/ci-templates/lfs-pull.yml @@ -2,7 +2,7 @@ "find $$current_ws -type f -name .lfsconfig -execdir git lfs pull \\; ;\ find $$current_ws -type f -name .lfsconfig -execdir git lfs ls-files \\; ;" -.industrial-ci: +.industrial_ci: BEFORE_INIT: "bash <(curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh);\ apt install -qq -y git-lfs; \ git config --global credential.helper store; \ -- GitLab From 2b7f7652bb5b199aab38c0ac3874d1f4da029492 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?I=C3=B1igo=20Moreno?= Date: Wed, 22 Sep 2021 11:26:04 +0200 Subject: [PATCH 03/10] Fix error --- ci-templates/lfs-pull.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/ci-templates/lfs-pull.yml b/ci-templates/lfs-pull.yml index 8e90299..5540872 100644 --- a/ci-templates/lfs-pull.yml +++ b/ci-templates/lfs-pull.yml @@ -3,9 +3,10 @@ find $$current_ws -type f -name .lfsconfig -execdir git lfs ls-files \\; ;" .industrial_ci: - BEFORE_INIT: "bash <(curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh);\ - apt install -qq -y git-lfs; \ - git config --global credential.helper store; \ - echo https://$ARTIFACT_CI_USER:$ARTIFACT_CI_TOKEN@artifact.tecnalia.com > ~/.git-credentials ;" - BEFORE_BUILD_TARGET_WORKSPACE: *lfs-pull-script - BEFORE_BUILD_UPSTREAM_WORKSPACE: *lfs-pull-script + variables: + BEFORE_INIT: "bash <(curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh);\ + apt install -qq -y git-lfs; \ + git config --global credential.helper store; \ + echo https://$ARTIFACT_CI_USER:$ARTIFACT_CI_TOKEN@artifact.tecnalia.com > ~/.git-credentials ;" + BEFORE_BUILD_TARGET_WORKSPACE: *lfs-pull-script + BEFORE_BUILD_UPSTREAM_WORKSPACE: *lfs-pull-script -- GitLab From eccf2c42ce2a6b9bc89906ec69f9d0ddd600e510 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?I=C3=B1igo=20Moreno?= Date: Wed, 22 Sep 2021 11:31:53 +0200 Subject: [PATCH 04/10] Add curl --- ci-templates/core.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci-templates/core.yml b/ci-templates/core.yml index 3553a10..aa37a69 100644 --- a/ci-templates/core.yml +++ b/ci-templates/core.yml @@ -18,7 +18,7 @@ stages: - build before_script: - - apk add --update bash coreutils tar wget + - apk add --update bash coreutils tar wget curl - docker login tecnalia-docker-dev.artifact.tecnalia.com -u ${ARTIFACT_CI_USER} -p ${ARTIFACT_CI_TOKEN} include: -- GitLab From 5b1ba8704adf022ca3f8a13f86b11d340bef9455 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?I=C3=B1igo=20Moreno?= Date: Wed, 22 Sep 2021 11:40:58 +0200 Subject: [PATCH 05/10] Install curl on BEFORE_INIT --- ci-templates/core.yml | 2 +- ci-templates/lfs-pull.yml | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/ci-templates/core.yml b/ci-templates/core.yml index aa37a69..3553a10 100644 --- a/ci-templates/core.yml +++ b/ci-templates/core.yml @@ -18,7 +18,7 @@ stages: - build before_script: - - apk add --update bash coreutils tar wget curl + - apk add --update bash coreutils tar wget - docker login tecnalia-docker-dev.artifact.tecnalia.com -u ${ARTIFACT_CI_USER} -p ${ARTIFACT_CI_TOKEN} include: diff --git a/ci-templates/lfs-pull.yml b/ci-templates/lfs-pull.yml index 5540872..245ba07 100644 --- a/ci-templates/lfs-pull.yml +++ b/ci-templates/lfs-pull.yml @@ -4,7 +4,8 @@ .industrial_ci: variables: - BEFORE_INIT: "bash <(curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh);\ + BEFORE_INIT: "apt-get update &> /dev/null && apt-get install -q -y curl; \ + bash <(curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh);\ apt install -qq -y git-lfs; \ git config --global credential.helper store; \ echo https://$ARTIFACT_CI_USER:$ARTIFACT_CI_TOKEN@artifact.tecnalia.com > ~/.git-credentials ;" -- GitLab From fb4f8e53c963dfe2349c029fba67804c19ef7980 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?I=C3=B1igo=20Moreno?= Date: Wed, 22 Sep 2021 11:48:25 +0200 Subject: [PATCH 06/10] Change script hooks --- ci-templates/lfs-pull.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/ci-templates/lfs-pull.yml b/ci-templates/lfs-pull.yml index 245ba07..ca15f2f 100644 --- a/ci-templates/lfs-pull.yml +++ b/ci-templates/lfs-pull.yml @@ -4,10 +4,9 @@ .industrial_ci: variables: - BEFORE_INIT: "apt-get update &> /dev/null && apt-get install -q -y curl; \ - bash <(curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh);\ + AFTER_INIT: "bash <(curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh);\ apt install -qq -y git-lfs; \ git config --global credential.helper store; \ echo https://$ARTIFACT_CI_USER:$ARTIFACT_CI_TOKEN@artifact.tecnalia.com > ~/.git-credentials ;" - BEFORE_BUILD_TARGET_WORKSPACE: *lfs-pull-script - BEFORE_BUILD_UPSTREAM_WORKSPACE: *lfs-pull-script + BEFORE_INSTALL_TARGET_WORKSPACE: *lfs-pull-script + BEFORE_INSTALL_UPSTREAM_WORKSPACE: *lfs-pull-script -- GitLab From 213323f12f2e5e0010f975a635460518a952389a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?I=C3=B1igo=20Moreno?= Date: Wed, 22 Sep 2021 11:56:20 +0200 Subject: [PATCH 07/10] Fix naming of hook --- ci-templates/lfs-pull.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci-templates/lfs-pull.yml b/ci-templates/lfs-pull.yml index ca15f2f..f311da4 100644 --- a/ci-templates/lfs-pull.yml +++ b/ci-templates/lfs-pull.yml @@ -8,5 +8,5 @@ apt install -qq -y git-lfs; \ git config --global credential.helper store; \ echo https://$ARTIFACT_CI_USER:$ARTIFACT_CI_TOKEN@artifact.tecnalia.com > ~/.git-credentials ;" - BEFORE_INSTALL_TARGET_WORKSPACE: *lfs-pull-script - BEFORE_INSTALL_UPSTREAM_WORKSPACE: *lfs-pull-script + BEFORE_INSTALL_TARGET_DEPENDENCIES: *lfs-pull-script + BEFORE_INSTALL_UPSTREAM_DEPENDENCIES: *lfs-pull-script -- GitLab From 78223b459bb39ba4d8decea8e538c2819bb1b649 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?I=C3=B1igo=20Moreno?= Date: Wed, 22 Sep 2021 12:47:31 +0200 Subject: [PATCH 08/10] Move ROS_REPO to main --- ci-templates/industrial-ci-templates.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/ci-templates/industrial-ci-templates.yml b/ci-templates/industrial-ci-templates.yml index b1c5d84..e5e3eba 100644 --- a/ci-templates/industrial-ci-templates.yml +++ b/ci-templates/industrial-ci-templates.yml @@ -12,6 +12,7 @@ PARALLEL_BUILDS: 2 # keep this for now, since there seems to be memory limit issues in the ci VM AFTER_SETUP_TARGET_WORKSPACE: "rosenv && bash <(curl -Ls https://git.code.tecnalia.com/tecnalia_robotics-public/gitlab_templates/raw/master/scripts/ci_run_entry_points.sh) $$current_ws" CMAKE_ARGS: -DCMAKE_CXX_FLAGS=-Wno-ignored-attributes -Wno-int-in-bool-context + ROS_REPO: main .industrial_ci_deploy: -- GitLab From ed16c55e32c0ee870be5c7afd74836f280d6879d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?I=C3=B1igo=20Moreno?= Date: Wed, 22 Sep 2021 16:08:22 +0200 Subject: [PATCH 09/10] Add documentation --- ci-templates/README.adoc | 7 ++++++- ci-templates/examples/include_lfs.yml | 5 +++++ ci-templates/industrial-ci-templates.yml | 1 - 3 files changed, 11 insertions(+), 2 deletions(-) create mode 100644 ci-templates/examples/include_lfs.yml diff --git a/ci-templates/README.adoc b/ci-templates/README.adoc index d4f1a0d..f99eb2c 100644 --- a/ci-templates/README.adoc +++ b/ci-templates/README.adoc @@ -28,7 +28,6 @@ The Core pipeline, defined in link:ci-templates/core.yml[], has the following st * Check files have no trailing whitespace and are `UTF-8` encoded * Check that `C++` files follow this link:clang-format[] -It is important to note that both templates set the variable `GIT_LFS_SKIP_SMUDGE`, which means that when the repositories are cloned/fetched, the files hosted using Git LFS will not be downloaded. An example on how to explicitely fetch the files from LFS can be found in the https://git.code.tecnalia.com/tecnalia_robotics/common_config/blob/e93dbe6379d94e6a895a33c59bf17ccab26eacf9/.gitlab-ci.yml#L132[common_config repository]. == Manual Jobs @@ -52,3 +51,9 @@ The auto rules also allow for using variables to explicitely activate each job: ```yml include::examples/build_vars.yml[] ``` + +== LFS +By default, lfs files are not downloaded. If you need to add LFS, include the link:ci-templates/lfs-pull.yml[] along with anything else: +```yml +include::examples/include_lfs.yml[] +``` \ No newline at end of file diff --git a/ci-templates/examples/include_lfs.yml b/ci-templates/examples/include_lfs.yml new file mode 100644 index 0000000..00876e5 --- /dev/null +++ b/ci-templates/examples/include_lfs.yml @@ -0,0 +1,5 @@ +include: + - project: tecnalia_robotics-public/gitlab_templates + file: + - ci-templates/auto-rules/kinetic-default.yml + - ci-templates/auto-rules/kinetic-default.yml diff --git a/ci-templates/industrial-ci-templates.yml b/ci-templates/industrial-ci-templates.yml index e5e3eba..b1c5d84 100644 --- a/ci-templates/industrial-ci-templates.yml +++ b/ci-templates/industrial-ci-templates.yml @@ -12,7 +12,6 @@ PARALLEL_BUILDS: 2 # keep this for now, since there seems to be memory limit issues in the ci VM AFTER_SETUP_TARGET_WORKSPACE: "rosenv && bash <(curl -Ls https://git.code.tecnalia.com/tecnalia_robotics-public/gitlab_templates/raw/master/scripts/ci_run_entry_points.sh) $$current_ws" CMAKE_ARGS: -DCMAKE_CXX_FLAGS=-Wno-ignored-attributes -Wno-int-in-bool-context - ROS_REPO: main .industrial_ci_deploy: -- GitLab From f06250ed7ed83f57db46aa6669b6fd032ed698cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?I=C3=B1igo=20Moreno?= Date: Wed, 22 Sep 2021 16:11:03 +0200 Subject: [PATCH 10/10] Fix example --- ci-templates/examples/include_lfs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci-templates/examples/include_lfs.yml b/ci-templates/examples/include_lfs.yml index 00876e5..4808de9 100644 --- a/ci-templates/examples/include_lfs.yml +++ b/ci-templates/examples/include_lfs.yml @@ -2,4 +2,4 @@ include: - project: tecnalia_robotics-public/gitlab_templates file: - ci-templates/auto-rules/kinetic-default.yml - - ci-templates/auto-rules/kinetic-default.yml + - ci-templates/lfs-pull.yml -- GitLab