From 717af3819e3ef5e4d531b4bf18712a74be868217 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?I=C3=B1igo=20Moreno=20i=20Caireta?= <inigo.moreno@tecnalia.com> Date: Thu, 4 Nov 2021 16:22:25 +0100 Subject: [PATCH] Resolve "Update deprecated deploy job" --- .gitlab-ci.yml | 18 +------ README.adoc | 8 --- ci-templates/README.adoc | 15 ++++-- ci-templates/auto-rules/no-default.yml | 60 +++++----------------- ci-templates/core.yml | 1 + ci-templates/industrial-ci-templates.yml | 63 +++++++++++++++++------- deploy/base/Dockerfile | 12 ----- deploy/base/ros_entrypoint.bash | 7 --- deploy/dev/Dockerfile | 28 ----------- 9 files changed, 70 insertions(+), 142 deletions(-) delete mode 100644 deploy/base/Dockerfile delete mode 100755 deploy/base/ros_entrypoint.bash delete mode 100644 deploy/dev/Dockerfile diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index be9910d..3d2cc4d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,25 +1,9 @@ shellcheck: image: tecnalia-docker-dev.artifact.tecnalia.com/koalaman/shellcheck-alpine - script: shellcheck deploy/base/ros_entrypoint.bash scripts/*sh + script: shellcheck scripts/*sh tags: - docker -# The ignored rules in hadolint checks are -# DL3006 Always tag the version of an image explicitly -# One of the Dockerfiles uses a temporary image which isn't important to tag -# and the other uses a tagged image through an ARG, which apparently is not -# properly recognized. -# DL3008 Pin versions in apt-get install -# A bit overkill -dockerfiles: - image: tecnalia-docker-dev.artifact.tecnalia.com/hadolint/hadolint:latest-debian - script: - - RET=0 - - hadolint --ignore DL3006 --ignore DL3008 --ignore DL3059 deploy/base/Dockerfile || RET=1 - - hadolint --ignore DL3006 --ignore DL3008 --ignore DL3059 deploy/dev/Dockerfile || RET=1 - - exit $RET - tags: - - docker yamllint: image: tecnalia-docker-dev.artifact.tecnalia.com/sdesbure/yamllint diff --git a/README.adoc b/README.adoc index eb4ff72..c8d344e 100644 --- a/README.adoc +++ b/README.adoc @@ -9,11 +9,3 @@ include::ci-templates/README.adoc[leveloffset=+1] == Custom `rosdep` definitions Custom `rosdep` definitions for the packages hosted in our internal debian repository are declared in link:rosdistro/rosdep_tecnalia.yaml[]. - -== Deployment image build resources - -Resources required for building the deployment docker images are also hosted in this repository. -This basically boils down to: - -- link:scripts/pre-deploy-cleanup.bash[] is run in the `.industrial_ci_deploy` template before commiting to an image -- link:deploy/Dockerfile[] is used to perform some additional cleanup and install link:deploy/ros_entrypoint.bash[] as entrypoint script on top of the `industrial_ci` image diff --git a/ci-templates/README.adoc b/ci-templates/README.adoc index f99eb2c..5683239 100644 --- a/ci-templates/README.adoc +++ b/ci-templates/README.adoc @@ -17,11 +17,14 @@ The Core pipeline, defined in link:ci-templates/core.yml[], has the following st ** If the package has `.rosinstall` dependencies, they can be installed using the variable `UPSTREAM_WORKSPACE` ** Uses `flexbotics-base-devel:${ROS_DISTRO}` as a starting image ** Runs link:scripts/ci_run_entry_points.sh[] in the `AFTER_SETUP_TARGET_WORKSPACE` stage, which in turn finds and runs every `ci_entry_script.bash` script found in the target repo. - * `.industrial_ci_deploy` template, which: - ** Runs `industrial_ci` by extending `.industrial_ci` - ** Runs some cleanup in the `industrial_ci` context (e.g. delete sources) - ** Commits image used by `industrial_ci` - ** Builds deployment image on top of it and push it to our registry +- Stage: `deploy` + * `.ddeploy` template (see https://git.code.tecnalia.com/tecnalia_robotics/flexbotics/flexbotics_utils/ddeploy[the ddeploy repo]), which: + ** Is run automatically for tags, but can also be run manually on normal commits + ** Calls `ddeploy` to generate the docker image + ** Renames the image created by ddeploy with different tags and pushes them to the registry. + *** Tag `${COMMIT_SHA}` (hash of commit) + *** Tag `${CI_COMMIT_REF_SLUG}` (branch or tag name) + *** Tag `latest` only on the default branch - Stage: `.post` * Check `bash`/`sh` script syntax * Check Markdown and AsciiDoc syntax @@ -52,6 +55,8 @@ The auto rules also allow for using variables to explicitely activate each job: include::examples/build_vars.yml[] ``` +If a `ddeploy.yaml` file exists, the auto-rules will add a job extending the `.ddeploy` template defined + == LFS By default, lfs files are not downloaded. If you need to add LFS, include the link:ci-templates/lfs-pull.yml[] along with anything else: ```yml diff --git a/ci-templates/auto-rules/no-default.yml b/ci-templates/auto-rules/no-default.yml index 7db60c8..011021a 100644 --- a/ci-templates/auto-rules/no-default.yml +++ b/ci-templates/auto-rules/no-default.yml @@ -2,40 +2,23 @@ include: ci-templates/core.yml workflow: rules: - - if: $CI_COMMIT_BRANCH && $CI_COMMIT_REF_NAME =~ /^kinetic-.*/ + - if: $CI_COMMIT_REF_NAME =~ /^kinetic-.*/ variables: DEFAULT_DISTRO: "kinetic" - - if: $CI_COMMIT_BRANCH && $CI_COMMIT_REF_NAME =~ /^melodic-.*/ + - if: $CI_COMMIT_REF_NAME =~ /^melodic-.*/ variables: DEFAULT_DISTRO: "melodic" - - if: $CI_COMMIT_BRANCH && $CI_COMMIT_REF_NAME =~ /^noetic-.*/ + - if: $CI_COMMIT_REF_NAME =~ /^noetic-.*/ variables: DEFAULT_DISTRO: "noetic" - - if: $CI_COMMIT_TAG && $CI_COMMIT_REF_NAME =~ /^kinetic-.*/ - variables: - DEFAULT_DISTRO: "kinetic" - DEPLOY: "true" - - if: $CI_COMMIT_TAG && $CI_COMMIT_REF_NAME =~ /^melodic-.*/ - variables: - DEFAULT_DISTRO: "melodic" - DEPLOY: "true" - - if: $CI_COMMIT_TAG && $CI_COMMIT_REF_NAME =~ /^noetic-.*/ - variables: - DEFAULT_DISTRO: "noetic" - DEPLOY: "true" - if: $CI_COMMIT_TAG - variables: - DEPLOY: "true" - if: $CI_COMMIT_BRANCH - industrial_ci_kinetic: extends: .industrial_ci variables: ROS_DISTRO: kinetic rules: - - if: $DEPLOY - when: never - if: $DEFAULT_DISTRO == "kinetic" - if: $BUILD_KINETIC @@ -44,8 +27,6 @@ industrial_ci_melodic: variables: ROS_DISTRO: melodic rules: - - if: $DEPLOY - when: never - if: $DEFAULT_DISTRO == "melodic" - if: $BUILD_MELODIC @@ -54,32 +35,17 @@ industrial_ci_noetic: variables: ROS_DISTRO: noetic rules: - - if: $DEPLOY - when: never - if: $DEFAULT_DISTRO == "noetic" - if: $BUILD_NOETIC - -industrial_kinetic_deploy: - extends: - - industrial_ci_kinetic - - .industrial_ci_deploy - rules: - - if: $DEPLOY && $DEFAULT_DISTRO == "kinetic" - - if: $DEPLOY && $BUILD_KINETIC - -industrial_melodic_deploy: - extends: - - industrial_ci_melodic - - .industrial_ci_deploy +ddeploy: + extends: .ddeploy rules: - - if: $DEPLOY && $DEFAULT_DISTRO == "melodic" - - if: $DEPLOY && $BUILD_MELODIC - -industrial_noetic_deploy: - extends: - - industrial_ci_noetic - - .industrial_ci_deploy - rules: - - if: $DEPLOY && $DEFAULT_DISTRO == "noetic" - - if: $DEPLOY && $BUILD_NOETIC + - if: $CI_COMMIT_TAG + exists: + - ddeploy.yaml + when: always + - exists: + - ddeploy.yaml + when: manual + allow_failure: true diff --git a/ci-templates/core.yml b/ci-templates/core.yml index 3553a10..8cacbee 100644 --- a/ci-templates/core.yml +++ b/ci-templates/core.yml @@ -16,6 +16,7 @@ services: stages: - build + - deploy before_script: - apk add --update bash coreutils tar wget diff --git a/ci-templates/industrial-ci-templates.yml b/ci-templates/industrial-ci-templates.yml index b1c5d84..4ab2d62 100644 --- a/ci-templates/industrial-ci-templates.yml +++ b/ci-templates/industrial-ci-templates.yml @@ -14,24 +14,51 @@ CMAKE_ARGS: -DCMAKE_CXX_FLAGS=-Wno-ignored-attributes -Wno-int-in-bool-context -.industrial_ci_deploy: - extends: .industrial_ci +.ddeploy: + stage: deploy variables: - AFTER_SCRIPT: "wget -q -O - https://git.code.tecnalia.com/tecnalia_robotics-public/gitlab_templates/raw/master/scripts/pre-deploy-cleanup.bash | bash" - DOCKER_COMMIT: industrial_ci_image + DOCKER_PUSH_REGISTRY: ${ARTIFACT_DOCKER_URL} + DOCKER_PUSH_NAME: ${CI_PROJECT_PATH} + DOCKER_PUSH_TAG: ${CI_COMMIT_REF_SLUG} + DDEPLOY_YAML: ddeploy.yaml + before_script: + - apk add --update python3 git py3-pip git-lfs + # forward the SSH authentication into the Docker executor + - "which ssh-agent || ( apk update && apk add openssh-client )" + - eval $(ssh-agent -s) + - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add - > /dev/null + - mkdir -p ~/.ssh + - echo "$SSH_SERVER_HOSTKEYS" > ~/.ssh/known_hosts + - chmod 700 ~/.ssh + # setup LFS authentication + - git lfs install + - git config --global credential.helper store + - echo https://$ARTIFACT_CI_USER:$ARTIFACT_CI_TOKEN@artifact.tecnalia.com > ~/.git-credentials + # install ddeploy + - pip install git+ssh://git@git.code.tecnalia.com/tecnalia_robotics/flexbotics/flexbotics_utils/ddeploy.git@v0.1.0 + - docker login ${ARTIFACT_DOCKER_URL} -u ${ARTIFACT_CI_USER} -p ${ARTIFACT_CI_TOKEN} script: - - .ci_config/gitlab.sh - - cd $(mktemp -d) - - wget https://git.code.tecnalia.com/tecnalia_robotics-public/gitlab_templates/-/archive/master/gitlab_templates-master.zip - - unzip gitlab_templates-master.zip && cd gitlab_templates-master/deploy - - cd base - - chmod a+x ros_entrypoint.bash - - docker build -t ${ARTIFACT_DOCKER_URL}/${CI_PROJECT_NAME}:${CI_COMMIT_REF_NAME} . - - docker push ${ARTIFACT_DOCKER_URL}/${CI_PROJECT_NAME}:${CI_COMMIT_REF_NAME} - - cd .. - - cd dev - - docker build --build-arg APPLICATION_IMAGE=${ARTIFACT_DOCKER_URL}/${CI_PROJECT_NAME}:${CI_COMMIT_REF_NAME} -t ${ARTIFACT_DOCKER_URL}/${CI_PROJECT_NAME}:${CI_COMMIT_REF_NAME}-dev . - - docker push ${ARTIFACT_DOCKER_URL}/${CI_PROJECT_NAME}:${CI_COMMIT_REF_NAME}-dev - - cd .. + # Run ddeploy + - ddeploy --yaml ${DDEPLOY_YAML} + # Get ID of image created by ddeploy + - 'DOCKER_ID=$(docker images --format="{{.ID}}" | head -1)' + # Tag and push with the branch or tag name. + - echo "Pushing to ${DOCKER_PUSH_REGISTRY}/${DOCKER_PUSH_NAME}:${DOCKER_PUSH_TAG}" + - docker tag ${DOCKER_ID} ${DOCKER_PUSH_REGISTRY}/${DOCKER_PUSH_NAME}:${DOCKER_PUSH_TAG} + - docker push ${DOCKER_PUSH_REGISTRY}/${DOCKER_PUSH_NAME}:${DOCKER_PUSH_TAG} + # Tag and push with commit sha + - echo "Pushing to ${DOCKER_PUSH_REGISTRY}/${DOCKER_PUSH_NAME}:${CI_COMMIT_SHA}" + - docker tag ${DOCKER_ID} ${DOCKER_PUSH_REGISTRY}/${DOCKER_PUSH_NAME}:${CI_COMMIT_SHA} + - docker push ${DOCKER_PUSH_REGISTRY}/${DOCKER_PUSH_NAME}:${CI_COMMIT_SHA} + # Tag and push with latest only if it's on the default branch + - > + if [ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]; then + echo "Pushing to ${DOCKER_PUSH_REGISTRY}/${DOCKER_PUSH_NAME}:latest" + docker tag ${DOCKER_ID} ${DOCKER_PUSH_REGISTRY}/${DOCKER_PUSH_NAME}:latest && + docker push ${DOCKER_PUSH_REGISTRY}/${DOCKER_PUSH_NAME}:latest; + fi rules: - - if: $DEPLOY + - if: $CI_COMMIT_TAG + when: always + - when: manual + allow_failure: true diff --git a/deploy/base/Dockerfile b/deploy/base/Dockerfile deleted file mode 100644 index b5cd8cb..0000000 --- a/deploy/base/Dockerfile +++ /dev/null @@ -1,12 +0,0 @@ -FROM industrial_ci_image - -RUN rm -rf /root/ici /root/src - -RUN apt-get update -qq && apt-get install -y -qq --no-install-recommends \ - ros-kinetic-rosbash \ - && rm -rf /var/lib/apt/lists/* - -COPY ./ros_entrypoint.bash / - -ENTRYPOINT ["/ros_entrypoint.bash"] -CMD ["bash"] diff --git a/deploy/base/ros_entrypoint.bash b/deploy/base/ros_entrypoint.bash deleted file mode 100755 index a203d40..0000000 --- a/deploy/base/ros_entrypoint.bash +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -set -e - -# setup application environment -# shellcheck disable=SC1091 -source "/root/target_ws/install/setup.bash" -exec "$@" diff --git a/deploy/dev/Dockerfile b/deploy/dev/Dockerfile deleted file mode 100644 index 8d1f938..0000000 --- a/deploy/dev/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -ARG APPLICATION_IMAGE -FROM ${APPLICATION_IMAGE} - -RUN apt-get update -qq && apt-get install -y -qq --no-install-recommends \ - apt-transport-https \ - curl \ - libasound2 \ - ca-certificates \ - && rm -rf /var/lib/apt/lists/* - -SHELL ["/bin/bash", "-o", "pipefail", "-c"] -RUN curl https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > microsoft.gpg \ - && mv microsoft.gpg /etc/apt/trusted.gpg.d/microsoft.gpg \ - && sh -c 'echo "deb [arch=amd64] https://packages.microsoft.com/repos/vscode stable main" > /etc/apt/sources.list.d/vscode.list' - -RUN apt-get update -qq && apt-get install -y -qq --no-install-recommends \ - code \ - && rm -rf /var/lib/apt/lists/* - -RUN echo "alias code='code --user-data-dir /root'" >> /root/.bashrc - -RUN code --user-data-dir /root \ - --install-extension ms-iot.vscode-ros \ - --install-extension joaompinto.asciidoctor-vscode \ - --install-extension ms-python.python \ - --install-extension ms-vscode.cpptools \ - --install-extension ms-azuretools.vscode-docker \ - --install-extension twxs.cmake -- GitLab