Merge branch '38-update-deprecated-deploy-job' into 'master'

Resolve "Update deprecated deploy job"

Closes #38

See merge request tecnalia_robotics-public/gitlab_templates!70

.gitlab-ci.yml

 shellcheck:
   image: tecnalia-docker-dev.artifact.tecnalia.com/koalaman/shellcheck-alpine
-  script: shellcheck deploy/base/ros_entrypoint.bash scripts/*sh
+  script: shellcheck scripts/*sh
   tags:
     - docker
 
-# The ignored rules in hadolint checks are
-# DL3006 Always tag the version of an image explicitly
-#   One of the Dockerfiles uses a temporary image which isn't important to tag
-#   and the other uses a tagged image through an ARG, which apparently is not
-#   properly recognized.
-# DL3008 Pin versions in apt-get install
-#   A bit overkill
-dockerfiles:
-  image: tecnalia-docker-dev.artifact.tecnalia.com/hadolint/hadolint:latest-debian
-  script:
-    - RET=0
-    - hadolint --ignore DL3006 --ignore DL3008 --ignore DL3059 deploy/base/Dockerfile || RET=1
-    - hadolint --ignore DL3006 --ignore DL3008 --ignore DL3059 deploy/dev/Dockerfile || RET=1
-    - exit $RET
-  tags:
-    - docker
 
 yamllint:
   image: tecnalia-docker-dev.artifact.tecnalia.com/sdesbure/yamllint

README.adoc

@@ -9,11 +9,3 @@ include::ci-templates/README.adoc[leveloffset=+1]
 == Custom `rosdep` definitions
 
 Custom `rosdep` definitions for the packages hosted in our internal debian repository are declared in link:rosdistro/rosdep_tecnalia.yaml[].
-
-== Deployment image build resources
-
-Resources required for building the deployment docker images are also hosted in this repository.
-This basically boils down to:
-
-- link:scripts/pre-deploy-cleanup.bash[] is run in the `.industrial_ci_deploy` template before commiting to an image
-- link:deploy/Dockerfile[] is used to perform some additional cleanup and install link:deploy/ros_entrypoint.bash[] as entrypoint script on top of the `industrial_ci` image

ci-templates/README.adoc

@@ -17,11 +17,14 @@ The Core pipeline, defined in link:ci-templates/core.yml[], has the following st
     ** If the package has `.rosinstall` dependencies, they can be installed using the variable `UPSTREAM_WORKSPACE`
     ** Uses `flexbotics-base-devel:${ROS_DISTRO}` as a starting image
     ** Runs link:scripts/ci_run_entry_points.sh[] in the `AFTER_SETUP_TARGET_WORKSPACE` stage, which in turn finds and runs every `ci_entry_script.bash` script found in the target repo.
-  * `.industrial_ci_deploy` template, which:
-    ** Runs `industrial_ci` by extending `.industrial_ci`
-    ** Runs some cleanup in the `industrial_ci` context (e.g. delete sources)
-    ** Commits image used by `industrial_ci`
-    ** Builds deployment image on top of it and push it to our registry
+- Stage: `deploy`
+  * `.ddeploy` template (see https://git.code.tecnalia.com/tecnalia_robotics/flexbotics/flexbotics_utils/ddeploy[the ddeploy repo]), which:
+    ** Is run automatically for tags, but can also be run manually on normal commits
+    ** Calls `ddeploy` to generate the docker image
+    ** Renames the image created by ddeploy with different tags and pushes them to the registry.
+    *** Tag `${COMMIT_SHA}` (hash of commit)
+    *** Tag `${CI_COMMIT_REF_SLUG}` (branch or tag name)
+    *** Tag `latest` only on the default branch
 - Stage: `.post`
   * Check `bash`/`sh` script syntax
   * Check Markdown and AsciiDoc syntax
@@ -52,6 +55,8 @@ The auto rules also allow for using variables to explicitely activate each job:
 include::examples/build_vars.yml[]
 ```
 
+If a `ddeploy.yaml` file exists, the auto-rules will add a job extending the `.ddeploy` template defined
+
 == LFS
 By default, lfs files are not downloaded. If you need to add LFS, include the link:ci-templates/lfs-pull.yml[] along with anything else:
 ```yml

ci-templates/auto-rules/no-default.yml

@@ -2,40 +2,23 @@ include: ci-templates/core.yml
 
 workflow:
   rules:
-    - if: $CI_COMMIT_BRANCH && $CI_COMMIT_REF_NAME =~ /^kinetic-.*/
+    - if: $CI_COMMIT_REF_NAME =~ /^kinetic-.*/
       variables:
         DEFAULT_DISTRO: "kinetic"
-    - if: $CI_COMMIT_BRANCH && $CI_COMMIT_REF_NAME =~ /^melodic-.*/
+    - if: $CI_COMMIT_REF_NAME =~ /^melodic-.*/
       variables:
         DEFAULT_DISTRO: "melodic"
-    - if: $CI_COMMIT_BRANCH && $CI_COMMIT_REF_NAME =~ /^noetic-.*/
+    - if: $CI_COMMIT_REF_NAME =~ /^noetic-.*/
       variables:
         DEFAULT_DISTRO: "noetic"
-    - if: $CI_COMMIT_TAG && $CI_COMMIT_REF_NAME =~ /^kinetic-.*/
-      variables:
-        DEFAULT_DISTRO: "kinetic"
-        DEPLOY: "true"
-    - if: $CI_COMMIT_TAG && $CI_COMMIT_REF_NAME =~ /^melodic-.*/
-      variables:
-        DEFAULT_DISTRO: "melodic"
-        DEPLOY: "true"
-    - if: $CI_COMMIT_TAG && $CI_COMMIT_REF_NAME =~ /^noetic-.*/
-      variables:
-        DEFAULT_DISTRO: "noetic"
-        DEPLOY: "true"
     - if: $CI_COMMIT_TAG
-      variables:
-        DEPLOY: "true"
     - if: $CI_COMMIT_BRANCH
 
-
 industrial_ci_kinetic:
   extends: .industrial_ci
   variables:
     ROS_DISTRO: kinetic
   rules:
-    - if: $DEPLOY
-      when: never
     - if: $DEFAULT_DISTRO == "kinetic"
     - if: $BUILD_KINETIC
 
@@ -44,8 +27,6 @@ industrial_ci_melodic:
   variables:
     ROS_DISTRO: melodic
   rules:
-    - if: $DEPLOY
-      when: never
     - if: $DEFAULT_DISTRO == "melodic"
     - if: $BUILD_MELODIC
 
@@ -54,32 +35,17 @@ industrial_ci_noetic:
   variables:
     ROS_DISTRO: noetic
   rules:
-    - if: $DEPLOY
-      when: never
     - if: $DEFAULT_DISTRO == "noetic"
     - if: $BUILD_NOETIC
 
-
-industrial_kinetic_deploy:
-  extends:
-    - industrial_ci_kinetic
-    - .industrial_ci_deploy
-  rules:
-    - if: $DEPLOY && $DEFAULT_DISTRO == "kinetic"
-    - if: $DEPLOY && $BUILD_KINETIC
-
-industrial_melodic_deploy:
-  extends:
-    - industrial_ci_melodic
-    - .industrial_ci_deploy
+ddeploy:
+  extends: .ddeploy
   rules:
-    - if: $DEPLOY && $DEFAULT_DISTRO == "melodic"
-    - if: $DEPLOY && $BUILD_MELODIC
-
-industrial_noetic_deploy:
-  extends:
-    - industrial_ci_noetic
-    - .industrial_ci_deploy
-  rules:
-    - if: $DEPLOY && $DEFAULT_DISTRO == "noetic"
-    - if: $DEPLOY && $BUILD_NOETIC
+    - if: $CI_COMMIT_TAG
+      exists:
+        - ddeploy.yaml
+      when: always
+    - exists:
+        - ddeploy.yaml
+      when: manual
+      allow_failure: true

ci-templates/core.yml

@@ -16,6 +16,7 @@ services:
 
 stages:
   - build
+  - deploy
 
 before_script:
   - apk add --update bash coreutils tar wget

ci-templates/industrial-ci-templates.yml

@@ -14,24 +14,51 @@
     CMAKE_ARGS: -DCMAKE_CXX_FLAGS=-Wno-ignored-attributes -Wno-int-in-bool-context
 
 
-.industrial_ci_deploy:
-  extends: .industrial_ci
+.ddeploy:
+  stage: deploy
   variables:
-    AFTER_SCRIPT: "wget -q -O - https://git.code.tecnalia.com/tecnalia_robotics-public/gitlab_templates/raw/master/scripts/pre-deploy-cleanup.bash | bash"
-    DOCKER_COMMIT: industrial_ci_image
+    DOCKER_PUSH_REGISTRY: ${ARTIFACT_DOCKER_URL}
+    DOCKER_PUSH_NAME: ${CI_PROJECT_PATH}
+    DOCKER_PUSH_TAG: ${CI_COMMIT_REF_SLUG}
+    DDEPLOY_YAML: ddeploy.yaml
+  before_script:
+    - apk add --update python3 git py3-pip git-lfs
+    # forward the SSH authentication into the Docker executor
+    - "which ssh-agent || ( apk update && apk add openssh-client )"
+    - eval $(ssh-agent -s)
+    - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add - > /dev/null
+    - mkdir -p ~/.ssh
+    - echo "$SSH_SERVER_HOSTKEYS" > ~/.ssh/known_hosts
+    - chmod 700 ~/.ssh
+    # setup LFS authentication
+    - git lfs install
+    - git config --global credential.helper store
+    - echo https://$ARTIFACT_CI_USER:$ARTIFACT_CI_TOKEN@artifact.tecnalia.com > ~/.git-credentials
+    # install ddeploy
+    - pip install git+ssh://git@git.code.tecnalia.com/tecnalia_robotics/flexbotics/flexbotics_utils/ddeploy.git@v0.1.0
+    - docker login ${ARTIFACT_DOCKER_URL} -u ${ARTIFACT_CI_USER} -p ${ARTIFACT_CI_TOKEN}
   script:
-    - .ci_config/gitlab.sh
-    - cd $(mktemp -d)
-    - wget https://git.code.tecnalia.com/tecnalia_robotics-public/gitlab_templates/-/archive/master/gitlab_templates-master.zip
-    - unzip gitlab_templates-master.zip && cd gitlab_templates-master/deploy
-    - cd base
-    - chmod a+x ros_entrypoint.bash
-    - docker build -t ${ARTIFACT_DOCKER_URL}/${CI_PROJECT_NAME}:${CI_COMMIT_REF_NAME} .
-    - docker push ${ARTIFACT_DOCKER_URL}/${CI_PROJECT_NAME}:${CI_COMMIT_REF_NAME}
-    - cd ..
-    - cd dev
-    - docker build --build-arg APPLICATION_IMAGE=${ARTIFACT_DOCKER_URL}/${CI_PROJECT_NAME}:${CI_COMMIT_REF_NAME} -t ${ARTIFACT_DOCKER_URL}/${CI_PROJECT_NAME}:${CI_COMMIT_REF_NAME}-dev .
-    - docker push ${ARTIFACT_DOCKER_URL}/${CI_PROJECT_NAME}:${CI_COMMIT_REF_NAME}-dev
-    - cd ..
+    # Run ddeploy
+    - ddeploy --yaml ${DDEPLOY_YAML}
+    # Get ID of image created by ddeploy
+    - 'DOCKER_ID=$(docker images --format="{{.ID}}" | head -1)'
+    # Tag and push with the branch or tag name.
+    - echo "Pushing to ${DOCKER_PUSH_REGISTRY}/${DOCKER_PUSH_NAME}:${DOCKER_PUSH_TAG}"
+    - docker tag ${DOCKER_ID} ${DOCKER_PUSH_REGISTRY}/${DOCKER_PUSH_NAME}:${DOCKER_PUSH_TAG}
+    - docker push ${DOCKER_PUSH_REGISTRY}/${DOCKER_PUSH_NAME}:${DOCKER_PUSH_TAG}
+    # Tag and push with commit sha
+    - echo "Pushing to ${DOCKER_PUSH_REGISTRY}/${DOCKER_PUSH_NAME}:${CI_COMMIT_SHA}"
+    - docker tag ${DOCKER_ID} ${DOCKER_PUSH_REGISTRY}/${DOCKER_PUSH_NAME}:${CI_COMMIT_SHA}
+    - docker push ${DOCKER_PUSH_REGISTRY}/${DOCKER_PUSH_NAME}:${CI_COMMIT_SHA}
+    # Tag and push with latest only if it's on the default branch
+    - >
+      if [ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]; then
+        echo "Pushing to ${DOCKER_PUSH_REGISTRY}/${DOCKER_PUSH_NAME}:latest"
+        docker tag ${DOCKER_ID} ${DOCKER_PUSH_REGISTRY}/${DOCKER_PUSH_NAME}:latest &&
+        docker push ${DOCKER_PUSH_REGISTRY}/${DOCKER_PUSH_NAME}:latest;
+      fi
   rules:
-    - if: $DEPLOY
+    - if: $CI_COMMIT_TAG
+      when: always
+    - when: manual
+      allow_failure: true

deploy/base/Dockerfile

-FROM industrial_ci_image
-
-RUN rm -rf /root/ici /root/src
-
-RUN apt-get update -qq && apt-get install -y -qq --no-install-recommends \
-  ros-kinetic-rosbash \
-  && rm -rf /var/lib/apt/lists/*
-
-COPY ./ros_entrypoint.bash /
-
-ENTRYPOINT ["/ros_entrypoint.bash"]
-CMD ["bash"]

deploy/base/ros_entrypoint.bash

-#!/bin/bash
-set -e
-
-# setup application environment
-# shellcheck disable=SC1091
-source "/root/target_ws/install/setup.bash"
-exec "$@"

deploy/dev/Dockerfile

-ARG APPLICATION_IMAGE
-FROM ${APPLICATION_IMAGE}
-
-RUN apt-get update -qq && apt-get install -y -qq --no-install-recommends \
-  apt-transport-https \
-  curl \
-  libasound2 \
-  ca-certificates \
-  && rm -rf /var/lib/apt/lists/*
-
-SHELL ["/bin/bash", "-o", "pipefail", "-c"]
-RUN curl https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > microsoft.gpg \
-  && mv microsoft.gpg /etc/apt/trusted.gpg.d/microsoft.gpg \
-  && sh -c 'echo "deb [arch=amd64] https://packages.microsoft.com/repos/vscode stable main" > /etc/apt/sources.list.d/vscode.list'
-
-RUN apt-get update -qq && apt-get install -y -qq --no-install-recommends \
-  code \
-  && rm -rf /var/lib/apt/lists/*
-
-RUN echo "alias code='code --user-data-dir /root'" >> /root/.bashrc
-
-RUN code --user-data-dir /root \
-  --install-extension ms-iot.vscode-ros \
-  --install-extension joaompinto.asciidoctor-vscode \
-  --install-extension ms-python.python \
-  --install-extension ms-vscode.cpptools \
-  --install-extension ms-azuretools.vscode-docker \
-  --install-extension twxs.cmake