Merge branch 'refactor-pipeline-configuration' into 'main'

Refactor pipeline configuration

See merge request gitlab-org/ci-cd/docker-machine!83

.gitlab-ci.yml

@@ -3,154 +3,13 @@ stages:
 - validate
 - build
 - release
-
-variables:
-  USE_CONTAINER: "true"
-  CI_IMAGE: registry.gitlab.com/gitlab-org/ci-cd/docker-machine/ci:alpine3.10
-
-default:
-  image: docker:19.03.2
-  tags:
-  - gitlab-org
-
-.docker_in_docker:
-  services:
-  - docker:19.03.2-dind
-  variables:
-    DOCKER_HOST: tcp://docker:2376/
-    DOCKER_DRIVER: overlay2
-    DOCKER_TLS_CERTDIR: "/certs"
-  tags:
-  - gitlab-org-docker
-
-.merge_request_pipelines:
-  only:
-    refs:
-    - merge_requests
-    - main@gitlab-org/ci-cd/docker-machine
-    - tags@gitlab-org/ci-cd/docker-machine
-
-.build_base: &build_base
-  extends:
-  - .docker_in_docker
-  - .merge_request_pipelines
-  stage: build
-  before_script:
-  - apk add -U make bash
-  - export TARGET_OS=$(echo $CI_JOB_NAME | cut -d ' ' -f 1)
-  - export TARGET_ARCH=$(echo $CI_JOB_NAME | cut -d ' ' -f 2)
-  after_script:
-  - "[[ \"$(find bin -type f -name docker-machine*)\" != \"\" ]]"
-  artifacts:
-    paths:
-    - bin/
-    expire_in: 1 week
-
-.build_x: &build_x
-  <<: *build_base
-  script: make build-x
-
-.release:
-  image: ${CI_IMAGE}
-  stage: release
-
-.release_development:
-  only:
-    refs:
-    - merge_requests@gitlab-org/ci-cd/docker-machine
-
-.release_beta:
-  only:
-    refs:
-    - main@gitlab-org/ci-cd/docker-machine
-    - /\Av[0-9]+\.[0-9]+\.[0-9]+-gitlab\.[0-9]+-rc[0-9]+\Z/@gitlab-org/ci-cd/docker-machine
-
-.release_stable:
-  only:
-    refs:
-    - /\Av[0-9]+\.[0-9]+\.[0-9]+-gitlab\.[0-9]+\Z/@gitlab-org/ci-cd/docker-machine
-
-.release_S3:
-  dependencies:
-  - linux amd64
-  - linux arm
-  - linux arm64
-  - windows amd64
-  - darwin amd64
-  variables:
-    S3_URL: s3://${S3_BUCKET}/${CI_COMMIT_REF_NAME}
-  script:
-  - ./.gitlab/ci/scripts/release_s3.sh
-  environment:
-    url: https://${S3_BUCKET}.s3.amazonaws.com/${CI_COMMIT_REF_NAME}/index.html
-
-prepare CI image:
-  extends:
-  - .docker_in_docker
-  - .merge_request_pipelines
-  stage: prepare
-  script:
-  - docker build --pull --no-cache -t ${CI_IMAGE} -f ./.gitlab/ci/Dockerfile ./.gitlab/ci/
-  - docker login --username ${CI_REGISTRY_USER} --password ${CI_REGISTRY_PASSWORD} ${CI_REGISTRY}
-  - docker push ${CI_IMAGE}
-  - docker logout ${CI_REGISTRY}
-  only:
-    changes:
-    - ./gitlab/ci/Dockerfile
-    - .gitlab-ci.yml
-
-validate:
-  stage: validate
-  extends:
-  - .docker_in_docker
-  - .merge_request_pipelines
-  before_script:
-  - apk add -U make bash
-  script: make build validate
-
-darwin amd64: *build_x
-linux amd64: *build_x
-openbsd amd64: *build_x
-windows amd64: *build_x
-linux arm: *build_x
-linux arm64: *build_x
-
-release development S3:
-  extends:
-  - .release
-  - .release_development
-  - .release_S3
-  environment:
-    name: development/S3/${CI_COMMIT_REF_NAME}
-    on_stop: stop release development S3
-
-stop release development S3:
-  dependencies: []
-  extends:
-  - .release
-  - .release_development
-  - .release_S3
-  variables:
-    GIT_STRATEGY: none
-  script:
-   - aws s3 rm ${S3_URL} --recursive
-  when: manual
-  environment:
-    name: development/S3/${CI_COMMIT_REF_NAME}
-    action: stop
-
-release beta S3:
-  extends:
-  - .release
-  - .release_beta
-  - .release_S3
-  environment:
-    name: beta/S3
-
-release stable S3:
-  extends:
-  - .release
-  - .release_stable
-  - .release_S3
-  environment:
-    name: stable/S3
+- postrelease
+
+include:
+- local: .gitlab/ci/yaml/_common.gitlab-ci.yml
+- local: .gitlab/ci/yaml/_rules.gitlab-ci.yml
+- local: .gitlab/ci/yaml/prepare.gitlab-ci.yml
+- local: .gitlab/ci/yaml/validate.gitlab-ci.yml
+- local: .gitlab/ci/yaml/build.gitlab-ci.yml
+- local: .gitlab/ci/yaml/release.gitlab-ci.yml
+- local: .gitlab/ci/yaml/postrelease.gitlab-ci.yml

.gitlab/ci/Dockerfile

-FROM golang:1.13-alpine3.10
+FROM golang:1.13.15-alpine3.12
 
 RUN apk add --no-cache make git py-pip bash curl && \
     pip install awscli

.gitlab/ci/scripts/gitlab_release.sh

-#!/usr/bin/env bash
-
-set -eo pipefail
-
-tag=${1:-$CI_COMMIT_TAG}
-
-if [[ -z "${tag}" ]]; then
-    echo -e "\033[0;31m****** gitlab publishing disabled ******\033[0m"
-    echo -e "usage:\n\t$0 tag"
-    exit 0
-fi
-
-if [[ -z "${CI_JOB_TOKEN}" ]]; then
-    echo -e "\033[0;31m****** Missing CI_JOB_TOKEN, cannot release ******\033[0m"
-    exit 0
-fi
-
-api=${CI_API_V4_URL:-https://gitlab.com/api/v4}
-projectID=${CI_PROJECT_ID:-1254421}
-
-projectUrl=${CI_PROJECT_URL:-https://gitlab.com/gitlab-org/ci-cd/docker-machine}
-
-changelogUrl="${projectUrl}/blob/${tag}/CHANGELOG.md"
-s3=${CI_ENVIRONMENT_URL/\/index.html/}
-
-release=$(cat <<EOS
-{
-  "name": "${tag}",
-  "tag_name": "${tag}",
-  "description": "See [the changelog](${changelogUrl}) :rocket:",
-  "assets": {
-    "links": [
-      { "name": "linux amd64", "url": "$s3/docker-machine-Linux-x86_64" },
-      { "name": "macOS amd64", "url": "$s3/docker-machine-Darwin-x86_64" },
-      { "name": "Windows amd64", "url": "$s3/docker-machine-Windows-x86_64.exe" },
-      { "name": "others", "url": "$s3/index.html" }
-    ]
-  }
-}
-EOS
-)
-
-curl -f \
-     --header 'Content-Type: application/json' \
-     --header "PRIVATE-TOKEN: ${CI_JOB_TOKEN}" \
-     --data "${release}" \
-     --request POST \
-     "${api}/projects/${projectID}/releases"
-

.gitlab/ci/scripts/release_s3.sh

@@ -2,6 +2,18 @@
 
 set -eo pipefail
 
+__aws_s3_sync() {
+  local source="$1"
+  local target="$2"
+
+  if [[ "${DEBUG_S3_SYNC}" == "true" ]]; then
+    set -x
+  fi
+
+  aws s3 sync "${source}" "${target}" --acl public-read
+  set +x
+}
+
 VERSION="$(./.gitlab/ci/scripts/version.sh 2>/dev/null || echo 'dev')"
 
 # Installing release index generator
@@ -26,13 +38,12 @@ chmod +x "${releaseIndexGen}"
 
 echo "Generated Index page"
 
-aws s3 sync bin ${S3_URL} --acl public-read
+__aws_s3_sync bin "${S3_URL}"
 
 # Copy the binaries to the latest directory.
 LATEST_STABLE_TAG=$(git -c versionsort.prereleaseSuffix="-rc" tag -l "v*.*.*" --sort=-v:refname | awk '!/rc/' | head -n 1)
 if [ $(git describe --exact-match --match ${LATEST_STABLE_TAG} >/dev/null 2>&1) ]; then
-      aws s3 sync bin s3://${S3_BUCKET}/latest --acl public-read
+  __aws_s3_sync bin "s3://${S3_BUCKET}/latest"
 fi
 
-# Add assets to release page
-bash ./.gitlab/ci/scripts/gitlab_release.sh
+echo "The release artifacts can be downloaded from ${CI_ENVIRONMENT_URL}"

.gitlab/ci/yaml/_common.gitlab-ci.yml

+variables:
+  USE_CONTAINER: "true"
+  CI_IMAGE: registry.gitlab.com/gitlab-org/ci-cd/docker-machine/ci:go1.13.15-alpine3.12-1
+  DOCKER_VERSION: "20.10.12"
+
+default:
+  image: "docker:${DOCKER_VERSION}"
+  tags:
+  - gitlab-org
+
+.common:docker-in-docker:
+  services:
+  - "docker:${DOCKER_VERSION}-dind"
+  variables:
+    DOCKER_HOST: tcp://docker:2376/
+    DOCKER_DRIVER: overlay2
+    DOCKER_TLS_CERTDIR: "/certs"
+  tags:
+  - gitlab-org-docker

.gitlab/ci/yaml/_rules.gitlab-ci.yml

+###################
+# Change patterns #
+###################
+
+.if-not-canonical-namespace: &if-not-canonical-namespace
+  if: '$CI_PROJECT_NAMESPACE !~ /^gitlab-org($|\/)/'
+
+.if-merge-request-pipeline: &if-merge-request-pipeline
+  if: $CI_PIPELINE_SOURCE == "merge_request_event"
+
+.if-canonical-default-branch: &if-canonical-default-branch
+  if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_PROJECT_PATH == "gitlab-org/ci-cd/docker-machine"
+
+.if-canonical-merge-request-pipeline: &if-canonical-merge-request-pipeline
+  if: $CI_PIPELINE_SOURCE == "merge_request_event" && $CI_PROJECT_PATH == "gitlab-org/ci-cd/docker-machine"
+
+.if-canonical-rc-release-ref: &if-canonical-rc-release-ref
+  if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+\.[0-9]+-gitlab\.[0-9]+-rc[0-9]+$/ && $CI_PROJECT_PATH == "gitlab-org/ci-cd/docker-machine"
+
+.if-canonical-stable-release-ref: &if-canonical-stable-release-ref
+  if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+\.[0-9]+-gitlab\.[0-9]+$/ && $CI_PROJECT_PATH == "gitlab-org/ci-cd/docker-machine"
+
+#######################
+# Merge Request Rules #
+#######################
+
+.rules:merge-requests:
+  rules:
+  - <<: *if-merge-request-pipeline
+  - <<: *if-canonical-default-branch
+  - <<: *if-canonical-stable-release-ref
+
+.rules:merge-requests:prepare:ci-image:
+  rules:
+  - <<: *if-canonical-merge-request-pipeline
+    changes:
+    - .gitlab/ci/Dockerfile
+    - .gitlab/ci/yaml/prepare.gitlab-ci.yml
+
+#########################
+# Release Request Rules #
+#########################
+
+.rules:release:development:
+  rules:
+  - <<: *if-not-canonical-namespace
+    when: never
+  - <<: *if-canonical-merge-request-pipeline
+
+.rules:release:beta:
+  rules:
+  - <<: *if-not-canonical-namespace
+    when: never
+  - <<: *if-canonical-default-branch
+  - <<: *if-canonical-rc-release-ref
+
+.rules:release:rc:
+  rules:
+  - <<: *if-not-canonical-namespace
+    when: never
+  - <<: *if-canonical-rc-release-ref
+
+.rules:release:stable-or-rc:
+  rules:
+  - <<: *if-not-canonical-namespace
+    when: never
+  - <<: *if-canonical-stable-release-ref
+  - <<: *if-canonical-rc-release-ref
+
+.rules:release:stable:
+  rules:
+  - <<: *if-not-canonical-namespace
+    when: never
+  - <<: *if-canonical-stable-release-ref

.gitlab/ci/yaml/build.gitlab-ci.yml

+.build_base:
+  stage: build
+  extends:
+  - .common:docker-in-docker
+  - .rules:merge-requests
+  before_script:
+  - apk add -U make bash
+  - export TARGET_OS=$(echo $CI_JOB_NAME | cut -d ' ' -f 1)
+  - export TARGET_ARCH=$(echo $CI_JOB_NAME | cut -d ' ' -f 2)
+  after_script:
+  - "[[ \"$(find bin -type f -name docker-machine*)\" != \"\" ]]"
+  artifacts:
+    paths:
+    - bin/
+    expire_in: 1 week
+
+.build_x: &build_x
+  extends:
+  - .build_base
+  script: make build-x
+
+darwin amd64: *build_x
+linux amd64: *build_x
+linux arm: *build_x
+linux arm64: *build_x
+openbsd amd64: *build_x
+windows amd64: *build_x

.gitlab/ci/yaml/postrelease.gitlab-ci.yml

+stable gitlab release:
+  stage: postrelease
+  extends:
+  - .rules:release:stable-or-rc
+  dependencies: []
+  image: registry.gitlab.com/gitlab-org/release-cli:latest
+  variables:
+    S3: https://gitlab-docker-machine-downloads.s3.amazonaws.com/$CI_COMMIT_TAG
+  environment:
+    name: stable/gitlab
+    url: https://gitlab.com/gitlab-org/ci-cd/docker-machine/-/releases
+  script:
+  - echo "Releasing to $S3/index.html"
+  release:
+    name: '$CI_COMMIT_TAG'
+    description: |
+      New release of GitLab's fork of Docker Machine is out!
+    tag_name: '$CI_COMMIT_TAG'
+    ref: '$CI_COMMIT_TAG'
+    assets:
+      links:
+      # binaries
+      - name: 'binary: macOS amd64'
+        url: '$S3/docker-machine-Darwin-x86_64'
+        filepath: '/docker-machine-Darwin-x86_64'
+
+      - name: 'binary: Linux amd64'
+        url: '$S3/docker-machine-Linux-x86_64'
+        filepath: '/docker-machine-Linux-x86_64'
+      - name: 'binary: Linux arm'
+        url: '$S3/docker-machine-Linux-armhf'
+        filepath: '/docker-machine-Linux-armhf'
+      - name: 'binary: Linux arm64'
+        url: '$S3/docker-machine-Linux-aarch64'
+        filepath: '/docker-machine-Linux-aarch64'
+
+      - name: 'binary: OpenBSD amd64'
+        url: '$S3/docker-machine-OpenBSD-x86_64'
+        filepath: '/docker-machine-OpenBSD-x86_64'
+
+      - name: 'binary: Windows amd64'
+        url: '$S3/docker-machine-Windows-x86_64.exe'
+        filepath: '/docker-machine-Windows-x86_64.exe'
+
+      # Other files
+      - name: 'checksums'
+        url: '$S3/release.sha256'
+        filepath: '/release.sha256'
+      - name: 'checksums GPG signature'
+        url: '$S3/release.sha256.asc'
+        filepath: '/release.sha256.asc'
+      - name: 'other release artifacts'
+        url: '$S3/index.html'
+        filepath: '/index.html'

.gitlab/ci/yaml/prepare.gitlab-ci.yml

+prepare CI image:
+  stage: prepare
+  extends:
+  - .common:docker-in-docker
+  - .rules:merge-requests:prepare:ci-image
+  script:
+  - docker build --pull --no-cache -t ${CI_IMAGE} -f ./.gitlab/ci/Dockerfile ./.gitlab/ci/
+  - docker login --username ${CI_REGISTRY_USER} --password ${CI_REGISTRY_PASSWORD} ${CI_REGISTRY}
+  - docker push ${CI_IMAGE}
+  - docker logout ${CI_REGISTRY}

.gitlab/ci/yaml/release.gitlab-ci.yml

+.release:
+  stage: release
+  image: ${CI_IMAGE}
+
+.release_S3:
+  dependencies:
+  - darwin amd64
+  - linux amd64
+  - linux arm
+  - linux arm64
+  - openbsd amd64
+  - windows amd64
+  variables:
+    S3_URL: s3://${S3_BUCKET}/${CI_COMMIT_REF_NAME}
+  script:
+  - ./.gitlab/ci/scripts/release_s3.sh
+  environment:
+    url: https://${S3_BUCKET}.s3.amazonaws.com/${CI_COMMIT_REF_NAME}/index.html
+
+release development S3:
+  extends:
+  - .rules:release:development
+  - .release
+  - .release_S3
+  environment:
+    name: development/S3/${CI_COMMIT_REF_NAME}
+    on_stop: stop release development S3
+
+stop release development S3:
+  dependencies: []
+  extends:
+  - .rules:release:development
+  - .release
+  - .release_S3
+  variables:
+    GIT_STRATEGY: none
+  script:
+  - aws s3 rm ${S3_URL} --recursive
+  when: manual
+  environment:
+    name: development/S3/${CI_COMMIT_REF_NAME}
+    action: stop
+
+release beta S3:
+  extends:
+  - .rules:release:beta
+  - .release
+  - .release_S3
+  environment:
+    name: beta/S3
+
+release RC S3:
+  extends:
+  - .rules:release:rc
+  - .release
+  - .release_S3
+  environment:
+    name: RC/S3
+
+release stable S3:
+  extends:
+  - .rules:release:stable
+  - .release
+  - .release_S3
+  environment:
+    name: stable/S3

.gitlab/ci/yaml/validate.gitlab-ci.yml

+validate:
+  stage: validate
+  extends:
+  - .common:docker-in-docker
+  - .rules:merge-requests
+  before_script:
+  - apk add -U make bash
+  script: make build validate