y1 baseline

.env

+# Reference documentation https://docs.docker.com/compose/environment-variables/
+# Focus default variables as production, to allow direct download and run in production
+# Required external variables that must be defined externally are those that have no value
+
+DEVSVC_BASE_PATH=development-services/
+TESTS_BASE_PATH=tests/
+
+#### Production related ####
+
+DOCKER_REGISTRY_PREFIX=optima-piacere-docker.artifact.tecnalia.com/
+PROJECT_NAME=piacere-monitoring
+COMPOSE_PROJECT_VERSION=production
+
+INFLUXD_BUCKET=
+INFLUXD_TOKEN=
+INFLUXD_ORG=
+INFLUXD_USERNAME=
+# Note: INFLUXD_PASSWORD piacere_must_be_longer_than_eight
+INFLUXD_PASSWORD=
+GRAFANA_ADMIN_USER=
+GRAFANA_ADMIN_PASSWORD=
+SMTP_USER_EMAIL=
+SMTP_API_KEY=
+TZ=
+GRAFANA_NOTIFICATION_PRIMARY_EMAIL=
+
+HTTPS_PORT=443
+# SERVER_HOST=
+
+# https://docs.docker.com/compose/reference/envvars/#compose_file#compose_project_name
+# these are docker-compose related environment variables
+COMPOSE_PROJECT_NAME=piacere-monitoring-production
+COMPOSE_FILE=docker-compose-pm.yaml:docker-compose-mc.yaml:docker-compose-iem.yaml:docker-compose-psl.yaml:docker-compose-traefik-selfsigned.yaml:docker-compose-expose.yaml:docker-compose-redirect-http.yaml
+
+#### Build related ####
+# DOCKER_BUILDKIT=1
+# COMPOSE_DOCKER_CLI_BUILD=1
+# COMPOSE_FILE=docker-compose-pm.yaml:docker-compose-mc.yaml:docker-compose-iem.yaml:docker-compose-psl.yaml:docker-compose-traefik-selfsigned.yaml:docker-compose-expose.yaml:docker-compose-redirect-http.yaml:build/docker-compose-pm.yaml:build/docker-compose-mc.yaml:build/docker-compose-iem.yaml:build/docker-compose-psl.yaml
+
+#### Development related, useful when docker-compose.dev is included ####
+# ADMIN_PASSWORD=adminpassword
+# CERTIFICATE_SIGNING_KEY_PASSPHRASE=
+
+# COMPOSE_FILE=docker-compose-pm.yaml:docker-compose-mc.yaml:docker-compose-iem.yaml:docker-compose-psl.yaml:development-services/docker-compose-traefik-tecnalia-selfsigned.yaml:docker-compose-expose.yaml:docker-compose-redirect-http.yaml:build/docker-compose-pm.yaml:build/docker-compose-mc.yaml:build/docker-compose-iem.yaml:build/docker-compose-psl.yaml:development-services/build/docker-compose-traefik-tecnalia-selfsigned.yaml:development-services/docker-compose.yaml:development-services/docker-compose-traefik-tecnalia-selfsigned.yaml:development-services/docker-compose-expose.yaml:development-services/docker-compose-redirect-http.yaml:development-services/build/docker-compose.yaml
+
+# HTTPS_PORT=8443
+# SERVER_HOST=192.168.56.1.nip.io

.gitlab-ci.yml

+stages:
+  - build
+  - deploy
+  - push
+  - promote
+  - stop
+
+variables:
+  GIT_SUBMODULE_STRATEGY: normal
+
+.common_variables:
+  variables:
+    TZ: Madrid
+    SERVER_HOST: piacere.esilab.org
+    INFLUXD_BUCKET: bucket
+    INFLUXD_ORG: piacere
+    INFLUXD_USERNAME: piacere
+    GRAFANA_ADMIN_USER: piacere
+    SMTP_USER_EMAIL: piacere@esilab.org
+    GRAFANA_NOTIFICATION_PRIMARY_EMAIL: piacere@esilab.org
+    PROJECT_NAME: piacere-monitoring
+
+.any_branch:
+  extends:
+    - .common_variables
+  variables:
+    # these variables take precedence over .env
+    # CERTIFICATE_SIGNING_KEY_PASSPHRASE defined in variables at settings/ci_cd
+    # SERVER_HOST defined in variables at settings/ci_cd
+    HTTPS_PORT: 8443
+    COMPOSE_FILE: "docker-compose.yaml"
+    COMPOSE_PROJECT_NAME: piacere-monitoring-any-branch
+    COMPOSE_PROJECT_VERSION: test
+    ADD_DEFAULT_CA: "true"
+  except:
+    - master
+    - develop
+    - pilots
+  tags:
+    - piacere-monitoring
+    - docker
+    - docker-compose
+    - any-branch
+
+.develop:
+  extends:
+    - .common_variables
+  variables:
+    # these variables take precedence over .env
+    # CERTIFICATE_SIGNING_KEY_PASSPHRASE defined in variables at settings/ci_cd
+    # SERVER_HOST defined in variables at settings/ci_cd
+    HTTPS_PORT: 8443
+    COMPOSE_FILE: "docker-compose.yaml:docker-compose-expose.yaml:docker-compose-dev.yaml:docker-compose-dev-expose.yaml"
+    COMPOSE_PROJECT_NAME: piacere-monitoring-develop
+    COMPOSE_PROJECT_VERSION: develop
+    ADD_DEFAULT_CA: "true"
+  only:
+    - develop
+  tags:
+    - piacere-monitoring
+    - docker
+    - docker-compose
+    - develop
+
+.master:
+  extends:
+    - .common_variables
+  variables:
+    # these variables take precedence over .env
+    # CERTIFICATE_SIGNING_KEY_PASSPHRASE defined in variables at settings/ci_cd
+    # SERVER_HOST defined in variables at settings/ci_cd
+    ADD_DEFAULT_CA: "true"
+  only:
+    - master
+  tags:
+    - piacere-monitoring
+    - docker
+    - docker-compose
+    - master
+
+.build:
+  script:
+    - echo "build images"
+    - docker-compose build --parallel
+
+.deploy:
+  script:
+    - echo "Deploy to the environment"
+    - docker-compose up -d
+    
+.push:
+  script:
+    - echo "pull base images from cache"
+    - docker login $DOCKER_REGISTRY -u $DOCKER_REGISTRY_USERNAME -p $DOCKER_REGISTRY_APIKEY
+    - echo "Deploy to the environment"
+    - docker-compose push
+
+.stop:
+  variables:
+    GIT_STRATEGY: none
+  script:
+    - echo "Stops the environment"
+    - docker-compose down --remove-orphans
+
+build_any_branch:
+  stage: build
+  extends:
+    - .any_branch
+    - .build
+
+deploy_any_branch:
+  stage: deploy
+  extends:
+    - .any_branch
+    - .deploy
+
+stop_any_branch:
+  variables:
+    COMPOSE_FILE: "docker-compose.yaml:docker-compose-tests.yaml"
+  stage: stop
+  extends:
+    - .any_branch
+    - .stop
+
+build_develop:
+  stage: build
+  extends:
+    - .develop
+    - .build
+
+deploy_develop:
+  stage: deploy
+  extends:
+    - .develop
+    - .deploy
+  environment:
+    name: develop
+    url: https://$SERVER_HOST:8443
+    on_stop: stop_develop
+
+stop_develop:
+  variables:
+    COMPOSE_FILE: "docker-compose.yaml:docker-compose-expose.yaml:docker-compose-dev.yaml:docker-compose-dev-expose.yaml"
+  stage: stop
+  extends:
+    - .develop
+    - .stop
+  environment:
+    name: develop
+    action: stop
+  when: manual
+
+build_master:
+  stage: build
+  extends:
+    - .master
+    - .build
+
+push_master_images_to_artifactory:
+  stage: push
+  extends:
+    - .master
+    - .push
+  when: manual
+
+deploy_master:
+  stage: deploy
+  extends:
+    - .master
+    - .deploy
+  environment:
+    name: master
+    url: https://$SERVER_HOST
+    on_stop: stop_master
+
+stop_master:
+  stage: stop
+  extends:
+    - .master
+    - .stop
+  environment:
+    name: master
+    action: stop
+  when: manual
+
+promote_to_master:
+  extends:
+    - .develop
+  stage: promote
+  when: manual
+  script:
+    - echo "in case custom-origin remote exist remove it"
+    - git remote rm custom-origin > /dev/null 2>&1 || true
+    - echo "checking the username"
+    - >
+      if ( [ -z "$AT" ] || [ "$AT" == "undefined" ] ) && ( ( [ -z "$USER" ] || [ "$USER" == "undefined" ] ) || ( [ -z "$PASSWORD" ] || [ "$PASSWORD" == "undefined" ] ) ); then 
+        echo "This job requires credentials to be added as variables (key/Value) in the job detail page before clicking the triggering this manual action button. Credentials can be provided in to ways: With an AT (access token) or with a USER and a PASSWORD. AT can be defined in your profile access tokens page https://git.code.tecnalia.com/-/profile/personal_access_tokens it must have read_repository and write_repository rights. USER and PASSWORD are those that you use to access gitlab. AT has prececedence over USER/PASSWORD."
+        exit 1
+      fi
+    - >
+      if [ ! -z "$AT" ]; then
+        export USER="oauth"
+        export PASSWORD="$AT"
+      fi
+    - echo "url encode credentials"
+    - export USER=$(python3 -c "import urllib.parse; print (urllib.parse.quote('''$USER'''))")
+    - export PASSWORD=$(python3 -c "import urllib.parse; print (urllib.parse.quote('''$PASSWORD'''))")
+    - git remote add custom-origin https://$USER:$PASSWORD@${CI_PROJECT_URL:8}
+    - git fetch custom-origin develop --depth=1
+    - git fetch custom-origin master --depth=1
+    - git checkout master
+    - git reset --hard custom-origin/develop 
+    - git push custom-origin -f
+    - git remote rm custom-origin > /dev/null 2>&1 || true

.gitmodules

+[submodule "development-services"]
+	path = development-services
+	url = ../../../../../smartdatalab/libraries/docker-compose/development-services.git
+[submodule "git/mc"]
+	path = git/mc
+	url = ../../../../../piacere/private/t61-runtime-monitoring/mc/mc.git
+[submodule "git/pmc"]
+	path = git/pmc
+	url = ../../../../../piacere/private/t61-runtime-monitoring/pm/pmc.git
+[submodule "git/iem"]
+	path = git/iem
+	url = ../../../../../piacere/private/t51-iem/iem.git
+[submodule "git/psl"]
+	path = git/psl
+	url = https://git.code.tecnalia.com/piacere/private/t62-self-learning/psl.git

README.md

+<!-- Gitlab Specific Markdown guide https://docs.gitlab.com/ee/user/markdown.html -->
+# docker-compose project
+This is a docker compose project that aim to instantiate the monitoring components for different purposes:
+* production
+* develop
+* debug
+* integration
+
+## Prerequisites
+In order to use this vagrantfile you need virtualbox and vagrant. 
+* https://www.docker.com/ (latest tested version 20.10.8, build 3967b7d)
+* https://docs.docker.com/compose/ (latest tested version 1.29.0, build 07737305)
+
+## How to use
+The docker-compose can be started in different ways depending your purpose. But in all the cases the first step is to download the repo. The repo relays in git submodules and therefore if we need to build the component imanges (the typical case) we need to user the "--recurse-submodules" option to download the submodules. 
+
+```bash
+git clone --recurse-submodules https://git.code.tecnalia.com/smartdatalab/management/deployments/pstaas/docker-compose.git
+cd catalogue-deploy
+````
+
+After that we will need to define those required variables that have no default value. We have put all these variables inside the [.env](.env) file. Are the ones that have no value assigned.
+
+```bash
+cat .env | grep -e ".*=\s*$"
+````
+
+Based on that we will be required to assign values to some variables:
+
+```bash
+export SERVER_HOST=somedomain
+export ADMIN_PASSWORD=somestrongpassword
+
+````
+
+### For production
+By default the project is configured to support the production scenario. 
+```bash
+docker-compose build
+docker-compose up -d --no-build --remove-orphans
+docker-compose down --remove-orphans
+echo if we what to remove also the persistent information
+docker-compose --volumes --remove-orphans
+````
+
+### For integration
+Another tipical scenario is to run the docker-compose as part of the continous integration in tecnalia in that case it can be interesting to use the custom tecnalia traefik. In this case, we will to define two additional variables:
+
+```bash
+export COMPOSE_FILE=docker-compose.yaml:docker-compose-dev.yaml:docker-compose-traefik-tecnalia-selfsigned.yaml:docker-compose-expose.yaml:docker-compose-redirect-http.yaml
+export CERTIFICATE_SIGNING_KEY_PASSPHRASE=thesecretkeytodecryptthecaprivatekey
+export HTTPS_PORT=8443
+docker-compose build
+docker-compose up -d --no-build --remove-orphans
+docker-compose down --remove-orphans
+echo if we what to remove also the persistent information
+docker-compose --volumes --remove-orphans
+````
+
+### For local development and debug
+Another tipical scenario is to run the docker-compose to test the whole solution during development or debug activities inside a development computer. This is a possible example for using this in tecnalia.
+
+```bash
+export SERVER_HOST=192.168.56.1.nip.io
+export COMPOSE_FILE=docker-compose.yaml:docker-compose-dev.yaml:docker-compose-traefik-tecnalia-selfsigned.yaml:docker-compose-expose.yaml:docker-compose-redirect-http.yaml:docker-compose-dev-expose.yaml
+export CERTIFICATE_SIGNING_KEY_PASSPHRASE=thesecretkeytodecryptthecaprivatekey
+export ADMIN_PASSWORD=theadminpassword
+export HTTPS_PORT=8443
+docker-compose build
+docker-compose up -d --no-build --remove-orphans
+docker-compose down --remove-orphans
+echo if we what to remove also the persistent information
+docker-compose down --volumes --remove-orphans
+````
+
+## How to access
+Once we sucessfully deploy the docker-compose, we will be able to access the services at (supposing SERVER_HOST=192.168.56.1.nip.io and HTTPS_PORT=8443):
+* https://influxdb.192.168.56.1.nip.io:8443/ to access performance monitoring influxdb
+* https://grafana.192.168.56.1.nip.io:8443/ to access performance monitoring grafana
+* https://pmc.192.168.56.1.nip.io:8443/pmc/api/v1/ to access performance monitoring controller swagger ui
+* https://mc.192.168.56.1.nip.io:8443/mc/api/v1/ui/ to access monitoring controller swagger ui
+* https://iem.192.168.56.1.nip.io:8443/docs to access iem swagger ui
+* https://registry.192.168.56.1.nip.io:8443/ to access jhipster registry for development support
+
+Appart from those that we can activate if we are using other services, such as [development-services](git/deploy/development-services/README.md)
+* https://traefik.192.168.56.1.nip.io:8443/ to access traefik dashboard 
+* https://traefik.192.168.56.1.nip.io:8443/api/http/routers to access traefik api 
+* https://portainer.192.168.56.1.nip.io:8443/ to access portainer 
+* https://ca.192.168.56.1.nip.io:8443/ to access the tecnalia ca in case we use the tecnalia traefik
+* https://auth.192.168.56.1.nip.io:8443/ to access keycloak 
+* ...
+
+## How to access at integration
+The projects has a gitlab-ci workflow to continously deploy the catalogue at a integration server. Providing the integration server exists and it has a properly configured gitlab-runner with the required packages (mainly docker and docker compose). We will be able to access the services at:
+* https://influxdb.tcn.poc.piacere.esilab.org:8443/ to access performance monitoring influxdb
+* https://grafana.tcn.poc.piacere.esilab.org:8443/ to access performance monitoring grafana
+* https://pmc.tcn.poc.piacere.esilab.org:8443/pmc/api/v1/ to access performance monitoring controller swagger ui
+* https://mc.tcn.poc.piacere.esilab.org:8443/mc/api/v1/ui/ to access monitoring controller swagger ui
+* https://iem.tcn.poc.piacere.esilab.org:8443/docs to access iem swagger ui
+* https://registry.tcn.poc.piacere.esilab.org:8443/ to access jhipster registry for development support
+
+Appart from those that we can activate if we are using other services, such as [development-services](git/deploy/development-services/README.md)
+* https://traefik.tcn.poc.piacere.esilab.org:8443/ to access traefik dashboard 
+* https://traefik.tcn.poc.piacere.esilab.org:8443/api/http/routers to access traefik api 
+* https://portainer.tcn.poc.piacere.esilab.org:8443/ to access portainer 
+* https://ca.tcn.poc.piacere.esilab.org:8443/ to access the tecnalia ca in case we use the tecnalia traefik
+* https://auth.tcn.poc.piacere.esilab.org:8443/ to access keycloak 
+* ...

build/docker-compose-iem.yaml

+version: "2.4"
+
+services:
+  ## Common structure
+  iem:
+    build:
+      context: git/iem
+      dockerfile: Dockerfile

build/docker-compose-mc.yaml

+version: "2.4"
+
+services:
+  ## Common structure
+  influxdb:
+    build:
+      context: git/influxdb
+      dockerfile: Dockerfile
+
+  grafana:
+    build:
+      context: git/grafana
+      dockerfile: Dockerfile
+
+  smtp:
+    build:
+      context: git/smtp
+      dockerfile: Dockerfile
+
+  mc:
+    build:
+      context: git/mc
+      dockerfile: Dockerfile
+
+  pmc:
+    build:
+      context: git/pmc
+      dockerfile: Dockerfile

build/docker-compose-pm.yaml

+version: "2.4"
+
+services:
+  ## Common structure
+  influxdb:
+    build:
+      context: git/influxdb
+      dockerfile: Dockerfile
+
+  grafana:
+    build:
+      context: git/grafana
+      dockerfile: Dockerfile
+
+  smtp:
+    build:
+      context: git/smtp
+      dockerfile: Dockerfile
+
+  mc:
+    build:
+      context: git/mc
+      dockerfile: Dockerfile
+
+  pmc:
+    build:
+      context: git/pmc
+      dockerfile: Dockerfile

build/docker-compose-psl.yaml

+version: "2.4"
+
+services:
+  ## Common structure
+  psl:
+    build:
+      context: git/psl
+      dockerfile: Dockerfile

development-services/.env

+# Reference documentation https://docs.docker.com/compose/environment-variables/
+# Focus default variables as production, to allow direct download and run in production
+# Required external variables that must be defined externally are those that have no value
+
+TZ=Madrid
+DOCKER_REGISTRY_PREFIX=optima-devsvc-docker.artifact.tecnalia.com/
+PROJECT_NAME=medina-devsvc
+COMPOSE_PROJECT_VERSION=master
+HTTPS_PORT=443
+SERVER_HOST=
+ADMIN_PASSWORD=
+BUILD_PROFILE=prod
+DEVSVC_BASE_PATH=development-services/
+
+# https://docs.docker.com/compose/reference/envvars/#compose_file#compose_project_name
+# these are docker-compose related environment variables
+COMPOSE_PROJECT_NAME=devsvc-master
+COMPOSE_FILE=docker-compose.yaml:docker-compose-expose.yaml:docker-compose-redirect-http.yaml:docker-compose-keycloak.yaml:docker-compose-traefik-tecnalia-selfsigned.yaml
+
+### Build related
+EXTRA_CA_URL=https://git.code.tecnalia.com/smartdatalab/ca/-/raw/master/ca.crt.pem
+DOCKER_BUILDKIT=1
+COMPOSE_DOCKER_CLI_BUILD=1
+# COMPOSE_FILE=docker-compose.yaml:docker-compose-expose.yaml:docker-compose-redirect-http.yaml:build/docker-compose.yaml:docker-compose-keycloak.yaml:build/docker-compose-keycloak.yaml:docker-compose-traefik-tecnalia-selfsigned.yaml:build/docker-compose-traefik-tecnalia-selfsigned.yaml
+
+#### Development related, useful when docker-compose.dev is included ####
+# HTTPS_PORT=8443
+# SERVER_HOST=192.168.56.1.nip.io

development-services/.gitmodules

+[submodule "traefik"]
+	path = traefik
+	url = ../../../../smartdatalab/libraries/docker/traefik.git
+[submodule "portainer"]
+	path = portainer
+	url = ../../../../smartdatalab/libraries/docker/portainer.git

development-services/README.md

+# development-services
+
+This docker-compose file generalizes some services that we use in tecnalia to monitor and debug containers in integration servers.
+* traefik dashboard
+* traefik api
+* portainer
+* openstack to provide sso access to services
+
+In future
+* monitoring with telegraf
+* monitoring db and render with infuxdb and grafana (this requires resources)
+
+## Use
+This docker compose is not planned to be used alone but as a resource in another docker compose projects by extending DOCKER_FILE variable.
+
+```bash
+export DOCKER_FILE=[...];path/docker-compose-traefik-tecnalia-selfsigned.yaml;path/docker-compose-dev.yaml  
+```
+
+## Resources
+Besides the docker-composes we also provide some additional resources:
+* traefik examples using the official or our customized traefik
+
+## How to access
+the entrypoints for the services will be in subdomains, asuming a base domain such as 192.168.56.1.nip.io:
+* https://traefik.192.168.56.1.nip.io:8443/ to access traefik dashboard 
+* https://traefik.192.168.56.1.nip.io:8443/api/http/routers to access traefik api 
+* https://portainer.192.168.56.1.nip.io:8443/ to access portainer 
+* ...
+

development-services/build/cache/docker-compose-keycloak.yaml

+version: "3.8"
+
+services:
+  keycloak:
+    build:
+      args:
+        - BUILDKIT_INLINE_CACHE=1
+      cache_from:
+        - ${DOCKER_REGISTRY_CACHE_PREFIX}${PROJECT_NAME:?err}/keycloak:${DOCKER_REGISTRY_CACHE_VERSION:?err}

development-services/build/cache/docker-compose-traefik-tecnalia-selfsigned.yaml

+version: "3.8"
+
+services:
+  ## Common structure
+  traefik:
+    build:
+      args:
+        - BUILDKIT_INLINE_CACHE=1
+      cache_from:
+        - ${DOCKER_REGISTRY_CACHE_PREFIX}${PROJECT_NAME:?err}/traefik:${DOCKER_REGISTRY_CACHE_VERSION:?err}

development-services/build/cache/docker-compose.yaml

+version: "3.8"
+
+services:
+  portainer:
+    build:
+      args:
+        - BUILDKIT_INLINE_CACHE=1
+      cache_from:
+        - ${DOCKER_REGISTRY_CACHE_PREFIX}${PROJECT_NAME:?err}/portainer:${DOCKER_REGISTRY_CACHE_VERSION:?err}

development-services/build/docker-compose-keycloak.yaml

+version: "3.8"
+
+services:
+  keycloak:
+    build:
+      context: ${DEVSVC_BASE_PATH}keycloak
+      dockerfile: Dockerfile

development-services/build/docker-compose-traefik-tecnalia-selfsigned.yaml

+version: "3.8"
+
+services:
+  ## Common structure
+  traefik:
+    build:
+      context: ${DEVSVC_BASE_PATH}traefik
+      dockerfile: Dockerfile

development-services/build/docker-compose.yaml

+version: "3.8"
+
+services:
+  portainer:
+    build:
+      context: ${DEVSVC_BASE_PATH}portainer
+      dockerfile: Dockerfile

development-services/docker-compose-dev-expose.yaml

+version: "3.8"
+
+services:
+  ## Common structure
+  traefik:
+    ports:
+      - 8080:8080
+
+  portainer:
+    ports:
+      - 9000:9000

development-services/docker-compose-expose.yaml

+version: "3.8"
+
+services:
+  ## Common structure
+  traefik:
+    ports:
+      - ${HTTPS_PORT:?err}:${HTTPS_PORT:?err}

development-services/docker-compose-keycloak.yaml

+# This configuration is intended for development purpose, it's **your** responsibility to harden it for production
+version: '3.8'
+services:
+  traefik:
+    networks:
+      default:
+        aliases:
+          - auth.${SERVER_HOST}
+
+  keycloak:
+    image: ${DOCKER_REGISTRY_PREFIX}${PROJECT_NAME:?err}/keycloak:${COMPOSE_PROJECT_VERSION:?err}
+    environment:
+      KEYCLOAK_USER: admin
+      KEYCLOAK_PASSWORD: ${ADMIN_PASSWORD:?err}
+      DB_VENDOR: h2
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.routers.keycloak.rule=Host(`auth.${SERVER_HOST:?err}`)"
+      - "traefik.http.routers.keycloak.entrypoints=websecure"
+      - "traefik.http.routers.keycloak.service=keycloak"
+      - "traefik.http.services.keycloak.loadbalancer.server.scheme=http"
+      - "traefik.http.services.keycloak.loadbalancer.server.port=9080"