From 1c700deb66270456ef0263b39754ed0b7b3e35de Mon Sep 17 00:00:00 2001
From: "Diaz de Arcaya Serrano, Josu" <josu.diazdearcaya@tecnalia.com>
Date: Thu, 18 Jan 2024 11:24:52 +0100
Subject: [PATCH] updating main branch
---
README.md | 32 +-
docs/docs_files/01-intro.rst | 7 +-
docs/kr-10.feature | 49 +++
docs/sequence-diagrams/.gitignore | 1 +
.../51-request-deployment-status.puml | 26 ++
.../51-start-deployment.puml | 34 ++
.../51-start-undeployment.puml | 34 ++
docs/sequence-diagrams/README.md | 27 ++
iem-api/.python-version | 2 +-
iem-api/Dockerfile | 25 +-
iem-api/certs/config | 3 +
iem-api/certs/id_rsa | 27 ++
iem-api/certs/id_rsa.pub | 1 +
iem-api/db/.gitignore | 1 +
iem-api/logging.ini | 31 ++
iem-api/main.py | 110 ++++--
iem-api/requirements.txt | 3 +-
iem-api/src/__init__.py | 1 +
iem-api/src/_version.py | 4 +
iem-api/src/core/engine.py | 139 ++++---
iem-api/src/core/iem.py | 342 +++++++++++++-----
iem-api/src/core/persistence.py | 86 +----
iem-api/src/core/utils.py | 39 +-
iem-api/src/resources/id_iem | 38 ++
iem-api/src/resources/id_iem.pub | 1 +
iem-api/tests/__init__.py | 0
iem-api/tests/it/__init__.py | 0
iem-api/tests/it/test_it_iem.py | 149 ++++++++
iem-api/tests/resources/ansible.zip | Bin 0 -> 3735 bytes
iem-api/tests/resources/aws.zip | Bin 0 -> 1230 bytes
iem-api/tests/resources/docker.zip | Bin 0 -> 5400 bytes
iem-api/tests/resources/dummy.zip | Bin 0 -> 3700 bytes
iem-api/tests/resources/main.yml | 8 +
iem-api/tests/resources/openstack.zip | Bin 0 -> 2117 bytes
iem-api/tests/resources/shs-bundle.zip | Bin 0 -> 2223 bytes
iem-api/tests/resources/shs.zip | Bin 0 -> 2169 bytes
iem-api/tests/unit/__init__.py | 0
iem-api/tests/unit/test_iem.py | 31 ++
iem-api/tests/unit/test_main.py | 224 ++++++++++++
iem-api/tests/unit/test_persistence.py | 32 ++
openapi.json | 2 +-
sonar-project.properties | 5 +
42 files changed, 1195 insertions(+), 319 deletions(-)
create mode 100644 docs/kr-10.feature
create mode 100755 docs/sequence-diagrams/.gitignore
create mode 100644 docs/sequence-diagrams/51-request-deployment-status.puml
create mode 100644 docs/sequence-diagrams/51-start-deployment.puml
create mode 100644 docs/sequence-diagrams/51-start-undeployment.puml
create mode 100755 docs/sequence-diagrams/README.md
create mode 100644 iem-api/certs/config
create mode 100644 iem-api/certs/id_rsa
create mode 100644 iem-api/certs/id_rsa.pub
create mode 100644 iem-api/db/.gitignore
create mode 100644 iem-api/logging.ini
create mode 100644 iem-api/src/__init__.py
create mode 100644 iem-api/src/_version.py
create mode 100644 iem-api/src/resources/id_iem
create mode 100644 iem-api/src/resources/id_iem.pub
create mode 100644 iem-api/tests/__init__.py
create mode 100644 iem-api/tests/it/__init__.py
create mode 100644 iem-api/tests/it/test_it_iem.py
create mode 100644 iem-api/tests/resources/ansible.zip
create mode 100644 iem-api/tests/resources/aws.zip
create mode 100644 iem-api/tests/resources/docker.zip
create mode 100644 iem-api/tests/resources/dummy.zip
create mode 100644 iem-api/tests/resources/main.yml
create mode 100644 iem-api/tests/resources/openstack.zip
create mode 100644 iem-api/tests/resources/shs-bundle.zip
create mode 100644 iem-api/tests/resources/shs.zip
create mode 100644 iem-api/tests/unit/__init__.py
create mode 100644 iem-api/tests/unit/test_iem.py
create mode 100644 iem-api/tests/unit/test_main.py
create mode 100644 iem-api/tests/unit/test_persistence.py
create mode 100644 sonar-project.properties
diff --git a/README.md b/README.md
index b18ac48..4daa1ef 100644
--- a/README.md
+++ b/README.md
@@ -1,28 +1,27 @@
# T51 IaC Executor Manager
-Running the server
+Running the server with uvicorn
+
```bash
uvicorn main:app --reload
```
-###### Containers
+Execute it directly
-Containerize the IEM
```bash
-docker build --build-arg API_KEY=$API_KEY -t optima-piacere-docker-dev.artifact.tecnalia.com/wp5/iem-api:y1 .
+./main.py
```
-Similarly, docker compose can be used to build both
-```bash
-docker-compose build
-```
+###### Containers
+
+Containerize the IEM
-It can also be used to push them to the registry
```bash
-docker-compose push
+docker build --build-arg API_KEY=$API_KEY -t optima-piacere-docker-dev.artifact.tecnalia.com/wp5/iem-api:y1 .
```
-Run the IEM
+Run the dockerized IEM
+
```bash
docker run -p 8000:8000 optima-piacere-docker-dev.artifact.tecnalia.com/wp5/iem-api:y1
```
@@ -41,4 +40,15 @@ Run a single test
nose2 -v tests.core.test_iem.TestIem.test_deploy_destroy_openstack
```
+Run unit and integration tests
+
+```bash
+nose2 -v tests.unit
+nose2 -v tests.it
+```
+
+Integration tests are prevented from being executed unless we deliberately define an environment variable
+```bash
+AWS=1 nose2 -v tests.it
+```
diff --git a/docs/docs_files/01-intro.rst b/docs/docs_files/01-intro.rst
index d3563bc..45533a1 100644
--- a/docs/docs_files/01-intro.rst
+++ b/docs/docs_files/01-intro.rst
@@ -4,6 +4,7 @@
Introduction
************
-The IaC Execution Manager utilizes different technologies that can be used for the provisioning, configuration, and orchestration of the different infrastructural devices that can be found in a production deployment. This has served us to provide evidence and reasoning for the selection of the technologies that the IEM prototype is going to utilize.
-
-This prototype is viable for the deployment of different IaC technologies that cover the provisioning and the configuration of the infrastructural devices required for the projects utilizing the PIACERE framework. It provides a unified interface for other components so they can interact with the IEM in a unified manner. It can also be deployed in production utilizing container-based technologies which makes this prototype viable to be operationalized in public and private cloud provides, and on premises. For this prototype, the IEM supports two well established technologies (i.e. Ansible and Terraform) that are able to provision the different infrastructural devices required by the use cases, and the configuration of each of these infrastructural devices so they can accommodate the applications to be allocated.
+..
+ TODO Provide a brief description of the component here. Outline its goals, functionalities, etc.;
+ Mention subcomponents or extra delivered tools etc., with rst references to adequate sections.
+
\ No newline at end of file
diff --git a/docs/kr-10.feature b/docs/kr-10.feature
new file mode 100644
index 0000000..2dbbdf9
--- /dev/null
+++ b/docs/kr-10.feature
@@ -0,0 +1,49 @@
+Feature: PIACERE Run Time
+
+# The input of this scenario is detailed in the following
+# https://git.code.tecnalia.com/piacere/private/t51-iem/iem/-/blob/y2/openapi.json#/deployments/deploy_deployments__post
+# The following scenario relates to REQ81, REQ83, REQ84, REQ87
+Scenario: Deploy a fresh project which comprises terraform, ansible, and docker
+Given a project bundle in the relevant IaC technologies (terraform, ansible, docker-compose), the deployment id, and the required cloud credentials
+ When the user triggers the deployment
+ Then the IEM is invoked
+ And executes the stages of the bundle asyncronously
+ And the user is notified that the deployment has been accepted
+
+# The input of this scenario is detailed in the following
+# https://git.code.tecnalia.com/piacere/private/t51-iem/iem/-/blob/y2/openapi.json#/deployments/read_status_deployment_deployments__deployment_id__get
+# The following scenario relates to REQ55, REQ82
+Scenario: Query the status of a running project
+Given the deployment id of an already existing project
+ When the user queries the status of the project
+ Then the IEM is invoked
+ And the user is notified of the status
+
+# The input of this scenario is detailed in the following
+# https://git.code.tecnalia.com/piacere/private/t51-iem/iem/-/blob/y2/openapi.json#/deployments/undeploy_undeploy__post
+# The following scenario relates to REQ81, REQ83, REQ84, REQ85
+Scenario: Undeploy a project
+Given the deployment id of an already existing project and the required cloud credentials
+ When the user triggers the undeployment
+ Then the IEM is invoked
+ And tears down the entire deployment asyncronously
+ And the user is notified that the undeployment has been accepted
+
+# The input of this scenario is detailed in the following
+# https://git.code.tecnalia.com/piacere/private/t51-iem/iem/-/blob/y2/openapi.json#/deployments/read_status_deployment_deployments__deployment_id__get
+# The following scenario relates to REQ55, REQ82
+Scenario: Query the status of an undeployed project
+Given the deployment id of an undeployed project
+ When the user queries the status of the project
+ Then the IEM is invoked
+ And the user is notified of the status
+
+# The input of this scenario is detailed in the following
+# https://git.code.tecnalia.com/piacere/private/t51-iem/iem/-/blob/y2/openapi.json#/deployments/deploy_deployments__post
+# The following scenario relates to REQ12, REQ81, REQ83, REQ84, REQ87
+Scenario: Redeploy a project
+Given a project bundle in the relevant IaC technologies (terraform, ansible, docker-compose), the deployment id, and the required cloud credentials
+ When the user triggers the deployment
+ Then the IEM is invoked
+ And executes the stages of the bundle asyncronously
+ And the user is notified that the deployment has been accepted
diff --git a/docs/sequence-diagrams/.gitignore b/docs/sequence-diagrams/.gitignore
new file mode 100755
index 0000000..981aeb8
--- /dev/null
+++ b/docs/sequence-diagrams/.gitignore
@@ -0,0 +1 @@
+/out
\ No newline at end of file
diff --git a/docs/sequence-diagrams/51-request-deployment-status.puml b/docs/sequence-diagrams/51-request-deployment-status.puml
new file mode 100644
index 0000000..752fd95
--- /dev/null
+++ b/docs/sequence-diagrams/51-request-deployment-status.puml
@@ -0,0 +1,26 @@
+@startuml
+
+title Request the Current Status of a Deployment
+
+participant "PRC" as DESIDE
+
+box "IaC Execution Manager" #LightBlue
+participant "Rest API" as RTIEM_api #99FF99
+participant Core as RTIEM_core #99FF99
+participant Persistence as RTIEM_db #99FF99
+end box
+
+
+DESIDE -> RTIEM_api: Deployment Status Request
+
+RTIEM_api -> RTIEM_core: Deployment Status Request
+
+RTIEM_core -> RTIEM_db: Deployment Status Request
+
+RTIEM_core <-- RTIEM_db: Deployment Status Response
+
+RTIEM_api <-- RTIEM_core: Deployment Status Response
+
+DESIDE <-- RTIEM_api: Deployment Status Response
+
+@enduml
diff --git a/docs/sequence-diagrams/51-start-deployment.puml b/docs/sequence-diagrams/51-start-deployment.puml
new file mode 100644
index 0000000..ef8b0ba
--- /dev/null
+++ b/docs/sequence-diagrams/51-start-deployment.puml
@@ -0,0 +1,34 @@
+@startuml
+
+title Initiate Deployment
+
+participant "Runtime Controller (PRC)" as RTPRC
+
+box "IaC Execution Manager" #LightBlue
+participant "Rest API" as RTIEM_api #99FF99
+participant Core as RTIEM_core #99FF99
+participant Persistence as RTIEM_db #99FF99
+participant "Executor" as executor #99FF99
+
+end box
+
+collections "Resource Provider" as infraresource
+
+RTPRC -> RTIEM_api: Deployment Request
+RTPRC <-- RTIEM_api: Deployment Response
+
+RTIEM_api -> RTIEM_core: Deployment Request
+
+RTIEM_core -> RTIEM_db: Save Deployment Started
+
+RTIEM_core -> executor: Deployment Request
+
+executor -> infraresource: Deploy Commands
+executor -> infraresource: ...
+executor -> infraresource: Deploy Commands
+
+executor -> RTIEM_core: Deployment Response
+
+RTIEM_core -> RTIEM_db: Save Deployment Status
+
+@enduml
diff --git a/docs/sequence-diagrams/51-start-undeployment.puml b/docs/sequence-diagrams/51-start-undeployment.puml
new file mode 100644
index 0000000..35bc219
--- /dev/null
+++ b/docs/sequence-diagrams/51-start-undeployment.puml
@@ -0,0 +1,34 @@
+@startuml
+
+title Initiate Undeployment
+
+participant "Runtime Controller (PRC)" as RTPRC
+
+box "IaC Execution Manager" #LightBlue
+participant "Rest API" as RTIEM_api #99FF99
+participant Core as RTIEM_core #99FF99
+participant Persistence as RTIEM_db #99FF99
+participant "Executor" as executor #99FF99
+end box
+
+collections "Resource Provider" as infraresource
+
+RTPRC -> RTIEM_api: Undeployment Request
+RTPRC <-- RTIEM_api: Undeployment Response
+
+RTIEM_api -> RTIEM_core: Undeployment Request
+
+RTIEM_core -> RTIEM_db: Save Undeployment Started
+
+RTIEM_core -> executor: Undeployment Request
+
+executor -> infraresource: Uneploy Commands
+executor -> infraresource: ...
+executor -> infraresource: Undeploy Commands
+
+executor -> RTIEM_core: Undeployment Response
+
+RTIEM_core -> RTIEM_db: Save Undeployment Status
+
+@enduml
+
diff --git a/docs/sequence-diagrams/README.md b/docs/sequence-diagrams/README.md
new file mode 100755
index 0000000..cc0a33c
--- /dev/null
+++ b/docs/sequence-diagrams/README.md
@@ -0,0 +1,27 @@
+# T51 IaC Executor Manager Secuence diagrams
+
+This folder contains the sequence diagrams developed for the T51 IEM. They have been developed using plantuml
+* https://plantuml.com
+
+These files follow a very simple text based syntax. ie
+```
+Bob->Alice : Hello!
+```
+which renders (providing plantuml is enabled in gitlab https://docs.gitlab.com/ee/administration/integration/plantuml.html) as
+
+```plantuml
+Bob->Alice : Hello!
+```
+we can also specify a file
+
+```plantuml source="51-start-deployment.puml"
+```
+
+To be able to edit them and check the rendering there are several options:
+* Edit and generate the file using the jar, which is not very user friendly
+``` java -jar plantuml.jar sequenceDiagram.txt ```
+* Use an IDE and a plugin. There are plugins available for different IDEs,i.e.
+ * eclipse https://plantuml.com/eclipse
+ * visual code https://marketplace.visualstudio.com/items?itemName=jebbs.plantuml
+
+
diff --git a/iem-api/.python-version b/iem-api/.python-version
index f69abe4..0a59033 100644
--- a/iem-api/.python-version
+++ b/iem-api/.python-version
@@ -1 +1 @@
-3.9.7
+3.9.10
diff --git a/iem-api/Dockerfile b/iem-api/Dockerfile
index 635043b..3010302 100644
--- a/iem-api/Dockerfile
+++ b/iem-api/Dockerfile
@@ -1,17 +1,16 @@
FROM hashicorp/terraform:1.1.4
-ARG API_KEY
+COPY requirements.txt /tmp/requirements.txt
+RUN apk add py3-pip cargo g++ python3-dev file libffi-dev openssl-dev bash python3 gnupg
+RUN pip3 install -r /tmp/requirements.txt
+# install docker stack
+RUN apk add docker docker-compose
-ENV API_KEY=$API_KEY
+ENV API_KEY=changeme
ENV IEM_HOME=/opt/iem/
+ENV DOCKERIZED=true
COPY src/resources/ansible.cfg /etc/ansible/ansible.cfg
-COPY requirements.txt /tmp/requirements.txt
-COPY src ${IEM_HOME}src
-COPY main.py ${IEM_HOME}main.py
-
-RUN apk add py3-pip cargo g++ python3-dev file libffi-dev openssl-dev bash python3=3.9.13-r1 gnupg
-RUN pip3 install -r /tmp/requirements.txt
# RUN adduser -h ${IEM_HOME} -S -D iem
COPY certs/config ${IEM_HOME}.ssh/config
@@ -24,11 +23,17 @@ RUN adduser -h ${IEM_HOME} -S -D iem && \
chmod 0600 ${IEM_HOME}.ssh/id_rsa && \
chmod 0644 ${IEM_HOME}.ssh/id_rsa.pub
USER iem
-RUN ansible-galaxy collection install community.general
COPY roles.yml /tmp/roles.yml
RUN ansible-galaxy install -r /tmp/roles.yml
+RUN mkdir -p ${IEM_HOME}db && \
+ mkdir -p ${IEM_HOME}deployments
+
+COPY src ${IEM_HOME}src
+COPY main.py ${IEM_HOME}main.py
+COPY logging.ini ${IEM_HOME}logging.ini
+
ENTRYPOINT ["/usr/bin/env"]
WORKDIR ${IEM_HOME}
-CMD /usr/bin/uvicorn main:app --host 0.0.0.0
+CMD /usr/bin/uvicorn main:app --host 0.0.0.0 --log-level info
EXPOSE 8000
diff --git a/iem-api/certs/config b/iem-api/certs/config
new file mode 100644
index 0000000..a3dd79d
--- /dev/null
+++ b/iem-api/certs/config
@@ -0,0 +1,3 @@
+Host *
+ StrictHostKeyChecking no
+ UserKnownHostsFile=/dev/null
\ No newline at end of file
diff --git a/iem-api/certs/id_rsa b/iem-api/certs/id_rsa
new file mode 100644
index 0000000..86a197b
--- /dev/null
+++ b/iem-api/certs/id_rsa
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEA1FrTNE42EgZr9WJNMtvpKFHYhPUJ4lzEp83EM0jYY3TyjmIe
+ThMuqMLAHCk22fl4a8PttucggJ5ZWKhcJh623/y8AybJcmqZgq9a41Q609dmirf0
+7frCl+6zL8Mqy2Le2BD4eRADcq11s8r8Ys6J+EBPHQgEnK9CeZLSc/WFRlVr4bOD
+s0bEouDxjTAMYjYcpsCwqYgGdIXI9WWsnt3RvcEe8CaiTqoyDN8ZtgkG6MweSrTQ
+js8ySHO6o25cOoF7aT9Ihhf32I+KUanNIOvk3RAw2z1FK5xkFbbqMggZqz7rJn3M
+sn2dDiCQi2CWox2OYXV/jJKLC3UFuOX64fS9cwIDAQABAoIBAQCs69Tm1/Vx0ibh
+aA4DJ06C1bsh8cP9v5soJgfp1xzWSGooBcA1xasOI6B6jhkrgNlNr/uIIEe4VLne
+1yJKrGIwnUagrloGQMYGxDKXwYQx80p+FXRuwe7p96eUcjIL8tQSUCd1tdOI87VQ
+FjBVaWiybfO+aUQQLytLgoK7iKfhb7vO+9F+ZK1iDjBDNxFuiOM5zoeWOI7boYkD
+2yXIkwoBePS2rosbPLa649sVakKex2WhQdUFst4Zba2RhnWQBXUY44LvEK5TzScF
+FyYphPOUSplbzzM2+fuOna91NIWmJyHmf15lj7X9kC66XFIZMlvapksB8stEpDiA
+4al3IdBJAoGBAPPuM3xkr/kQYYn7E42fgpmINZ78V48gMLhpyUOabZtF8inMyMPB
+q7kfHns8Vx0ET8grSNr3qwDDV82lwvGqRCFChASMdQRR9LanydkDSeqpuZyQtVlt
+A/65YUdcNY7Vy+M+fRh5Srh/6qcO3beLeLWXbJ4RHBP/OEmHuF4mLfgVAoGBAN7c
+qdxTOhXPvOU69Bs5rJdfo6qBI1Yg8MCGctsUwPFpw1kW773ROOPa6XI6D74Dsdg8
+ypZ+IC3pRVtx61Xi3NOwxWNTTG+dyUgTSFz+WKjywXZXeHIbWngiFqk8JFYQWPzk
+6YaJk4tZhk2YuNNaCCYRgQqyWv8coEurRlMXZHlnAoGBALcJwdaQ0z8oXJimL4jw
+7ZX5kIrpPWanuAdZUe4Jfj+qX8mf4fKKbCowQLYmlBOw/ZDtcfDlMYsUCdnFjZ+7
+rP3sJJYpM1F3khJRm3PdNOUCUMY8C+i7lejZADcE6SdyJFkztbjcowYI7nJHBHZL
+ENvqcVW27wPOWlVKozz6lzn1AoGALVwmaoS6DtRwcwuzwZLUkR7TNhIAujgMKHN1
+DyhDOR+4tfpYI39hH+dfmnM83wTrfsKozUawkAepqToflySMo72X/2Zl6VXpMPVT
+xjGyo/h87fRRvI/asxblG9702luLcTW6XjrEQBmhn0uVWtc5T15CsIWqxb/y1FPx
+BVp+hcMCgYAlJXbjzjbbDoIOCsXPSPe9voBL8zVwp0aNuvQcuB/vCt1n1c1DWuPr
+AGMy/fRwY0Znag+ODMuulm7RgXUQy6ifJHiz9cKVGg/mGifaJSjgC+1AI9HFlij3
+asM5CueU0gK974rDxQkwmIWpRH57+kf6s8tGDrPPvqX9S4p3oxFlTw==
+-----END RSA PRIVATE KEY-----
diff --git a/iem-api/certs/id_rsa.pub b/iem-api/certs/id_rsa.pub
new file mode 100644
index 0000000..245caaf
--- /dev/null
+++ b/iem-api/certs/id_rsa.pub
@@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDUWtM0TjYSBmv1Yk0y2+koUdiE9QniXMSnzcQzSNhjdPKOYh5OEy6owsAcKTbZ+Xhrw+225yCAnllYqFwmHrbf/LwDJslyapmCr1rjVDrT12aKt/Tt+sKX7rMvwyrLYt7YEPh5EANyrXWzyvxizon4QE8dCAScr0J5ktJz9YVGVWvhs4OzRsSi4PGNMAxiNhymwLCpiAZ0hcj1Zaye3dG9wR7wJqJOqjIM3xm2CQbozB5KtNCOzzJIc7qjblw6gXtpP0iGF/fYj4pRqc0g6+TdEDDbPUUrnGQVtuoyCBmrPusmfcyyfZ0OIJCLYJajHY5hdX+MkosLdQW45frh9L1z josu@WKM0092A
diff --git a/iem-api/db/.gitignore b/iem-api/db/.gitignore
new file mode 100644
index 0000000..8d96d78
--- /dev/null
+++ b/iem-api/db/.gitignore
@@ -0,0 +1 @@
+iem.db
diff --git a/iem-api/logging.ini b/iem-api/logging.ini
new file mode 100644
index 0000000..7a1ca11
--- /dev/null
+++ b/iem-api/logging.ini
@@ -0,0 +1,31 @@
+[loggers]
+keys=root,src,uvicorn
+
+[handlers]
+keys=stream_handler
+
+[formatters]
+keys=formatter
+
+[logger_root]
+level=INFO
+handlers=stream_handler
+
+[logger_src]
+level=INFO
+handlers=stream_handler
+qualname=src
+propagate=0
+
+[logger_uvicorn]
+level=INFO
+handlers=stream_handler
+qualname=uvicorn
+propagate=0
+
+[handler_stream_handler]
+class=StreamHandler
+formatter=formatter
+
+[formatter_formatter]
+format=%(asctime)s %(name)-12s %(levelname)-8s %(message)s
diff --git a/iem-api/main.py b/iem-api/main.py
index 3fb84f4..ec8ad09 100755
--- a/iem-api/main.py
+++ b/iem-api/main.py
@@ -1,33 +1,34 @@
+#!/usr/bin/env python3
+
import json
import logging
-import os
-
-from fastapi import FastAPI, BackgroundTasks, status, Security, Depends, HTTPException
-from fastapi.openapi.utils import get_openapi
-from fastapi.security.api_key import APIKeyHeader, APIKey
from typing import List
+import uvicorn
+from fastapi import (BackgroundTasks, Depends, FastAPI, HTTPException,
+ Security, status)
+from fastapi.openapi.utils import get_openapi
+from fastapi.security.api_key import APIKey, APIKeyHeader
+from src import buildno, major, minor, revision
from src.core.iem import Iem
-from src.core.persistence import Sqlite
-from src.core.utils import (
- BaseResponse,
- DeploymentResponse,
- DeploymentRequest,
- DeleteDeploymentRequest,
-)
-
-LOGGER = logging.getLogger("iem")
+from src.core.persistence import Persistence
+from src.core.utils import (BaseResponse, DeleteDeploymentRequest,
+ DeploymentRequest, DeploymentResponse,
+ SelfHealingRequest, DeploymentStatusRequest)
api_key_header = APIKeyHeader(name="x-api-key", auto_error=False)
app = FastAPI(
- title="IaC Execution Manager", version="0.1.15", description="IaC Execution Manager"
+ title="IaC Execution Manager",
+ version=f"{major}.{minor}.{revision}.{buildno}",
+ description="IaC Execution Manager",
)
+logging.config.fileConfig("logging.ini")
-async def get_api_key(api_key_query: str = Security(api_key_header)):
- if Sqlite().valid_api_key(api_key_query=api_key_query):
+async def get_api_key(api_key_query: str = Security(api_key_header)):
+ if Persistence().valid_api_key(api_key_query=api_key_query):
return api_key_query
else:
raise HTTPException(
@@ -37,22 +38,18 @@ async def get_api_key(api_key_query: str = Security(api_key_header)):
@app.get("/", tags=["greeting"])
-async def read_root(api_key: APIKey = Depends(get_api_key)):
+async def read_root(_: APIKey = Depends(get_api_key)):
return {
"message": "Hello from the IaC Execution Manager!",
"version": app.version,
"terraform": "1.1.4",
- "ansible": "5.5.0",
+ "ansible": "8.5.0",
}
@app.get("/deployments/", response_model=List[DeploymentResponse], tags=["deployments"])
async def read_status(
- start: int = 0,
- count: int = 25,
- start_date: str = "1970-01-01",
- end_date: str = "2100-01-01",
- api_key: APIKey = Depends(get_api_key),
+ _: APIKey = Depends(get_api_key),
):
all_deployments = Iem(credentials=None).get_all_deployments()
return list(all_deployments)
@@ -65,14 +62,29 @@ async def read_status(
)
async def read_status_deployment(
deployment_id: str,
- start: int = 0,
- count: int = 1,
- api_key: APIKey = Depends(get_api_key),
+ _: APIKey = Depends(get_api_key),
):
deployment = Iem().get_deployment(deployment_id=deployment_id)
return list(deployment)
+@app.get(
+ "/deployments/{deployment_id}/stages/{stage_id}/outputs",
+ response_model=dict,
+ tags=["deployments"],
+)
+async def read_deployment_outputs(
+ deployment_id: str,
+ stage_id: str,
+ d: DeploymentStatusRequest,
+ _: APIKey = Depends(get_api_key),
+):
+ outputs = Iem(credentials=d.credentials).get_deployment_outputs(
+ deployment_id=deployment_id, stage_id=stage_id
+ )
+ return outputs
+
+
@app.post(
"/deployments/",
status_code=status.HTTP_201_CREATED,
@@ -82,11 +94,10 @@ async def read_status_deployment(
async def deploy(
d: DeploymentRequest,
background_tasks: BackgroundTasks,
- api_key: APIKey = Depends(get_api_key),
+ _: APIKey = Depends(get_api_key),
):
- logging.warning(d)
i = Iem(credentials=d.credentials)
- background_tasks.add_task(i.deploy, d.deployment_id, d.repository, d.commit)
+ background_tasks.add_task(i.deploy, d.deployment_id, d.bundle.base64)
return BaseResponse(message="Deployment Request Created")
@@ -99,15 +110,48 @@ async def deploy(
async def undeploy(
d: DeleteDeploymentRequest,
background_tasks: BackgroundTasks,
- api_key: APIKey = Depends(get_api_key),
+ _: APIKey = Depends(get_api_key),
):
- logging.warning(d)
i = Iem(credentials=d.credentials)
background_tasks.add_task(i.destroy, d.deployment_id)
return BaseResponse(message="Undeployment Request Created")
-if os.getenv("STAGE") == "dev":
+@app.post(
+ "/deployments/{deployment_id}/self-healing",
+ status_code=status.HTTP_201_CREATED,
+ response_model=BaseResponse,
+ tags=["deployments"],
+)
+async def self_healing_strategy(
+ deployment_id: str,
+ d: SelfHealingRequest,
+ background_tasks: BackgroundTasks,
+ _: APIKey = Depends(get_api_key),
+):
+ i = Iem(credentials=d.credentials)
+ background_tasks.add_task(i.self_healing_strategy, deployment_id, d.playbook)
+ return BaseResponse(message=f"Self-Healing Strategy Request Triggered")
+
+
+@app.post(
+ "/update-iac-bundle/",
+ status_code=status.HTTP_201_CREATED,
+ response_model=BaseResponse,
+ tags=["deployments"],
+)
+async def self_healing_bundle(
+ d: DeploymentRequest,
+ background_tasks: BackgroundTasks,
+ _: APIKey = Depends(get_api_key),
+):
+ i = Iem(credentials=d.credentials)
+ background_tasks.add_task(i.self_healing_bundle, d.deployment_id, d.bundle.base64)
+ return BaseResponse(message="Bundle Replacement Created")
+
+
+if __name__ == "__main__":
+ uvicorn.run("main:app", host="127.0.0.1", port=8000, log_level="info")
with open("../openapi.json", "w") as f:
json.dump(
get_openapi(
diff --git a/iem-api/requirements.txt b/iem-api/requirements.txt
index 00500ee..19fad26 100644
--- a/iem-api/requirements.txt
+++ b/iem-api/requirements.txt
@@ -1,7 +1,6 @@
fastapi==0.73.0
uvicorn==0.17.0.post1
-ansible==5.5.0
-ansible-core==2.12.3
+ansible==8.5.0
GitPython==3.1.26
requests==2.26.0
ratelimiter==1.2.0.post0
diff --git a/iem-api/src/__init__.py b/iem-api/src/__init__.py
new file mode 100644
index 0000000..aa9f25d
--- /dev/null
+++ b/iem-api/src/__init__.py
@@ -0,0 +1 @@
+from ._version import buildno, major, minor, revision
diff --git a/iem-api/src/_version.py b/iem-api/src/_version.py
new file mode 100644
index 0000000..9d0e591
--- /dev/null
+++ b/iem-api/src/_version.py
@@ -0,0 +1,4 @@
+major = 3
+minor = 0
+revision = 1
+buildno = 18
diff --git a/iem-api/src/core/engine.py b/iem-api/src/core/engine.py
index 08f25cb..43cb59e 100644
--- a/iem-api/src/core/engine.py
+++ b/iem-api/src/core/engine.py
@@ -1,12 +1,12 @@
import logging
import os
import subprocess
-
+import time
from abc import ABC, abstractmethod
-from jinja2 import Template
-from subprocess import CalledProcessError
-LOGGER = logging.getLogger("iem")
+from jinja2 import Environment, FileSystemLoader
+
+LOGGER = logging.getLogger(__name__)
class Factory:
@@ -26,6 +26,18 @@ class Engine(ABC):
self._repo_path = repo_path
self._env = env
+ def _run_command(self, args: list) -> subprocess.CompletedProcess:
+ output = subprocess.run(
+ args=args, cwd=self._repo_path, env=self._env, capture_output=True
+ )
+ if output.returncode == 0:
+ LOGGER.info(output.stdout.decode("utf-8"))
+ LOGGER.info(output.stderr.decode("utf-8"))
+ else:
+ LOGGER.error(output.stdout.decode("utf-8"))
+ LOGGER.error(output.stderr.decode("utf-8"))
+ return output
+
@abstractmethod
def apply(self):
pass
@@ -43,107 +55,80 @@ class Engine(ABC):
class Terraform(Engine):
- def __init__(self, repo_path, my_env):
+ def __init__(self, repo_path, my_env, skip_inventory=False):
super().__init__(name="Terraform", repo_path=repo_path, env=my_env)
def apply(self):
LOGGER.info("About to apply terraform")
- try:
- output = subprocess.run(
- ["terraform", "init"],
- check=True,
- cwd=self._repo_path,
- env=self._env,
- capture_output=True,
- )
- output = subprocess.run(
- ["terraform", "apply", "-auto-approve"],
- check=True,
- cwd=self._repo_path,
- env=self._env,
- # capture_output=True,
- )
- return "CREATED", output.stdout, output.stderr
- except CalledProcessError as e:
- LOGGER.exception(e)
- return "ERROR", None, None
+
+ args = ["terraform", "init"]
+ output = self._run_command(args=args)
+ if output.returncode != 0:
+ return output.returncode, output.stdout, output.stderr
+
+ args = ["terraform", "apply", "-auto-approve"]
+ output = self._run_command(args=args)
+ return output.returncode, output.stdout, output.stderr
def destroy(self):
- try:
- output = subprocess.run(
- ["terraform", "destroy", "-auto-approve"],
- check=True,
- cwd=self._repo_path,
- env=self._env,
- # capture_output=True,
- )
- return "DESTROYED", output.stdout, output.stderr
- except CalledProcessError as e:
- LOGGER.exception(e)
- return "ERROR", None, None
+ args = ["terraform", "destroy", "-auto-approve"]
+ output = self._run_command(args=args)
+ return output.returncode, output.stdout, output.stderr
def output(self):
- try:
- output = subprocess.run(
- ["terraform", "output", "-json"],
- check=True,
- cwd=self._repo_path,
- env=self._env,
- capture_output=True,
- )
- return output.stdout
- except CalledProcessError as e:
- LOGGER.exception(e)
- return None
+ args = ["terraform", "output", "-json"]
+ output = self._run_command(args=args)
+ output.check_returncode()
+ return output.stdout
class Ansible(Engine):
- def __init__(self, repo_path, my_env):
+ def __init__(self, repo_path, my_env, skip_inventory=False):
super().__init__(name="Ansible", repo_path=repo_path, env=my_env)
- self.__parse_inventory()
+ self.__parse_inventory() if not skip_inventory else None
def __parse_inventory(self):
- with open(f"{self._repo_path}/inventory.j2", "r") as f:
- inventory = Template(f.read())
+ environment = Environment(loader=FileSystemLoader({self._repo_path}))
with open(f"{self._repo_path}/inventory", "w") as f:
- f.write(inventory.render(self._env))
-
- with open(f"{self._repo_path}/ssh_key.j2", "r") as f:
- ssh_key = Template(f.read())
+ template = environment.get_template("inventory.j2")
+ f.write(template.render(self._env))
with open(f"{self._repo_path}/ssh_key", "w") as f:
- f.write(ssh_key.render(self._env))
+ template = environment.get_template("ssh_key.j2")
+ f.write(template.render(self._env))
os.chmod(f"{self._repo_path}/ssh_key", 0o0600)
def apply(self):
LOGGER.info("About to apply ansible")
- try:
- output = subprocess.run(
- ["ansible", "all", "-i", "inventory", "-m", "wait_for_connection"],
- check=True,
- cwd=self._repo_path,
- env=self._env,
- capture_output=True,
- )
+ for _ in range(2):
+ args = [
+ "ansible",
+ "all",
+ "-i",
+ "inventory",
+ "-m",
+ "wait_for_connection",
+ ]
+ output = self._run_command(args=args)
+ if output.returncode != 0:
+ time.sleep(10)
+ continue
LOGGER.info("All hosts in the inventory are reachable.")
- output = subprocess.run(
- ["ansible-playbook", "-i", "inventory", "main.yml"],
- check=True,
- cwd=self._repo_path,
- env=self._env,
- capture_output=True,
- )
- return "CREATED", output.stdout, output.stderr
- except CalledProcessError as e:
- LOGGER.exception(e.output)
- raise e
+
+ args = ["ansible-playbook", "-i", "inventory", "main.yml"]
+ output = self._run_command(args=args)
+ if output.returncode != 0:
+ time.sleep(10)
+ continue
+
+ return output.returncode, output.stdout, output.stderr
def destroy(self):
LOGGER.info("Nothing to be seen here.")
- return "DESTROYED", None, None
+ return 0, None, None
def output(self):
LOGGER.info("Nothing to be seen here.")
diff --git a/iem-api/src/core/iem.py b/iem-api/src/core/iem.py
index e4ad215..8ba06ec 100644
--- a/iem-api/src/core/iem.py
+++ b/iem-api/src/core/iem.py
@@ -1,54 +1,38 @@
-import git
+import base64
+import binascii
import json
import logging
import os
-import subprocess
-
-from git import GitCommandError, InvalidGitRepositoryError
-from omegaconf import OmegaConf
+import shutil
+from io import BytesIO
from subprocess import CalledProcessError
+from zipfile import BadZipFile, ZipFile
+from omegaconf import OmegaConf
from src.core.engine import Factory
-from src.core.persistence import Sqlite
-from src.core.utils import DeploymentResponse, Credentials
+from src.core.persistence import Persistence
+from src.core.utils import Credentials, DeploymentResponse
-LOGGER = logging.getLogger("iem")
+LOGGER = logging.getLogger(__name__)
class Iem:
def __init__(self, credentials: Credentials = None):
- logging.basicConfig(
- level=logging.INFO, format="%(asctime)s %(levelname)-8s %(message)s"
- )
-
self._credentials = credentials
- # if credentials.aws:
- # self._aws_access_key_id = credentials.aws.access_key_id
- # self._aws_secret_access_key = credentials.aws.secret_access_key
-
- # if credentials.openstack:
- # self._os_username = credentials.openstack.user_name
- # self._os_password = credentials.openstack.password
- # self._os_auth_url = credentials.openstack.auth_url
- # self._os_project_name = credentials.openstack.project_name
- # Check IEM_HOME variable
- if os.getenv("IEM_HOME") is None:
- LOGGER.error("Please define IEM_HOME environment variable.")
- exit(-1)
- self._iem_home = os.getenv("IEM_HOME")
+ self._path_deployments = f"{os.environ['IEM_HOME']}deployments/"
- self._persistence = Sqlite()
+ self._persistence = Persistence()
def get_all_deployments(self):
rows = self._persistence.get_all_deployments()
for r in rows:
d = DeploymentResponse(
- status_time=r[0],
- deployment_id=r[1],
- status=r[2],
- stdout=r[3],
- stderr=r[4],
+ status_time=r.status_time,
+ deployment_id=r.deployment_id,
+ status=r.status,
+ stdout=r.stdout,
+ stderr=r.stderr,
)
yield d
@@ -56,67 +40,139 @@ class Iem:
row = self._persistence.get_deployment(deployment_id=deployment_id)
if row:
d = DeploymentResponse(
- status_time=row[0],
- deployment_id=row[1],
- status=row[2],
- stdout=row[3],
- stderr=row[4],
+ status_time=row.status_time,
+ deployment_id=row.deployment_id,
+ status=row.status,
+ stdout=row.stdout,
+ stderr=row.stderr,
)
yield d
- def deploy(self, deployment_id: str, repository: str, commit: str):
+ def get_deployment_outputs(self, deployment_id: str, stage_id: str):
+ function_name = "get_deployment_outputs"
+ LOGGER.info(f"Running {function_name} method.")
+
+ repo_path = f"{self._path_deployments}{deployment_id}"
+
+ LOGGER.info(f"Reading credentials.")
+ my_env = self._get_env(
+ deployment_id=deployment_id, credentials=self._credentials
+ )
+
+ LOGGER.info(f"About to read outputs for project {repo_path}.")
+
+ conf = OmegaConf.load(f"{repo_path}/{stage_id}/config.yaml")
+ self.validate(env=my_env, io=conf.input)
+
+ my_eng = Factory().get_engine(conf.engine)(
+ repo_path=f"{repo_path}/{stage_id}",
+ my_env=my_env,
+ )
+
+ output = my_eng.output()
+
+ return dict(json.loads(output))
+
+ def _get_env(self, deployment_id: str, credentials: Credentials) -> dict:
+ my_env = os.environ.copy()
+ if credentials.aws:
+ my_env["AWS_ACCESS_KEY_ID"] = credentials.aws.access_key_id
+ my_env["AWS_SECRET_ACCESS_KEY"] = credentials.aws.secret_access_key
+ my_env["AWS_REGION"] = credentials.aws.region
+ if credentials.openstack:
+ my_env["OS_USERNAME"] = credentials.openstack.user_name
+ my_env["OS_PASSWORD"] = credentials.openstack.password
+ my_env["OS_AUTH_URL"] = credentials.openstack.auth_url
+ my_env["OS_PROJECT_NAME"] = credentials.openstack.project_name
+ if credentials.azure:
+ my_env["ARM_CLIENT_ID"] = credentials.azure.arm_client_id
+ my_env["ARM_CLIENT_SECRET"] = credentials.azure.arm_client_secret
+ my_env["ARM_SUBSCRIPTION_ID"] = credentials.azure.arm_subscription_id
+ my_env["ARM_TENANT_ID"] = credentials.azure.arm_tenant_id
+ if credentials.vmware:
+ my_env["VSPHERE_USER"] = credentials.vmware.user_name
+ my_env["VSPHERE_PASSWORD"] = credentials.vmware.password
+ my_env["VSPHERE_SERVER"] = credentials.vmware.server
+ my_env[
+ "VSPHERE_ALLOW_UNVERIFIED_SSL"
+ ] = credentials.vmware.allow_unverified_ssl
+ if credentials.custom:
+ for key, value in credentials.custom.items():
+ my_env[key] = value
+
+ my_env["DEPLOYMENT_ID"] = deployment_id
+
+ return my_env
+
+ def _apply_stage(
+ self, my_env: dict, repo_path: str, stage: str, skip_inventory=False
+ ):
+ conf = OmegaConf.load(f"{repo_path}/{stage}/config.yaml")
+
+ self.validate(env=my_env, io=conf.input) if not skip_inventory else None
+
+ LOGGER.info(f"About to run stage: {stage}")
+ my_eng = Factory().get_engine(conf.engine)(
+ repo_path=f"{repo_path}/{stage}",
+ my_env=my_env,
+ skip_inventory=skip_inventory,
+ )
+ returncode, stdout, stderr = my_eng.apply()
+
+ if returncode == 0 and conf.output:
+ output = my_eng.output()
+ self.update_env(output=output, env=my_env, io=conf.output)
+ self.validate(env=my_env, io=conf.output)
+
+ return returncode, stdout, stderr
+
+ def deploy(self, deployment_id: str, bundle: str) -> int:
function_name = "deploy"
- LOGGER.info(f"Running {function_name} method")
+ LOGGER.info(f"Running {function_name} method.")
self._persistence.insert_deployment(
deployment_id=deployment_id, status="STARTED", stdout=None, stderr=None
)
+ repo_path = f"{self._path_deployments}{deployment_id}"
+ if not os.path.exists(repo_path):
+ os.makedirs(repo_path)
+
try:
- repo_path = f"{self._iem_home}{deployment_id}"
- self.get_repo(repo_path=repo_path, repository=repository, commit=commit)
-
- LOGGER.info(f"About to deploy project {repo_path}")
- my_env = os.environ.copy()
- if self._credentials.aws:
- my_env["AWS_ACCESS_KEY_ID"] = self._credentials.aws.access_key_id
- my_env[
- "AWS_SECRET_ACCESS_KEY"
- ] = self._credentials.aws.secret_access_key
- if self._credentials.openstack:
- my_env["OS_USERNAME"] = self._credentials.openstack.user_name
- my_env["OS_PASSWORD"] = self._credentials.openstack.password
- my_env["OS_AUTH_URL"] = self._credentials.openstack.auth_url
- my_env["OS_PROJECT_NAME"] = self._credentials.openstack.project_name
+ LOGGER.info(f"Decompressing base64 bundle.")
+ bundle_decode = base64.b64decode(bundle)
+ ZipFile(BytesIO(bundle_decode)).extractall(repo_path)
+
+ LOGGER.info(f"Reading credentials.")
+ my_env = self._get_env(
+ deployment_id=deployment_id, credentials=self._credentials
+ )
+ LOGGER.info(f"About to deploy project {repo_path}.")
project_conf = OmegaConf.load(f"{repo_path}/config.yaml")
for stage in project_conf.iac:
- conf = OmegaConf.load(f"{repo_path}/{stage}/config.yaml")
-
- self.validate(env=my_env, io=conf.input)
-
- my_eng = Factory().get_engine(conf.engine)(
- repo_path=f"{repo_path}/{stage}", my_env=my_env
+ returncode, stdout, stderr = self._apply_stage(
+ my_env=my_env, repo_path=repo_path, stage=stage
)
- status, stdout, stderr = my_eng.apply()
-
- if conf.output:
- output = my_eng.output()
- self.update_env(output=output, env=my_env, io=conf.output)
- self.validate(env=my_env, io=conf.output)
+ if returncode != 0:
+ LOGGER.error(f"Deployment failed for project {repo_path}.")
+ break
self._persistence.insert_deployment(
deployment_id=deployment_id,
- status=status,
+ status="CREATED" if returncode == 0 else "FAILED",
stdout=stdout,
stderr=stderr,
)
+ LOGGER.info(f"Deployment completed for project {repo_path}")
+ return returncode
except (
+ binascii.Error,
+ BadZipFile,
CalledProcessError,
- GitCommandError,
FileNotFoundError,
- InvalidGitRepositoryError,
NameError,
+ KeyError,
) as e:
LOGGER.exception(e)
self._persistence.insert_deployment(
@@ -124,23 +180,15 @@ class Iem:
)
raise e
- LOGGER.info(f"The {function_name} method finished successfully")
-
def destroy(self, deployment_id: str):
function_name = "destroy"
LOGGER.info(f"Running {function_name} method")
- repo_path = f"{self._iem_home}{deployment_id}"
+ repo_path = f"{self._path_deployments}{deployment_id}"
- my_env = os.environ.copy()
- if self._credentials.aws:
- my_env["AWS_ACCESS_KEY_ID"] = self._credentials.aws.access_key_id
- my_env["AWS_SECRET_ACCESS_KEY"] = self._credentials.aws.secret_access_key
- if self._credentials.openstack:
- my_env["OS_USERNAME"] = self._credentials.openstack.user_name
- my_env["OS_PASSWORD"] = self._credentials.openstack.password
- my_env["OS_AUTH_URL"] = self._credentials.openstack.auth_url
- my_env["OS_PROJECT_NAME"] = self._credentials.openstack.project_name
+ my_env = self._get_env(
+ deployment_id=deployment_id, credentials=self._credentials
+ )
LOGGER.info(f"About to destroy project {repo_path}")
project_conf = OmegaConf.load(f"{repo_path}/config.yaml")
@@ -151,11 +199,19 @@ class Iem:
my_eng = Factory().get_engine(conf.engine)(
repo_path=f"{repo_path}/{stage}", my_env=my_env
)
- status, stdout, stderr = my_eng.destroy()
+ returncode, stdout, stderr = my_eng.destroy()
+ if returncode != 0:
+ LOGGER.error(f"Undeployment failed for project {repo_path}.")
+ break
+
+ if "/home" in repo_path:
+ LOGGER.exception(f"Cannot delete {repo_path}")
+ elif os.getenv("DOCKERIZED") == "true":
+ shutil.rmtree(repo_path)
self._persistence.insert_deployment(
deployment_id=deployment_id,
- status=status,
+ status="DESTROYED" if returncode == 0 else "FAILED",
stdout=stdout,
stderr=stderr,
)
@@ -174,20 +230,114 @@ class Iem:
LOGGER.error(f"Variable {variable} not found in environment.")
raise NameError
- def get_repo(self, repo_path: str, repository: str, commit: str):
- LOGGER.info(
- f"About to download the project {repo_path} with repository {repository}"
+ def self_healing_strategy(self, deployment_id: str, playbook: str):
+ function_name = "self_healing_strategy"
+ LOGGER.info(f"Running {function_name} method")
+
+ self._persistence.insert_deployment(
+ deployment_id=deployment_id, status="SHS-STARTED", stdout=None, stderr=None
)
- if not os.path.isdir(repo_path):
- os.makedirs(repo_path)
- repo = git.Repo.clone_from(
- url=repository, to_path=repo_path, no_checkout=True
+
+ LOGGER.info(f"The {function_name} method finished successfully")
+ repo_path = f"{self._path_deployments}{deployment_id}"
+ if not os.path.exists(repo_path):
+ LOGGER.error(
+ f"The deployment_id = {deployment_id} does not correspond to any active deployment."
)
- else:
- repo = git.Repo(repo_path)
+ return
- repo.git.checkout(commit)
+ try:
+ LOGGER.info(f"Updating main.yml with new playbook.")
+ with open(f"{repo_path}/self_healing_monitoring/main.yml", "w") as f:
+ f.write(playbook)
- for submodule in repo.submodules:
- submodule.update(init=True)
- submodule.module().git.checkout()
+ LOGGER.info(f"Reading credentials.")
+ my_env = self._get_env(
+ deployment_id=deployment_id, credentials=self._credentials
+ )
+
+ returncode, stdout, stderr = self._apply_stage(
+ my_env=my_env,
+ repo_path=repo_path,
+ stage=f"self_healing_monitoring",
+ skip_inventory=True,
+ )
+ if returncode != 0:
+ LOGGER.error(f"Update failed for project {repo_path}.")
+
+ self._persistence.insert_deployment(
+ deployment_id=deployment_id,
+ status="UPDATED" if returncode == 0 else "FAILED",
+ stdout=stdout,
+ stderr=stderr,
+ )
+ LOGGER.info(f"Update completed for project {repo_path}")
+ except (
+ binascii.Error,
+ CalledProcessError,
+ FileNotFoundError,
+ NameError,
+ KeyError,
+ ) as e:
+ LOGGER.exception(e)
+ self._persistence.insert_deployment(
+ deployment_id=deployment_id, status="ERROR", stdout=None, stderr=None
+ )
+ raise e
+
+ def self_healing_bundle(self, deployment_id: str, bundle: str) -> int:
+ function_name = "self_healing_bundle"
+ LOGGER.info(f"Running {function_name} method")
+
+ self._persistence.insert_deployment(
+ deployment_id=deployment_id, status="UPDATING", stdout=None, stderr=None
+ )
+
+ repo_path = f"{self._path_deployments}{deployment_id}"
+ if not os.path.exists(repo_path):
+ LOGGER.error(
+ f"The deployment_id = {deployment_id} does not correspond to any active deployment."
+ )
+ return -1
+
+ try:
+ LOGGER.info(f"Decompressing base64 bundle.")
+ bundle_decode = base64.b64decode(bundle)
+ ZipFile(BytesIO(bundle_decode)).extractall(repo_path)
+
+ LOGGER.info(f"Reading credentials.")
+ my_env = self._get_env(
+ deployment_id=deployment_id, credentials=self._credentials
+ )
+
+ LOGGER.info(f"About to update project {repo_path}.")
+ project_conf = OmegaConf.load(f"{repo_path}/config.yaml")
+ for stage in project_conf.iac:
+ returncode, stdout, stderr = self._apply_stage(
+ my_env=my_env, repo_path=repo_path, stage=stage, skip_inventory=True
+ )
+ if returncode != 0:
+ LOGGER.error(f"Update failed for project {repo_path}.")
+ break
+
+ self._persistence.insert_deployment(
+ deployment_id=deployment_id,
+ status="UPDATED" if returncode == 0 else "FAILED",
+ stdout=stdout,
+ stderr=stderr,
+ )
+ LOGGER.info(f"Update completed for project {repo_path}")
+ return returncode
+ except (
+ binascii.Error,
+ BadZipFile,
+ CalledProcessError,
+ FileNotFoundError,
+ NameError,
+ KeyError,
+ ) as e:
+ LOGGER.exception(e)
+ self._persistence.insert_deployment(
+ deployment_id=deployment_id, status="ERROR", stdout=None, stderr=None
+ )
+ raise e
diff --git a/iem-api/src/core/persistence.py b/iem-api/src/core/persistence.py
index 2f4979e..10c3a2d 100644
--- a/iem-api/src/core/persistence.py
+++ b/iem-api/src/core/persistence.py
@@ -1,91 +1,21 @@
+import logging
import os
-import sqlite3
-# from dataclasses import dataclass
-from datetime import datetime
from ratelimiter import RateLimiter
+from sqlalchemy import (Column, DateTime, Integer, String, create_engine, desc,
+ func, select)
+from sqlalchemy.orm import Session, declarative_base
-from sqlalchemy import create_engine
-from sqlalchemy import Column
-from sqlalchemy import DateTime, Integer, String
-from sqlalchemy import select, desc
-from sqlalchemy.orm import declarative_base, Session
+LOGGER = logging.getLogger(__name__)
Base = declarative_base()
-class Sqlite:
- def __init__(self):
- self._db_name = "iem.db"
-
- # create database if it does not exist
- if not os.path.isfile(self._db_name):
- self.__create_database()
-
- def __create_database(self):
- conn = sqlite3.connect(self._db_name)
- with conn:
- sql = """CREATE TABLE deployments (id INTEGER PRIMARY KEY AUTOINCREMENT,
- status_time DATETIME DEFAULT CURRENT_TIMESTAMP,
- deployment_id TEXT NOT NULL, status TEXT NOT NULL, stdout TEXT, stderr TEXT);"""
- conn.execute(sql)
-
- def insert_deployment(
- self, deployment_id: str, status: str, stdout: str, stderr: str
- ):
- conn = sqlite3.connect(self._db_name)
- with conn:
- sql = """INSERT INTO deployments (deployment_id, status, stdout, stderr) VALUES (?,?,?,?)"""
-
- cursor = conn.cursor()
- cursor.execute(
- sql,
- (
- deployment_id,
- status,
- stdout,
- stderr,
- ),
- )
- conn.commit()
-
- def get_deployment(self, deployment_id: str):
- conn = sqlite3.connect(self._db_name)
- with conn:
- sql = """SELECT status_time, deployment_id, status, stdout, stderr FROM deployments
- WHERE deployment_id=? ORDER BY id DESC LIMIT 1"""
-
- cursor = conn.cursor()
- cursor.execute(sql, (deployment_id,))
- row = cursor.fetchone()
-
- return row
-
- def get_all_deployments(self):
- conn = sqlite3.connect(self._db_name)
- with conn:
- sql = """SELECT status_time, deployment_id, status, stdout, stderr
- FROM deployments ORDER BY id DESC LIMIT 25 OFFSET 0"""
-
- cursor = conn.cursor()
- cursor.execute(sql)
- rows = cursor.fetchall()
-
- return rows
-
- @RateLimiter(max_calls=10, period=1)
- def valid_api_key(self, api_key_query: str):
- if api_key_query == os.getenv("API_KEY"):
- return True
- else:
- return False
-
-
class Deployment(Base):
__tablename__ = "deployments"
id = Column(Integer, primary_key=True)
- status_time = Column(DateTime, default=datetime.now())
+ status_time = Column(DateTime, default=func.now())
deployment_id = Column(String, nullable=False)
status = Column(String, nullable=False)
stdout = Column(String)
@@ -96,8 +26,8 @@ class Deployment(Base):
class Persistence:
- def __init__(self):
- self._engine = create_engine("sqlite:///:memory:", future=True)
+ def __init__(self, engine_url="sqlite:///db/iem.db"):
+ self._engine = create_engine(url=engine_url, future=True)
Base.metadata.create_all(self._engine)
def insert_deployment(
diff --git a/iem-api/src/core/utils.py b/iem-api/src/core/utils.py
index 5ec5df2..4c4bcf1 100644
--- a/iem-api/src/core/utils.py
+++ b/iem-api/src/core/utils.py
@@ -1,10 +1,7 @@
-import logging
-
from datetime import datetime
-from pydantic import BaseModel
-from typing import Dict, Optional
+from typing import Any, Dict, Optional
-LOGGER = logging.getLogger("iem")
+from pydantic import BaseModel
class BaseResponse(BaseModel):
@@ -22,6 +19,7 @@ class DeploymentResponse(BaseModel):
class Aws(BaseModel):
access_key_id: str
secret_access_key: str
+ region: Optional[str] = "us-west-2"
class Azure(BaseModel):
@@ -42,19 +40,46 @@ class Openstack(BaseModel):
user_domain_name: Optional[str]
+class Docker(BaseModel):
+ server: str
+ user_name: str
+ password: str
+
+
+class Vmware(BaseModel):
+ user_name: str
+ password: str
+ server: str
+ allow_unverified_ssl: Optional[str]
+
+
class Credentials(BaseModel):
aws: Optional[Aws] = None
azure: Optional[Azure] = None
openstack: Optional[Openstack] = None
+ vmware: Optional[Vmware] = None
+ docker: Optional[Docker] = None
+ custom: Optional[dict[str, Any]] = None
+
+
+class Bundle(BaseModel):
+ base64: str
class DeploymentRequest(BaseModel):
deployment_id: str
- repository: str
- commit: str
credentials: Credentials
+ bundle: Bundle
+
+
+class SelfHealingRequest(BaseModel):
+ credentials: Credentials
+ playbook: str
class DeleteDeploymentRequest(BaseModel):
deployment_id: str
credentials: Credentials
+
+class DeploymentStatusRequest(BaseModel):
+ credentials: Credentials
diff --git a/iem-api/src/resources/id_iem b/iem-api/src/resources/id_iem
new file mode 100644
index 0000000..2f5c307
--- /dev/null
+++ b/iem-api/src/resources/id_iem
@@ -0,0 +1,38 @@
+-----BEGIN OPENSSH PRIVATE KEY-----
+b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAABlwAAAAdzc2gtcn
+NhAAAAAwEAAQAAAYEA1d1XmDw9VikKatj87TVENCAJ5uCQpgjyaQi3HpvZDtw7zW1QTeUN
+CRdJ36oHPG9cD+7gR9W9xvI2l+2bKugqTbuDr1ysPxAcTt6XYgMer9JkWTCwcdqnAXdUwf
+dLMUPo7mOv/9OnOdjoEo0NwnsvGvPxinbUzMUSofqwLgHIdlo3GQZk2T3vvRN3zoYmx4OV
+AN7C4RXAD7hLhEEW0wwUrAqnQqh13yHuDYi6iWFSzh5B7h3700Wi0Toe85n9tBZiD+7NYG
+FNxqQykYGXrqNmTPe/dTffAdCYEp3qW8dvzA7htlrwBcByCu1kjVcRrsXb17KVFnwxmpvx
+uGRrsc2sVR7cGIEssTaAFF3L/NhkvzDS16qX2qbz0fyxnFgBHINRYM45AvMuDAWerUA6MA
+GK/wRJ1CO8ZrSJtIsQVnXdsxrcjKEm76hzdA0rETPmA5A+FzHH3fSqfZIiFh5dTfUUnldw
+9STXwxjRgBNasptuTYtM1jW0shFMaS5F4+E9sBAzAAAFkHZrcNl2a3DZAAAAB3NzaC1yc2
+EAAAGBANXdV5g8PVYpCmrY/O01RDQgCebgkKYI8mkItx6b2Q7cO81tUE3lDQkXSd+qBzxv
+XA/u4EfVvcbyNpftmyroKk27g69crD8QHE7el2IDHq/SZFkwsHHapwF3VMH3SzFD6O5jr/
+/TpznY6BKNDcJ7Lxrz8Yp21MzFEqH6sC4ByHZaNxkGZNk9770Td86GJseDlQDewuEVwA+4
+S4RBFtMMFKwKp0Kodd8h7g2IuolhUs4eQe4d+9NFotE6HvOZ/bQWYg/uzWBhTcakMpGBl6
+6jZkz3v3U33wHQmBKd6lvHb8wO4bZa8AXAcgrtZI1XEa7F29eylRZ8MZqb8bhka7HNrFUe
+3BiBLLE2gBRdy/zYZL8w0teql9qm89H8sZxYARyDUWDOOQLzLgwFnq1AOjABiv8ESdQjvG
+a0ibSLEFZ13bMa3IyhJu+oc3QNKxEz5gOQPhcxx930qn2SIhYeXU31FJ5XcPUk18MY0YAT
+WrKbbk2LTNY1tLIRTGkuRePhPbAQMwAAAAMBAAEAAAGAB5BynrHSwY9mDO1r1MADj4xqjT
+34H8dFO63RPEXq4Xmsq9Fn+7lUQrQOKtkKtHqD2RRr3l6S/cxnXexLhrL7fBBb0gIHHZvm
+RGvfEtplZXadkgIE26IOMiEUYF/syutJ+9SOzw+fZI5ldvKCQBS3T869Bla5pBx8UjpZrO
+bnPjhmpn3xZzWnmxprLGTWTkw7IvK+FdP9HRE5qo3aztAokwU1cUggEypSDyx83IsSsLOl
+RVTOKWTXI2tY2OjjblE0SiFimH7btHH2VhdAWEtOujs04cKU1mQ7pPnZS0uK5PfkC89I7v
+zeslnsMsZ73jtBMtCWnNwh3Om7saP4GK0G58pQqyC4SAN/4Clf6PGB+/dtQsJG1XDQ+Idc
+IjlSly43tX6Q3t+M7VtfGbLYy/vUF01r2PaG5G+BUucH11DQrGxw/gMQhpqHaXu5C7PpxG
+4yBxF58BaMqRPPTJWgTFuMw1Ib2Ogda5dRuC6Qi2hJ3T2S2IkFXDwGWabRGIpqKXMhAAAA
+wChxJsE8SV5uxbYtgPKGFcHvZLqsX7Hg3q9GXDjqCiFmJCl2GpFnqacxqfhTlTGShHBB6q
+O71zUOktX/Dbje7VTTBKhHLrpqsp7JXWao7v9w007LxhKAR3XEencoynmWTL9lGBvbQ8mX
+X5Q4zqMt+1GqH5ae4BuT4BtlqD+R9WO4ZvnsEKUuCFXbndZlWsMaRD7J/PQXM/vrsDg0yF
+9dHUH8INayw4utgQ/FwmdkFx6YAxnwqgcoYX0KzoX3FJhN+QAAAMEA+fiPdeB4tVLGLQyF
+k/4SDjE1pyba0XUwl0hoc8IWILaiwmncn1uRLCbRSYZp+gikFheFY7zIzbdnZXPv4vpX8a
+0ydCdksE9iwPxuhzMb5EbyoWCffUuDmiWaVu5H3Q9rveJwFPB5HJXmT4S0ZTS5JZHZi6nb
+h55CTKFAoCd9pUBEZUATWUBfml+WurF72C9VzxiSqyd9S8XVwE2Oq72sxyI3S4VUGKvJ+t
+wZZFU1BE7qoJWyV0RiV3eiez9Mum0rAAAAwQDbBdgafpny35V4Yc5YFr+k1FSv3Xw9fduH
++uqyskdHpB3anUe366ZsSpnQcLjbDFsW45K9tZxe8GacTwyiTfbdrT/m5zWzLV3cMBnVyv
+Pf1oYSjrV4H1BDShikc5LCWGje+FWxONgtB2oOcUCQo7pPq/YYD984DsurBY9l1VWJM+T4
+U1KTMp9x9b+xoINP0+9uXZHLmL7LsxOGhygXhUCmE4XBCmJYZdEEodwlUco8RBraapFYqN
+inyHJxviLktRkAAAAUdmFncmFudEB1YnVudHUtZm9jYWwBAgMEBQYH
+-----END OPENSSH PRIVATE KEY-----
diff --git a/iem-api/src/resources/id_iem.pub b/iem-api/src/resources/id_iem.pub
new file mode 100644
index 0000000..c4368a5
--- /dev/null
+++ b/iem-api/src/resources/id_iem.pub
@@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDV3VeYPD1WKQpq2PztNUQ0IAnm4JCmCPJpCLcem9kO3DvNbVBN5Q0JF0nfqgc8b1wP7uBH1b3G8jaX7Zsq6CpNu4OvXKw/EBxO3pdiAx6v0mRZMLBx2qcBd1TB90sxQ+juY6//06c52OgSjQ3Cey8a8/GKdtTMxRKh+rAuAch2WjcZBmTZPe+9E3fOhibHg5UA3sLhFcAPuEuEQRbTDBSsCqdCqHXfIe4NiLqJYVLOHkHuHfvTRaLROh7zmf20FmIP7s1gYU3GpDKRgZeuo2ZM9791N98B0JgSnepbx2/MDuG2WvAFwHIK7WSNVxGuxdvXspUWfDGam/G4ZGuxzaxVHtwYgSyxNoAUXcv82GS/MNLXqpfapvPR/LGcWAEcg1FgzjkC8y4MBZ6tQDowAYr/BEnUI7xmtIm0ixBWdd2zGtyMoSbvqHN0DSsRM+YDkD4XMcfd9Kp9kiIWHl1N9RSeV3D1JNfDGNGAE1qym25Ni0zWNbSyEUxpLkXj4T2wEDM=
diff --git a/iem-api/tests/__init__.py b/iem-api/tests/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/iem-api/tests/it/__init__.py b/iem-api/tests/it/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/iem-api/tests/it/test_it_iem.py b/iem-api/tests/it/test_it_iem.py
new file mode 100644
index 0000000..529b412
--- /dev/null
+++ b/iem-api/tests/it/test_it_iem.py
@@ -0,0 +1,149 @@
+import base64
+import logging
+import os
+import unittest
+import uuid
+
+from src.core.iem import Iem
+from src.core.utils import Aws, Credentials, Openstack
+
+LOGGER = logging.getLogger(__name__)
+
+
+class TestIem(unittest.TestCase):
+ def __init__(self, *args, **kwargs):
+ super(TestIem, self).__init__(*args, **kwargs)
+
+ # Check IEM_HOME variable
+ self._iem_home = os.environ["IEM_HOME"]
+
+ @unittest.skipUnless(os.getenv("AWS"), "Define AWS variable to execute")
+ def test_deploy_destroy_aws(self):
+
+ deployment_id = str(uuid.uuid4())
+ a = Iem(
+ Credentials(
+ aws=Aws(
+ access_key_id=os.environ["AWS_ACCESS_KEY_ID"],
+ secret_access_key=os.environ["AWS_SECRET_ACCESS_KEY"],
+ )
+ )
+ )
+ with open("tests/resources/aws.zip", "rb") as binary_file:
+ bundle = base64.b64encode(binary_file.read())
+
+ a.deploy(deployment_id=deployment_id, bundle=bundle)
+
+ a.destroy(deployment_id=deployment_id)
+
+ @unittest.skipUnless(os.getenv("OS"), "Define OS variable to execute")
+ def test_deploy_destroy_openstack(self):
+
+ deployment_id = str(uuid.uuid4())
+ a = Iem(
+ Credentials(
+ openstack=Openstack(
+ user_name=os.environ["OS_USERNAME"],
+ password=os.environ["OS_PASSWORD"],
+ auth_url=os.environ["OS_AUTH_URL"],
+ project_name=os.environ["OS_PROJECT_NAME"],
+ )
+ )
+ )
+ with open("tests/resources/openstack.zip", "rb") as binary_file:
+ bundle = base64.b64encode(binary_file.read())
+
+ output = a.deploy(deployment_id=deployment_id, bundle=bundle)
+
+ a.destroy(deployment_id=deployment_id)
+
+ self.assertEqual(output, 0)
+
+ @unittest.skipUnless(os.getenv("ANSIBLE"), "Define ANSIBLE variable to execute")
+ def test_deploy_destroy_ansible(self):
+
+ deployment_id = str(uuid.uuid4())
+ a = Iem(
+ Credentials(
+ openstack=Openstack(
+ user_name=os.environ["OS_USERNAME"],
+ password=os.environ["OS_PASSWORD"],
+ auth_url=os.environ["OS_AUTH_URL"],
+ project_name=os.environ["OS_PROJECT_NAME"],
+ )
+ )
+ )
+ with open("tests/resources/ansible.zip", "rb") as binary_file:
+ bundle = base64.b64encode(binary_file.read())
+
+ output = a.deploy(deployment_id=deployment_id, bundle=bundle)
+
+ a.destroy(deployment_id=deployment_id)
+
+ self.assertEqual(output, 0)
+
+ @unittest.skipUnless(
+ os.getenv("INTEGRATION"), "Define INTEGRATION variable to execute"
+ )
+ def test_deploy_destroy_docker(self):
+
+ deployment_id = str(uuid.uuid4())
+ a = Iem(
+ Credentials(
+ openstack=Openstack(
+ user_name=os.environ["OS_USERNAME"],
+ password=os.environ["OS_PASSWORD"],
+ auth_url=os.environ["OS_AUTH_URL"],
+ project_name=os.environ["OS_PROJECT_NAME"],
+ )
+ )
+ )
+ with open("tests/resources/docker.zip", "rb") as binary_file:
+ bundle = base64.b64encode(binary_file.read())
+
+ a.deploy(deployment_id=deployment_id, bundle=bundle)
+
+ a.destroy(deployment_id=deployment_id)
+
+ @unittest.skipUnless(os.getenv("SHS"), "Define SHS variable to execute")
+ def test_self_healing_strategy(self):
+ deployment_id = str(uuid.uuid4())
+ a = Iem(
+ Credentials(
+ custom={
+ "instance_usr": "vagrant",
+ "instance_pwd": "vagrant",
+ "instance_ip": "192.168.56.201",
+ }
+ )
+ )
+ with open("tests/resources/shs.zip", "rb") as binary_file:
+ bundle = base64.b64encode(binary_file.read())
+
+ a.deploy(deployment_id=deployment_id, bundle=bundle)
+
+ a.self_healing_strategy(deployment_id=deployment_id, strategy="25")
+
+ @unittest.skipUnless(os.getenv("SHS"), "Define SHS variable to execute")
+ def test_self_healing_bundle(self):
+ deployment_id = str(uuid.uuid4())
+ a = Iem(
+ Credentials(
+ aws=Aws(
+ access_key_id=os.environ["AWS_ACCESS_KEY_ID"],
+ secret_access_key=os.environ["AWS_SECRET_ACCESS_KEY"],
+ )
+ )
+ )
+ with open("tests/resources/shs.zip", "rb") as binary_file:
+ bundle = base64.b64encode(binary_file.read())
+
+ returncode = a.deploy(deployment_id=deployment_id, bundle=bundle)
+ self.assertEqual(returncode, 0)
+
+ with open("tests/resources/shs-bundle.zip", "rb") as binary_file:
+ bundle = base64.b64encode(binary_file.read())
+ returncode = a.self_healing_bundle(deployment_id=deployment_id, bundle=bundle)
+ self.assertEqual(returncode, 0)
+
+ a.destroy(deployment_id=deployment_id)
diff --git a/iem-api/tests/resources/ansible.zip b/iem-api/tests/resources/ansible.zip
new file mode 100644
index 0000000000000000000000000000000000000000..7d2d24ddd670766af89958f489b8cfceff66c99a
GIT binary patch
literal 3735
zcmWIWW@Zs#;9%fjSmGEH#sCG13=9n13<?a%`FUxX>3WrkxjCUBybSEEEB_|LaA^fM
z10%}|W(Ec@arx@ir>A^;wmrHOs^O`t<>hl){}k({&zFopYwh^*<;xC+0B?4VHILqM
zg3JbCh|M4i4<gwN;<GU*Fyy6Y=2hr}E#1BHZ}Og%f0IEpnxzmkK(0R+Y!V963&PVF
z7#M`$W@P4-rRJ677gg$I8G+5?uF^~fDJx(3HyO>m+-^s{W&<9U_jRoCFGM@1E2OlZ
z*b<i5{6=Mw(;wL#1+NOz(>tofS0|kdHaZr)k$21c-b2L>BG;zA{*=|%{BK1r?=xoB
z9etY%d>W3d;hg*X`N5~Jb{UC1`LyBn^Q&9hbpA}0_qRGNd$-vS6dFguH$-GHFff4J
zi`^%2%nS^CaG&HRX6ETtLZbqYFG>Rs1|3!q*!x`j+C3YWW7%tLU+;WeaV95Wwn0!r
z!tE`K|LvZz%<QCw_=C6A`QIG_r=&(sxv=xZ#wL%49*Wnj#j|c_RckRsJ?V4W>pWBP
zn(7M0m)}AT&P>{6c&tBSDYFBoSn<QN+9^#A_gv5Fy=a^vUbXI+@`1z!SA|l|cb$!i
z%S|z1TQ~X1)4;j}Z}WccuTC>pZtrPKzct72bp4j_9v!o`E2ei=JTXb1y8XZsvDyyC
znaz28f3!P~l!_gV-Yg=Tzuo1c<*v=vcdvQ1bFpj`Ic%VGnE%!L>m573x3T?vp*Gjv
z|5v~huCV#@CrDKP_!swg+9ch{zE{_ulzDCaRAs)c<MHavGsBD+*m8gGQQAFgI>%KJ
z4}om6EtQJNGJAF>T)7(+vtea`&eYe(R(?LTs`^fjzF~FNrTN>Zl=O2?y1ys;-GK~0
z+jTb|ysXpxv*+<9$4|3V-cI(MJ3s4FxN3(o+r{sH9Yk+wS=l~$Jf*0^@dzJt@BVE|
z#Oy&)YJO;Sb2%de11RFKN2vw_1A_oON{fp#;<HnUNt&zIg`GH~t><~h+pkyOPgl!Z
zw@Y8+l#aKKXYYBxwLSW*U6NW?t`#wY^O?`e(>|IE3=AO4keq>(&r%o|7zE+YK+or(
ztOmnqStRs~Ay<O|55oc5^(WYaUff#W(W2$pB;`F@?9X1SUH1xHShAK@Gr2fzeReY}
zQP<|&kL_xu9#4ZxQ^fB*efBS^W%Y0K2PVxomZ@=WHjU+L6kxIeh1)`2i@QM#3=AM=
zK^)A$z`(E*$-y8#7lQ&rNorA1Vp@JtE~Ic+xAJfDhLwMlK{T36A?APz2V`^3FfcHP
zqL_p3d5}>sjAmqMk0aM10}+?+macje9Jb$O)$Ld@amm4_>eJTOY{~SP+^{{orEKYG
z^UrB50z8N3GHtU-_WEbGbYftK>h_-EgIihCt&4+>=RMrK>D!{SWr<IJ$yyXgR4@L1
zcUk-UG_A_Niwx4`*;*unkFsoxnQwA{IX=KH@N=|RNbRwDG1nEzhnGxw!wU+oCD)~P
zw=gg;fINfnCUO~Q!~=<6L~wyikCHS(Mc~@7yZN`x1nRHH7fy)JzFM?*QME_oqSIab
zwzyuh;c-?p5Xlvu=+V1q4~z4!?eF{KjdHa=UOU^y`?8{Z|8KjrnWD!wY&FyUGgb2A
zafRL$tLGZ*y~3rZP+!g1y(mVs+lE;}(rR(}CO5q)On+YZt+;1Y*RC{=>B8%S1&m8l
zSZ2B({HgJ=sOD62Lesy$x>Js>o^NAyAlYsIah=%*tJhvDGL~R0(|%z5YDd|RcqRw2
zi3<0bEp9J)#j5V8aC?np%k>J?H4~H67o<%~{W85H%<G2z)}52HgymP?HENH!v$(j+
zG;Vv+T7}*2S}WDEdlSMA_(@rBV2e7oJ&2R1f+L9ONa=0fu#idjM8)OOu2181`k^r4
zmPe`m{a5#bT2fA%E&8NWQBiEIVST3l>KxP0PZve3dA8r+rp<%TMV806e@dSeWuCL#
zS2Xv1^feQg{lfgA$CrEzST6R+Qg(6U(F@$cVoMr*zI^0ty^xvAe`K=9CCN_Zl{zVN
zQ?4J+zMGLYQ82|^a<xln%GA3O8zzRGJjcHN@-9ms{sTPMKFN7Um6c4Y{!=dYH9xp!
zgH)2&ytC0s9Iva-s#+OkUaIKX`tGbZ-~6r{jkAwVJl1s8bk?=sO5d+4zdm_K^{Q)>
z#*dJdH*?tL+eA%j^RS%U)Vy!O6uECK*Sn+}Z&XZ?U-GkRhwClNuX`Wg%I0smyp3b+
z#7D08YD<F>9_-kpKG$>A<kPog7EW2;%~~6;7rta~*xqcFH|$c$Cq7pHTUK!Y4{OWu
z(0^|iocXq4UGzCtrINFm6XuuHA9`Q^*lcz5O06fZxz6J4YJL^77GJUpkzU-+XKGz<
zJY$7<%2%HyGt$Je>Kf<VU-2gS|IbFjPc<Psf?ESrQVmS)1O$HniLtN^4+@`W5F1|j
zMy=+A=yT7hEI$vuo%kqb%HoWNpN)3&ynifGI=A!o$;QO^$Ny(7G`7)kFt`7=uz&HF
z&h5XI-<L13o7rNR<+|vo`orVGFD7jl^pcovw=C<fQ-5}3t`NWN-cKbZ%I{WoZJY4g
z?!B|}m1BAQLgD+=ep@vrZ})teeeFxgMd$0Ci~0K9vocyI{`sP@cE`T^mu|h8e^=o5
z-p*sY&sGLyix)gdDSVaG$l;uPm+5ueZMULJ-dkUqvTa_ccfa=UkC)6b@%#4g{PpAO
z_E(3mKA)|x@pVnB=k{k0`#yeod$`y3VzIKdmTP%|tQxb0X+j&@^qpn~TozsLzi)N4
zzpLBm@_easl&mMG;B>iruhu-P>-N4^_AhS6xm25omWgljdi&2W+#Bh$fc25!tf&7)
zbGBBQHj1tMZ+)l8vQezVp3Tj?Wj`aR)OZ~*ebaSj1_n^sfW6f4VPs$sMkyTgOG^q$
zOY{hr8*_WTxtI)j+P>Sm&R`ZPc^~Pd6L_d()+O<dit^inb0RgGZvX#Sdn`kDqu30U
zCr$?q)BJjck5zqpRPaptnw+fK4(SHBeMWN}l>7J2zWDOef19?6Dv8#&O7qPdDrdZ(
zrP3Ssz1~*(;bIS^ZBNy|J(}9pk{us>?p2Dy?#1!<W#4&SOep_pU=Q}E%g3OP?F<YI
zAm0RdGct)VBU-q~El^Mk7gVi)@RmjpiCz`J+PWYbttx;S4AO?J0Sgib;lGYm3=Bj!
zXAzDC73RndL{NhkWF81_Y1{`lla%Hy%*~)W5oA7cYX#KI1ep!OTN*nUVQvOj8$>iZ
z(alC~f(S4ufQ$v>EsfF4Ftdqo{h*lc7!!uvUH~;?z(zALFl=dj1UDT$GGPrG5RE4?
zksAvjV?m7$5DmhA9VfFekkkZ0L@3By<a!&_2mzT1!dn_|qnL~CbXao)MB{KeEKZR#
zIcmrpVnud3I9>^7Ym|TwHVH+}TcDBx<a!X^()fcD*?6MMjR0>}Hjn{a3|tHgnHd-!
HaDjLLJSJi(
literal 0
HcmV?d00001
diff --git a/iem-api/tests/resources/aws.zip b/iem-api/tests/resources/aws.zip
new file mode 100644
index 0000000000000000000000000000000000000000..5813bd61d580f52708e4ed71dcb3a583fb107ddc
GIT binary patch
literal 1230
zcmWIWW@h1H00A%O@G!5g-~2KR3=AO5%^<^&oS&DLnXXrvn41$C!pXoaBl<mAUi5o%
zX$3a}Bg+eB1_m&ptE;QPm6@1q#igL2t5A|!RFs&OUzE$Gr>Dn-upq%XJPc$J40AEa
zFu>L7gUxpl{hkcMXy${=5Mf|o;9$u0j0}#R#2lK(z`)SOz`!7aVn%*xNkM5zv0h0U
z*tEG3Wr<aot~Kp7<T`91!1BBL=pyzR6Yhl;Ogv{V&{cAf-A!oM<Y1QnwrZW*Wy_Y8
ztiJm|Wp%5}P3`Tyw~u<4RtQgsNS>S(WaPGEr+rxGj`JLvreUs?hs2JR?0z(5$K&m-
zTaD|a1>;`%Tm2Ds{=2JB=&j2aMugk7yN-ngBwk;2fPsO*nvsD)7{%?$Ihm<>B`~Km
z`qig^Fj_#k2HfU4WWZDVKD=d5b>gF^uJ#AxB{rUoNsLavrK$Am?Xu#-4vKTMGm`{Y
zHEsD|#BgZ)LGIm~rWbtgQ|&WguvL;-F00VysMD+z5xizE@1^x2O~Uozi}_=p*{k2s
z>HgbaIs4DOV`n}#+%$cr74oxf^YKDg=g9`zt;;v9(Ww;|E}bT5uqos0uk>Zt4%eGF
zH6Fcn=~|v%p7+sr2dc{*176AmE=#xHwlwKxl{RDVrp|T$ozGwC>zXxNP3O{|V2jv~
ztcbw);v62vC(~--!@$50%fP@OiW(Sb=`LFAdvdxMM!LIv^=jyYQ)jd_bT<Y)2nx6m
zTEN5KS6)(9#>?0Dy{xzAdr4zA+iolNoox!9MNg+sYcRc5xb)f5^k++>G^Z>P6IEUI
zRQzet%a<<;85sh+8JXmmab*PwaH3^oU|`tN2x6jS3|2_SK#MD612L06#6SiH0fxVh
zi3|)V8G)678EYm0IS^MifS3ny;8GOxkTV4<12ZU7V09_PWXwd0Y;qzavP+SYEGq*u
vmgEWv0*Jww5sqwd9utbe$kETrzzm9h3^%i~fg*yHft6uBBLf2`Gl&NOz*ui;
literal 0
HcmV?d00001
diff --git a/iem-api/tests/resources/docker.zip b/iem-api/tests/resources/docker.zip
new file mode 100644
index 0000000000000000000000000000000000000000..888efb0b9697997713ed9d800eb25fcf8bf989c9
GIT binary patch
literal 5400
zcmWIWW@Zs#U|`^2Fsh6R6L=XPuFt^0V93D0z|Ek*ker{FmYJ?snV6ds8p6xKuE6~-
zS(W!+GKem%;AUWCdBM!U046S9z54W&kI%M8mqIl>b+x>FPV1jy-4q<e9rQUU=<_7w
z;BLn|f1bR#qNHbQYpcfqvzLLvswyUo0SWLjC@`d?7Nq8-q~;}OrWWgiZ5H7Dmn_8l
zFBwFm*$gv^gW+JXN$8RA4G~!k3=Gp47#Jkcjmpd`OU*0EFRIkbG6EaRU8R{k{rTVI
zt<U}@qZymq?a0?`z{B#sjy3*;Xy<f=l-3hl!V;U`s4Q~&BfF#ERbhI1N0s>Mq;tVW
z$D%j#Zh7B(sMtZ|+SJ#dvih3;t;pql#>~2-Z*zf9!;v+dbALZS`1I8-Be5r+HoSg*
zbxWJhpQ-ZxR;Oj}Hv6G^z-Q%YA58`Z1`P%V261!`6c=a2XQ!e#znS$PHs`Ni7k1)|
zww~u1Z@*rBKV2<v-7bBNQ##%{p1tS&*7oSLc1dboxmLsoj|mQjgN`v_3wbT>1~D)&
zq%bfrNTAz}o*Z_s{F@xch}HVgGlpCZ20RQ0Y}cP)4|;KHeMgIyW0REkY_UIkt#;ij
zaAC<>TFvC*wDsA|utZ&(b3eALm3ll4Doqi;_w?DnsFu~g%^#RF-&m%`x!E+9uTg->
z1}T(EV#3}gcKNSlWMEj$!oVPk?#|rA%sjnHXqw&4{4e<s^S@*ejTXgggLkjHZNPK)
zv*@1&4Y^&FyPeya=5pP+IYHG;Y*ElEjk_xK@m79+GMc+ZR;a(6U}IyoWNMH#tMijX
zx(3hq&)ktses`jB*Bgd8dd$z>tvEZwF>ryL@1IWHn2ArrOy=*H%G|(oDfxkIWUPSE
zvJ?A`yvgr;ygWGby?aBI_v;?B!)LZls8QClV)2kP{<kV_M%t_;XFaWBRxF$J(`u8%
zTApW*`tQA1`bloJWe$V8=}umu9pRG9S;u<Um}ff*?^(+9Mz*r3;LvM1E#<?NA3lkF
zEI4w&_ytpv{GR_m_DNju%t>|P)8$#Jk~jO<G2_-1Zx6Z^AF2_UKl{PTxLLQtwYC;&
zPfhX{bbQ3GJhj*3n9{p@QRkg^tP|0Bpp<eXYQv5FYuNfOzA$+8f_b0P>;3PJs&Ey=
zgxqQF40$r)ZQQYB`M`-fdq2ouOVgTiwX#o3_)po?CF}Q@+Wq*kX4!hX8{J+DyDltW
zGWl5PvBzuUGq;457|&1M#yInRXt~nOU4NbzFMD`q!Rd|5%3{A=mw$G7Ia{vF@@z?^
z+Q9xr>M5ICI8Q3gOg+G~KjF9fw>M?`8DVkHz~F#ZTCjuSK0i4-wFpv9Nb>$mmg4=F
z45HCu8)gUx!{SoZgf4_I1U-o-u|i5b5RGOYEtB~A(wH#hl1v!kLX0x38tOz4jpoFG
zD8n|F#)Ki4VS)&!qm^Euk_%LufM_&_lTmsJRK<kF9gYv|XJBA>%fP_EkFXb|JUGY$
ziP1wm|Imy4(B6x@hYWaJ{#zf_6<|A||3<L&%F?B(+a7UDd|Q~Qa=?DyX8-qdiklCg
ziMr~%(q?7Gj2$gC(Vwyz@;06eIV5s<-ie&$+NyOE+?7qk7q5He%fM&(uHV?3S(;0E
z`PFGpTc_G7&Y1jcH`^AK$KSRkyIp!!_$xWi!z`~u>Gq_gSc~sJ0tKs%PS<XIKPBtO
zL*!KdE)Po`2}$*N>6v*IkhFi`=ig)|hJVQ*8ZA5#Y2Q6NCd@H)VXQL)1A`v}1A{yw
zm>@=_fX&rS&d)8#FHVIQqt8F#G+f(TSL@uF^P7VL%)_2*`RQt%_VLr*_9!So<AlDR
zr+(80Q4ynJ;mg5mMLu0J{;VZ(K#BFIt7?E}EUPLvvo<%gDsppx0WE`bF(@#Uq!tw=
zrsWsqLj1Xq=U*}_@4sXajnkhy9b>|lT$kG2!oa|Ah5=GZAk0C}=Id7eO%~wBT9cLb
zIC32_5OMi#>8dxuVf$TH-HsI#mmGYmK5c!?mQ0Vy4co(8%9frs|D4t$z;k#m(>9xA
zuYYDsCkA$?Ztp2RxRo{Cx;W@~-owqCzAZXimiY9StVMA|_2Tb$m$ko7)2jTt$RJ&w
ztwl2UD9gr}`6dUL;{)shKSz6o)E=uBb6t^qc*&GEyol%nHE~`COy6{!nSsHGhk*f<
zyAa+2C-jmuc+=+PGptRUwPAPjZ<`6!Uym=G5TAXuXz!wGkH$r(yY_8yy=24VtY{#T
zD?HJochMde=U?03_sJXOYJa?TwvG2?Mfv{Uc4;$3k8Rj$ru%2A<j3O*y(?DFHQ0NF
zOHZM`nz4IPjA*wFvxKD8;_^*ydQ+JGyzpCb
lX&%#s*9Qw2m!z=FbU*l0<6}|H
zspf>He}8qS99=!%#_B+_+y3J^vkz9Uy;fu_!C0pK!1&dUvLEqG4q_7(?lW85Uh;}n
z-BIE88p)RH6{>3{CaEt-o0j@zdPkVo4g0M-Cua%EufA*49&=}LahGY__N286yWO=`
zs%7^kgdOmcvfjWJb!>YOCr<@O5Yv&;+q_{RlkSO%%cWhP#_9A!VZtqsQv3U_?gh1^
zoHkqZNvEQs*jmH-O#Rh4rk|fKidge(zrjtL2cL^9k8l5!J}JsPXSuIv?)&I#CNBGh
z`9qH{`53TV?2)DH;>M#FxP!%(H2Qq`$k}=!GnxO$WRFXdoyse9Qs$;yKc0OzBW<E!
zin-)!m(Y}{cO^DV3_E#_ef{NKmOlIkc&>et^NuPjnN<C!T<mLpaLop(B(HgAqmwvZ
zSD#h2GRnME(X;j4S#Q4iT{jwMADwut>8k0hYrmDgUsZm6@{a0N*C>r2AuDg@u+6uL
zn$+fDIk~BM--0P}-&n49NjKi8m?FRAXVnhZTb5t<KE9RB-*R~y$J&XHT<_JE1|>Y$
zu}OWd=c>u4Z^<m2vc8+OHeN4$$=<NN*(z_?rIJs4tp2yG;Qk-hmgAxS-Yz)vZNs|g
zbF4}wXEP_vFR4HDzW%Y<>gbhPPh4}I#oN{VDrPOdWEUd6xSh|`y54xk3iFh&K1*h#
ziDlI_&bhzhP4fSrje?(QLUshV2B@SOnA!;l{QeVTVH+M4KF=UFyzq@$%?Z)xo>N(V
z9(p_RQOuOZ84o`j?dEy^Sfq4r=k1e?iSdvB&su0~qvK$1|8HUc;xC=se=EN)Ut%}2
z#W2ft(NXn>$Aw=^+AioNG2L!i)?KIm?8sape%rmDN=lU9t?b%1;kDg+XXPu$^7w_q
z_o@B1YE0hl`7-<3mynCj*E<*U^}A<fv`+l<MPu!befKZjdNcp7!0)}C$9A8s49XTS
zc#u-~DyfmfIr%Qr>$cl&MVGv{zBFaqyiV_a?cX0SnPcMj?ce$9$Jgzz4qtsfTVLbr
znpV&4&mQ)D{POm2ukFQRWos?h@&Z{kW((7VHn!<I%?!9Ky54`^>S%vgx6$SKQspRF
zPfo$<a`#@Xc~;l$eXs0a+>CRnHW4io-{kf7pI^8)(q{qdBf(iu|BL2qtuk#CTlwGm
zPLXA!Scg5En|TYOzJPRuTs{VUY-eC#@L^<N05u;Fr9^&dNkM6e9^rCgZm%~NlOa#r
zcU#vP%pxW4Bb{^t50%WiB;HX`ep_%(q(;;2{~v3QW$11co1yZ=>7ZenU$5}7s&9`9
zo=IPmlU3Ux-Qc#*XpVz&|K8abUtaoe(>75h(fU?tzIj9CjQ6utdgH#=+e$xN?4h*n
zsrt7^Q@dKS<Acw=N>SLoIR3uuJFklg<v$JV83Vi-nM9az_fi?4U`r#2fnKzO`l_H-
z28hO}Eno&??}mbm1L41p6`(E=X`ND-(U90cZp?wYr66NLcuV79bfYmEcz8Og$Uy_D
z8ju@apw1)63J~7X*uuzw+O0-wjDaixHN>#=gi$QP(nkbY0K!`unV8TlL5~bPJw_B;
zu=MaiHh}P!##QLHfU7+s`g;f~uy-Ut#)I%*$7L)G43LHpq{#v5PT=fTAYvL@#{y&^
z2ybb8jW85FTtM9mP$C7<I1?!c1Ga7f$ZQba(%8!icQi&C0(A~R7Jz6x7GP_<gA51Z
zEsY**a0}2<4=Ck;j0e$pjK|hS2N?~*TN)1_j7N!4JS}$Q7{%5|1{n;(e;wzr!`%*P
z=Yktapr$g;CNm;lu{D}O#)9yc#^>lpLsAYYO=x6)U~2(`ECS)bj_n)_B(;YTp1{^3
z2AK)MTN=-!n2Vl1@wAT-PKT6q=&e(b@gTgVaTzDF)4@?pxDG-$9=%!v84bc)8qe?|
b8&7m47vRmx1~Nd9L69Mije%haABYD4)j5y@
literal 0
HcmV?d00001
diff --git a/iem-api/tests/resources/dummy.zip b/iem-api/tests/resources/dummy.zip
new file mode 100644
index 0000000000000000000000000000000000000000..20d2bc346232701a6a840e0834e7709fdc67bab3
GIT binary patch
literal 3700
zcmWIWW@h1H00FMXmard9H=imoFff2HH-ijAa(-S~X1ZQwVs1`o2qyz`h`vq=2$xoH
zGcdBeU}j(d6S}&(T$zc<R$RIYC8<S4iD~&oxm>ymDf!9SsYP6RdU{+SYeYb{FgR4j
zgn=1Qf}KHu0isYpG=!IdU6S`-vJ~&XWDt#J4a^V@2EoRbF!>qJ9W)pi7<3sJ7=##P
zV1}SOIZ<CH1%%Pex_tHO(^F@(HFP%yb@V+wrLWs)82tH?@#mnmB0`_Lj3*Zb?fmlP
z%MJ#Fo$E_u!j6P*h{$4KV3@|hz#xopcxGN%YF<fxQKep%5!e&etpAc#dH*GYXf%s+
zyB+zO4R~1I*RjUG5bd0<kkWc$OITv_8<j;)e`I$Qyedpj@2C=AopdhP=veed-YxHY
z4;4FzT$}p(Q&wN|zZJQ>&zM<v^ldKiX*jZmbMEiw2cN#$WhD0G(}vg2uWo76`7>4C
z-|DpN-DW?8(>IsKg!!yI?W4)SzyJy|L4?zbi!<W0Q{fJ8hB_QXqd9!_y08;xwDmmC
zc>DG0`{`<V>vrjDoYL{u@$5bCx3)*0wM$a#%C#a!q#zThiV2H593R-vz`*d9fq{V^
zVQ+3?W}aRpC|U6`upi`s1lJ*+f5~XJhW1|MJ!HV+^56QXt^nH!{WpTGSC%eS-S&uM
z;@iSZl>_$sHv7MyQ`~&`Ow?89l{PCgX6$IGiT;$$khk$%$RUx-^G@U}*H*2Y;I3>M
zzIfd$Uj{zQcm2lR%+g%S%dbv*+B(%vamM6lyV<s=JpQ&V+3nJ+!e7a89%gwRO1CE^
z#aev-5hz%7bh>uy`zcvJ9`0v^g((BWyFAo%%!}|_UV3I;1tc{e`1v=PiQ!)|h(_}n
z%qR{9_w1N3$JB+f&I}9;ehdr@^5{lE@`rA6er`d2aVk96o`1q=xVE>h*10q1HwOin
zhdtNw)73id<EOjrQBZ)!34J|J{iY3~B1Xl+mxI@ee7a=(Sxe-A66;S_)d0^}R#k3h
zZEj{&J$N)SFfgEHSuO?z26#S)_;Vr8zhqY4f5{*kr$1F2Tf(@!o}@)GFfbG_FffQR
z$RL@6o^jjsby7eW%{WlT^*ZZ){*3q5APrAlEia$b`lp~-`B6~7h0qWDef+#-yq}7{
zl(v3kmT7K2{I@wKq2~Anf&L#C6y$f*OxQntb+Ly{gnwNo=LwzG7ZG~4wzdfWaxfeW
zHVJ(lFn!Z?W(Ec$9tH+bc0u?ToN`Oj!1<iJN;CQ8v%kq(pZ!HoxNF1i=HE6GsJ|Xx
zI3Yg!YSG?B)gFzDPIv9w;(E!3$63)pBv*K%NAIFNEY82SzweVb%GLgO?Q9$G%Zl>-
zzwOdyiXPjr)lB!#RLPIW6?#{!o@=o83YVTjeKlkEq8QO`8)gYftHtG;-1Md}{dwWH
z;+|1myV5+S3$G6rFfK`9ndyG;r^d&knp4dQP5=JtPC2@IzKzv^WVijtb!H!|UVE*`
zSc0)k`+@PR9c4e_nH<C>D%@wbxV_{RtGc7Y?KP4u*DF-lOiWT=kTxy#%k+*guN(GT
zcTUa{mS26>s6FP+;^HpTxa~=66?VI8tyIhIO$a;SCuO~XE$Z0zAWohNjv%HZrMG#*
zLMGi46_-o9K8@4qhr)zg9;Np8U)>98NjYt{=#x%GMX|Mp^_lvsb4))!T@<nA*?xnY
zHV-}*Ssvg1DSc9udCqcQ(cJgZ*GydY3-gB_U-B_vx!5C1*~N`VFK`EoEot=m@{zOk
zLS{1mk;xvHBs-N?>ZHs~xqdwRZbsTf!4z}J)h?kaQ}0S_m>72Q9Q*pqyDWY95Aa<3
zB<CGfRx+vjPr2CF{NS1mQb}I(&PFG3yskd0YGsspsiJ4=yR+VW^Sf>|&OSQvSkqP0
zS=W9meZQ*w`s5wetFBQRKSEaC%wd~v6E&&L!*X&{^S%XB<i4?7?~-o3Q87h+$<L}C
zuD2||?tOeKo4@7qHjcFuAGzMEEe%R|uw#?@T+daLPv4SRIAwh|Yi+z<_>#S0d$U#E
zuuCPM_*ngKS;753tS!ew|Giys=G%sK(dSr|O3r3Zm|s$V=zaZTv(?cnwVt@<I*Yff
z`BltXe910EdT~3Ssdc^aj1}f7UwxL$NE6GdYn*d`#hc{+KN|%<)r9N_ZVgaLH88ak
z5cvHk#=<r{D14qlY<S@twVD&6&poHI{5<q_;-i=;i!&a6Hrmbe{;^2u+|Ju48x!Ln
z|DUzc*ha^}-2UIf{>5K9xBpgtU%td{W{Y8#>!PFT504AKn6zEcOJcg+vaGvK{n?SZ
zLj1OSKb4dyzgyY0ZNh82_s+^!j^*(Sh3`}QZPl2(-ScJkwJ#wTov(K;=IeLQ%4nVV
z=ZnVL9sBNIy7gxMU4h?wJCE%?TN#usUhp8L@KsVHhja2>rq^w^-HI-GZ+&UXwt1c2
z{o21jUNXnT@7urg*N?B;Umd>se73&E*EOx4+n+t``}pPU;a=N|#md%NuH^-?YRnd<
z32kiCcbXY+S#-VszSYtGu5P2t^QFpBvYwoR)8+2HTJx-~+xuSGzqlFaQf(qyCceq*
z?LWV8Z=}xx)<=S~p8glj*;-}VD7Nyy^_?QiMzIciHaGJYL<ImTC0srReQal7VDMpt
zv<DESM1E;WL1~E|;c{YbuQwNyAy3<PTh|%PA|>x5opb^ZmCU*%-ceD0TX0UKM$_&8
zA8U_g=x!97q4LD(pkbO{ukf*|Z;uL|NnewbRofxm;I_|bj)QXl-q{ylUixp-Hc=(f
z`c`Sac|+xl_p?-b<G$D1N<Uoep|tI(`nN|@yIQj2gU`K6QP{mW{=V!xuZs!gKMm{|
z1H2iT<d|`_1SJ?ifRTZLVM`;3g;G|tLfVCB<uu4Z5oX+NK9DgW{MRv^fdSTd0~e~G
zmLJX*Ajp*<L$S31kzKh4VJNy|v9tyej^$v$*3JVt6oj`lDlozwOG>K`*#c~>F_7UP
zyruCr!UE#kX2`~4YjlB(2H`D@!Ax-XgDWy3nqUaSvA23a27~ZlM}20v;l#Fq7?~i6
z1bZt8WGo18X{<swn)EgivOlmjX+Rc%@LxwM76zn-2{=)L8a6l^IEX}v*~CFklu0P&
zqNhtNjU9xOA>|Evs|4g+5Z=<r#ft1?aMThmd(e$XFU~+lgYcF{15RY)i7wm%yjj^m
R2JkTOFodx%FmQki7y#hT4>te+
literal 0
HcmV?d00001
diff --git a/iem-api/tests/resources/main.yml b/iem-api/tests/resources/main.yml
new file mode 100644
index 0000000..435d9f3
--- /dev/null
+++ b/iem-api/tests/resources/main.yml
@@ -0,0 +1,8 @@
+---
+- hosts: all
+ become: yes
+ tasks:
+ - name: install nmap
+ apt:
+ name: nmap
+ state: latest
diff --git a/iem-api/tests/resources/openstack.zip b/iem-api/tests/resources/openstack.zip
new file mode 100644
index 0000000000000000000000000000000000000000..ddadb660c3f9e3cbffc71126f543e8a9dd293689
GIT binary patch
literal 2117
zcmWIWW@h1H0D&u!d&2@)%f3o7Fff2HH-ijAa(-S~X1ZQwVs1`o2qyzGLvBe52$xoH
zGcdBeU}j(d6S}&(T$zc<R$RIYC8<S4iD~&oxm<dBdR!pWML>o#>~xF?12dom7lQ%=
zT&;d+2rmQsx|M&EH>~`d45HC2fSJR=urhgX*pll~yIU9-7|t*-Fo-h9Aen>i-rBm7
z6c9!;uC&LI>yUwn%Xdpxy$KH6@3QK4teCju;8XQ!>ua`TdQ5KE9^O*6^tAcsv=#xL
z!*iLo*(7`YGg~?_utRlwPw~O6tm)RpLC5nRZr=25(b=-Zr@v$^iX*BQf4{q|{e7BN
z<=;gH>GEtXlEFt=Hpa|1IlvqrU>Eo~+AE~?SiP9*isZvfro7=rc<ErUN$Bf<>6@-I
zGcXwOFfa(B1Wj&YW}aS28aRZwt2C2A;a9%$Z!(&f)`s29zilQ^e?7i%LVWhsqP>f%
zJsKCC?%KD-^^y&bv!a1WuJA;U-bH&@oPTY9-zRUBtNroX**4yn73KSX+ojDEJ+@)1
zneLydk{^#N^sZPv*I@4zE<J_%YR2wGF{0fz%o37Ti_15;=}lq!^TKb%J)^pIrFl#j
zULP!AT#~{v)BWI2jgLh&r<xO*{{7XRa&+~48><7!Zu^hx%syDX_F9p#1Y?=@1LIda
z%6`N%IfzYExX)~Hd&w(Sbw`EUYb0B)SE#O;n54cSZCdJ==^bHSH|)3WoSY>rzxu9G
zd(551#a*Ux+mqHR>~_~$sg~WF5O%;%%6bD^)UoYBoIDjAK}<(VZ}WzQOu8p3E|+$F
z8mH3_g$cJjO6~8zx);=va@uUsC!LClVrvcSGxb;Jn0|h`C}PdC{RTH}9(*pcJih%?
z`lKlHoaMfvx$mQ|nYip1<_|r-<YT~cu}7A&iyMz#;0_jB(&+Q$BWLS{%w+x}lRYj;
zb}FybNtv5+{do4>jI@b@Ddv)^T|!f)-j&!eG3?|y_Vt%{S^DrF;JNlm&O55CWK#8?
za<Q-Z!8IGClDy`fjZWftU42&7$|&<vMbFlEXTAC6cim{5eRSfnrmLp2uKiZ}epUJP
z$vdi7U86L9gsi-o!#3Y0YEqkr<>aR3eG8_@ePg-aCEa+VVv78dpH(|tZ&`lb`}kHi
zf6L`<9BU^&a=lku8kF#0$0qf;o~tIGz9qA8%KC2B+IYS2C40m6W~;nmmr6eIvHIV#
zg8P41TaJhRd%NJww+-u}&#@|%oXwmtzoh=q`})UbtD{$HJ#o!-7H?PctC+R;l3j@O
z;&wh$>w4oEE6h{A`Yf4|CYDv#IOqO~H_88hHVS^K3E2_c8laMDU}`5I@cU1Ug>86H
z_&kHy@WMB0H77)$droEfdFbuLM=?_tXFU9Dw43MsW0BIiowrXmCdNPhKWm|}jgEu4
z{lA6%i@$Vk|E>JKe2Lx67Q-ypMMu>i9v6NwX}h48#B{r5S$CcKvm<kb_-*%oDk)KZ
zx3X*7gx7ZOot3W~%i|Xc->3H5sxf)H=gaJCUqUWAU+-MZ*YBQ{(K_+Z7mc+$_T9g9
z>&^VT0>Afm9@~AkGALWT;6X~^tE5H_=j6LguiI|B6<zY)`qGqb^E$ozwSRxSWR8j7
zw}0oaA78h>I(+r{Y<-QdYg#?GKYQ5s@ypx8y|x#Nm94d0%L`=Hm@P~b+SsP=G&A6`
z=z9NstE2s0-A0$^OO>N!JvjxZ%iVjm=2>00_r0=zaWl@P+C;QWe3RGPe}3WKNS_6)
zj|68u{V$rcwaT<nY~_FJJ4Kd_VjcEuZsslf84;z#p<t6xmybao+Zh-bd>9!Rgi#8<
z{L+$w(h_hv!O6f3D<{y(h`GJqTug>MZQpHOXE2MDypMF!2|QFX>ymg!Mfq*PIguJo
zxBq{vJ(i)nQEZ0F6Q_fQX@0%J$Ev<PDtIP+O-@#AhjfG6KBGAf%Kdw1UwnD#zfIdj
zl|<`XrTOL!l{4PYQt6HRUT-V?aIuHdwx{af9!>3P$&L>`_bNqU_u}~bvhTbuCY1j)
zuxAYLW@M6M##QM^fC@MUMg|6kEsY>1N|DYAse;h_1u{^C8FxJcG6sbII>s?DAk`M&
zG8I(Y;H-5(4h5Nut=>U)=rR;@(VdK?20}QQg8^Il4{|ODZ)vP#M0PTfwE@U@IdF(@
nFkmJeWS7rpMK&HO8L>i=5k^q6vVq*i%D~F7hmnDy7|a6zJV`oy
literal 0
HcmV?d00001
diff --git a/iem-api/tests/resources/shs-bundle.zip b/iem-api/tests/resources/shs-bundle.zip
new file mode 100644
index 0000000000000000000000000000000000000000..afdb59146f1195c79fd82a8bd6f657ec1270b942
GIT binary patch
literal 2223
zcmWIWW@h1H00Hrk$6*1iWnU#37#Kj9n?Z&lIX^EgGhMGTF*hePgp+}JUHhUG5H79Y
zW?*D_!OXw_CUkXmxiS-zt+;d*N>Yo864UaFa=G;M^teE#i+~JgV5o`-gD@Z@7lQ%=
zT&;d+2rmQsLY{xgti1n{K{T2LFmpH<S~Ks3MVdZh)?#2_&}U#^5M_`-G6&tgT|4Hb
zfH0bIm#<!Z>UGxp{2A}9K^mUAT3$Y<^-n!Lbw*o5cVkdT-_uk2x{ZckzkJ!jfUwv+
z>v5QW>a)s43=9l!85kG@Q7q0)%*@j(NdpH`<hn&EAdF_IY42ISCIg=1@2k7g?m3!V
zoE%rd=pQ$I=hO`rciu4g^4RUxPMD>3_W$GSa-Ai@E1LTm6l(bTF0ArMED_T@Yt&Gu
zYO&Vbfs0wPWzC7O<=199Y&;{l^H;Lot`7<StJ`WfTI-!tsZdw^S25wvteufrbKKuW
z<a~Kheexvlm6l!Z7Yg=XX)5@8*45Na+3`_y`NhXghUX^j`kC;~XVtc&HmRivxA;Mx
zg`|a)lzS+##mgYWP@I~RrkjzPn3I{8t`GL;o%Qol9<QI5g62_#F{Nn62%sBdWC}6q
z-}-qe%p0(p1WG;}44!e1!_-R@lpPru7(gzQ#$pnB<lmgRC<TPkoCk`0NK`+%6nf%}
zww~u1Z@*rBT|K8~VpEnY6dQBz_~h;FP$jY`d1uh)OU9pr)`|#y?lPWS6twdTYMSVA
zd>poB!6xNo1_p*z3=9l1SX`BvSC*Prl3!G*mt_PFwG(NJQa~8ZS-EE&xf&D%S|9FR
z{(>u)*<VO#GM~x~4db-#7w=RXMYh!bXU}gDXk*fGW9*#I>}Mvl@T}b=vB+C{I5*k#
zotvDZ5Y7JX+~%(=O4TXf4KvI0jTiU6cK$y@>V1FH9qF&@?5<WYA|hr>X-t^U%F{j?
z3=9k)EQQ74#l;!%*{N`6H?#gr-h>%N)zu1_dBr7(dC95q#i>PQsYUSxMVV!ZC8;3I
z@ukJ7MTQEsweXUFgW+Ms<FLiM#mb@>7#P|Z7#Jk5*bUB3mC!<GYyYAY5Jn3y-Cjqo
z1_K_J_q9js6y#5&t6XzBdP_`*|KGikBF^2LnH3kgtZ?>~ycEUsZPM{riTuEPwJObO
zg_6i0Z?g0j-8YhDHtPM~U~sKT;oYpo&%$+Y81t`>o)+`?EPH@ABa<96Bqv~&iW1-g
zjFEwXVM`;3iIVJCA!Q_*Q$YrbFoRMu1J;5NWDE%Zb&O+RKq}4%7mFZ=g3QHMFd{p2
z8H%~+PR3G1f}AV|b}|P8X3>Xiv^*oSlff|uDhOE_n6VUs5Mv=hi&@A(jAdXDVEF6E
z!^D7I<ghX@gNht1feVW*%mN45$Y69M!Nm_N12d@j!D=cbXfX>Ph^de?Fc*ud=mCtS
z7=nZZ!~)E61la;(X0(tnGSx>dRahCAv6Ltf8z5ngnc<Obc!R|T<m}JNz}(FGFBwk|
rVP<M%^D|g5eE?1<;4IF{z>Fn(2Y9oxf${_w0~bRpGXsMND~JaG{_T7^
literal 0
HcmV?d00001
diff --git a/iem-api/tests/resources/shs.zip b/iem-api/tests/resources/shs.zip
new file mode 100644
index 0000000000000000000000000000000000000000..0541812d6c84e6b1f5d87cdd0cef5255b9a6a16d
GIT binary patch
literal 2169
zcmWIWW@h1H00Hrk$6*1iWnU#37#Kj9n?Z&lIX^EgGhMGTF*hePgp+}JUHhUG5H79Y
zW?*D_!OXw_CUkXmxiS-zt+;d*N>Yo864UaFa=G;M^teE#i+~JgV5o`-gD@Z@7lQ%=
zT&;d+2rmQsLY{xgti1n{K{T2LFmpH<S~Ks3MVdZh)?#2_&}U#^5M_`-G6&tgT|4Hb
zfH0bIm#<!Z>UGxp{2A}9K^mUAT3$Y<^-n!Lbw*o5cVkdT-_uk2x{ZckzkJ!jfUvkX
z>v33=bF^L#0|Ub}1_lN}6pM2cGxPLH(!hcAYu%z05Jt098)i}Hif2Iq8eY0OXLY=M
zJM}eA>7VxT)YW3+*49v&Ca!(M=_VJ~(Vt6QPp;&05M0`G<WbL=Cw~?()h-ZHT~Q+$
zkepU0=dtgc-@%CUCuEd@PqQ96@l9dnmd&9CSBx(i2RvH!lnLx?NUWx$+(QX>UIrP4
z;?$fp-Hg=4oXos*eXx`7te=<ic>TN-G$(_64>6_`%@_f6V~k88CjDDKFNJvnR+B&p
zfrG&_?s1rUiGs2t0|NudWztwoLXXCqGZ&?RFq-o~(FlpON0&lRoYB_vJmc-xtFNo)
z^h|8Za)n}J?j4`Jy&bAV7A5Zt`h3awbI@85q0e2$lZ%3OenE}h9>>RFYZh!$PG(?W
zSjE7=AcMtKnR#WYc_sNpm3mo5;7~h}wkQRJ(VUfg){(10L7?^F-sLa2a+&>wgeLQ;
z+|V#i>wfW0wNYeC?SJ<C7J)V<9XH0#`OJQ1LJQB@O%jW|wTE+)UEjIMDGJf-@6K)h
z%A!=A^4&1AJl}Y6?`!A(Go;@4H{Fr`y3X!u1tTJ2wv@(%`K&zcqrt$y0K!sO99~?U
z5ucq3cXl)DzvNArQB+;6keOFpl9-pA8eg1RRF+y4Ur>};mROPs(i~q}oLXe4P+JSn
z(;N&BBOZq>-Yr%Z#lXPO#=yWJfyHic#;AlAA6xqurGPM6fa&%+ay1z6u)ME5TBjg?
zB3<R0)6rXELj3>kg%ok_-ps7H$Yq7IujHjDrf-vu$4cY}=Brg{Rx6Z5{&<t6x9Gl+
zEVEJX{|19=O$zU3Eq)fRd&8K2ee|@L$7k6CycwC~m?23RvwV{P7e$N=3=CTuK}?in
z#|kO!(3}b~P=pzjk{PfTX&_@j_^)Fe0|QdQM7V$hITU0rwjvJMq03OrMRzim!VctQ
zIk1yC7%&SlWTWL7k(~^VIZ%<t%D{}JID;4q30llT24XA&g8;){$7_rX=tT}I12d?|
z!4kNz*upGukd2gOLNgLv{ID`GgNh%lrb2=iv+#kK3P}U$SWHC^U@XNDBqSgfV3s4u
z7W~Cx0cxqj%D{}JM1j}<32V#@k8Hy>X3T&=&i<?n%+0L-lJNu)W~N3q--rd%d~iYm
eXK_{rW-Qq|z?+o~lqa|txENZQ85r)dfOr5mWm{GN
literal 0
HcmV?d00001
diff --git a/iem-api/tests/unit/__init__.py b/iem-api/tests/unit/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/iem-api/tests/unit/test_iem.py b/iem-api/tests/unit/test_iem.py
new file mode 100644
index 0000000..97fa04d
--- /dev/null
+++ b/iem-api/tests/unit/test_iem.py
@@ -0,0 +1,31 @@
+import logging
+import os
+import unittest
+import uuid
+
+from src.core.iem import Iem
+from src.core.utils import Aws, Credentials, Openstack
+
+LOGGER = logging.getLogger(__name__)
+
+
+class TestIem(unittest.TestCase):
+ def __init__(self, *args, **kwargs):
+ super(TestIem, self).__init__(*args, **kwargs)
+
+ # Check IEM_HOME variable
+ self._iem_home = os.environ["IEM_HOME"]
+
+ def test_get_all_deployments(self):
+ i = Iem()
+ all_deployments = i.get_all_deployments()
+ self.assertGreaterEqual(len(list(all_deployments)), 0)
+
+ def test_get_deployment_ok(self):
+ i = Iem()
+ i.get_deployment(deployment_id="deployment2")
+
+ def test_get_deployment_not_ok(self):
+ i = Iem()
+ deployment_id = str(uuid.uuid4())
+ i.get_deployment(deployment_id=deployment_id)
diff --git a/iem-api/tests/unit/test_main.py b/iem-api/tests/unit/test_main.py
new file mode 100644
index 0000000..fb4591d
--- /dev/null
+++ b/iem-api/tests/unit/test_main.py
@@ -0,0 +1,224 @@
+import base64
+import os
+import unittest
+import uuid
+from unittest.mock import Mock, patch
+
+from fastapi.testclient import TestClient
+from main import app
+
+
+class TestMain(unittest.TestCase):
+ def __init__(self, *args, **kwargs):
+ super(TestMain, self).__init__(*args, **kwargs)
+
+ self.client = TestClient(app)
+
+ # Check API_KEY variable
+ self._api_key = os.environ["API_KEY"]
+
+ def test_root_no_apikey(self):
+ response = self.client.get("/")
+ assert response.status_code == 403
+
+ def test_root(self):
+ response = self.client.get("/", headers={"x-api-key": self._api_key})
+ assert response.status_code == 200
+
+ def test_get_all_deployments(self):
+ response = self.client.get(
+ f"/deployments/", headers={"x-api-key": self._api_key}
+ )
+ assert response.status_code == 200
+
+ def test_get_one_deployment(self):
+ response = self.client.get(
+ f"/deployments/25/", headers={"x-api-key": self._api_key}
+ )
+ assert response.status_code == 200
+
+ @patch("subprocess.run")
+ def test_deployments(self, mock_run):
+ mock_run.return_value = Mock(returncode=0, stdout=b"{}", stderr=b"{}")
+ with open("tests/resources/dummy.zip", "rb") as binary_file:
+ bundle = base64.b64encode(binary_file.read())
+ response = self.client.post(
+ f"/deployments/",
+ headers={"x-api-key": self._api_key},
+ json={
+ "deployment_id": str(uuid.uuid4()),
+ "credentials": {
+ "openstack": {
+ "user_name": "string",
+ "password": "string",
+ "auth_url": "string",
+ "project_name": "string",
+ "region_name": "string",
+ "domain_name": "string",
+ "project_domain_name": "string",
+ "user_domain_name": "string",
+ }
+ },
+ "bundle": {"base64": bundle.decode("utf-8")},
+ },
+ )
+ assert response.status_code == 201
+
+ @patch("subprocess.run")
+ def test_undeploy(self, mock_run):
+ mock_run.return_value = Mock(returncode=0, stdout=b"{}", stderr=b"{}")
+
+ deployment_id = str(uuid.uuid4())
+
+ with open("tests/resources/dummy.zip", "rb") as binary_file:
+ bundle = base64.b64encode(binary_file.read())
+ response = self.client.post(
+ f"/deployments/",
+ headers={"x-api-key": self._api_key},
+ json={
+ "deployment_id": deployment_id,
+ "credentials": {
+ "openstack": {
+ "user_name": "string",
+ "password": "string",
+ "auth_url": "string",
+ "project_name": "string",
+ "region_name": "string",
+ "domain_name": "string",
+ "project_domain_name": "string",
+ "user_domain_name": "string",
+ }
+ },
+ "bundle": {"base64": bundle.decode("utf-8")},
+ },
+ )
+ assert response.status_code == 201
+
+ response = self.client.post(
+ f"/undeploy/",
+ headers={"x-api-key": self._api_key},
+ json={
+ "deployment_id": deployment_id,
+ "credentials": {},
+ },
+ )
+ assert response.status_code == 202
+
+ @patch("subprocess.run")
+ def test_custom(self, mock_run):
+ mock_run.return_value = Mock(returncode=0, stdout=b"{}", stderr=b"{}")
+ with open("tests/resources/dummy.zip", "rb") as binary_file:
+ bundle = base64.b64encode(binary_file.read())
+ response = self.client.post(
+ f"/deployments/",
+ headers={"x-api-key": self._api_key},
+ json={
+ "deployment_id": str(uuid.uuid4()),
+ "credentials": {
+ "openstack": {
+ "user_name": "string",
+ "password": "string",
+ "auth_url": "string",
+ "project_name": "string",
+ "region_name": "string",
+ "domain_name": "string",
+ "project_domain_name": "string",
+ "user_domain_name": "string",
+ },
+ "custom": {
+ "CUSTOM_VAR1": "string",
+ "CUSTOM_VAR2": "string",
+ "CUSTOM_VAR3": "string",
+ },
+ },
+ "bundle": {"base64": bundle.decode("utf-8")},
+ },
+ )
+ assert response.status_code == 201
+
+ @patch("subprocess.run")
+ def test_self_healing_strategy(self, mock_run):
+ mock_run.return_value = Mock(returncode=0, stdout=b"{}", stderr=b"{}")
+
+ deployment_id = str(uuid.uuid4())
+ with open("tests/resources/shs.zip", "rb") as binary_file:
+ bundle = base64.b64encode(binary_file.read())
+
+ response = self.client.post(
+ f"/deployments/",
+ headers={"x-api-key": self._api_key},
+ json={
+ "deployment_id": deployment_id,
+ "credentials": {},
+ "bundle": {"base64": bundle.decode("utf-8")},
+ },
+ )
+ assert response.status_code == 201
+
+ response = self.client.post(
+ f"/self-healing/{25}",
+ headers={"x-api-key": self._api_key},
+ json={
+ "deployment_id": deployment_id,
+ "credentials": {
+ "custom": {
+ "instance_usr": "vagrant",
+ "instance_pwd": "vagrant",
+ "instance_ip": "192.168.56.201",
+ }
+ },
+ },
+ )
+ assert response.status_code == 201
+
+ @patch("subprocess.run")
+ def test_self_healing_bundle(self, mock_run):
+ mock_run.return_value = Mock(returncode=0, stdout=b"{}", stderr=b"{}")
+
+ deployment_id = str(uuid.uuid4())
+ with open("tests/resources/dummy.zip", "rb") as binary_file:
+ bundle = base64.b64encode(binary_file.read())
+
+ response = self.client.post(
+ f"/deployments/",
+ headers={"x-api-key": self._api_key},
+ json={
+ "deployment_id": deployment_id,
+ "credentials": {
+ "openstack": {
+ "user_name": "string",
+ "password": "string",
+ "auth_url": "string",
+ "project_name": "string",
+ "region_name": "string",
+ "domain_name": "string",
+ "project_domain_name": "string",
+ "user_domain_name": "string",
+ }
+ },
+ "bundle": {"base64": bundle.decode("utf-8")},
+ },
+ )
+ assert response.status_code == 201
+
+ response = self.client.post(
+ f"/update-iac-bundle/",
+ headers={"x-api-key": self._api_key},
+ json={
+ "deployment_id": deployment_id,
+ "credentials": {
+ "openstack": {
+ "user_name": "string",
+ "password": "string",
+ "auth_url": "string",
+ "project_name": "string",
+ "region_name": "string",
+ "domain_name": "string",
+ "project_domain_name": "string",
+ "user_domain_name": "string",
+ }
+ },
+ "bundle": {"base64": bundle.decode("utf-8")},
+ },
+ )
+ assert response.status_code == 201
diff --git a/iem-api/tests/unit/test_persistence.py b/iem-api/tests/unit/test_persistence.py
new file mode 100644
index 0000000..6683e8b
--- /dev/null
+++ b/iem-api/tests/unit/test_persistence.py
@@ -0,0 +1,32 @@
+import unittest
+import uuid
+
+from src.core.persistence import Persistence
+
+
+class TestPersistence(unittest.TestCase):
+ @classmethod
+ def setUpClass(self):
+ self.engine_url = "sqlite:///:memory:"
+
+ def test_insert_deployment(self):
+ Persistence(engine_url=self.engine_url).insert_deployment(
+ "deployment1", "STARTED", "stdout", "stderr"
+ )
+
+ def test_get_deployment(self):
+ p = Persistence(engine_url=self.engine_url)
+ p.insert_deployment("deployment2", "STOPPED", "stdout", "stderr")
+ p.insert_deployment("deployment2", "STARTED", "stdout", "stderr")
+ d = p.get_deployment("deployment2")
+ self.assertEqual(d.status, "STARTED")
+
+ def test_get_deployment_not_ok(self):
+ row = Persistence(engine_url=self.engine_url).get_deployment(str(uuid.uuid4()))
+ self.assertIsNone(row)
+
+ def test_get_all_deployments(self):
+ persistence = Persistence(engine_url=self.engine_url)
+ persistence.insert_deployment("deployment3", "STARTED", "stdout", "stderr")
+ rows = persistence.get_all_deployments()
+ self.assertGreaterEqual(len(list(rows)), 1)
diff --git a/openapi.json b/openapi.json
index 4727b5d..a127e22 100644
--- a/openapi.json
+++ b/openapi.json
@@ -1 +1 @@
-{"openapi": "3.0.2", "info": {"title": "IaC Execution Manager", "description": "IaC Execution Manager", "version": "0.1.15"}, "paths": {"/": {"get": {"tags": ["greeting"], "summary": "Read Root", "operationId": "read_root__get", "responses": {"200": {"description": "Successful Response", "content": {"application/json": {"schema": {}}}}}, "security": [{"APIKeyHeader": []}]}}, "/deployments/": {"get": {"tags": ["deployments"], "summary": "Read Status", "operationId": "read_status_deployments__get", "parameters": [{"required": false, "schema": {"title": "Start", "type": "integer", "default": 0}, "name": "start", "in": "query"}, {"required": false, "schema": {"title": "Count", "type": "integer", "default": 25}, "name": "count", "in": "query"}, {"required": false, "schema": {"title": "Start Date", "type": "string", "default": "1970-01-01"}, "name": "start_date", "in": "query"}, {"required": false, "schema": {"title": "End Date", "type": "string", "default": "2100-01-01"}, "name": "end_date", "in": "query"}], "responses": {"200": {"description": "Successful Response", "content": {"application/json": {"schema": {"title": "Response Read Status Deployments Get", "type": "array", "items": {"$ref": "#/components/schemas/DeploymentResponse"}}}}}, "422": {"description": "Validation Error", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/HTTPValidationError"}}}}}, "security": [{"APIKeyHeader": []}]}, "post": {"tags": ["deployments"], "summary": "Deploy", "operationId": "deploy_deployments__post", "requestBody": {"content": {"application/json": {"schema": {"$ref": "#/components/schemas/DeploymentRequest"}}}, "required": true}, "responses": {"201": {"description": "Successful Response", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/BaseResponse"}}}}, "422": {"description": "Validation Error", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/HTTPValidationError"}}}}}, "security": [{"APIKeyHeader": []}]}}, "/deployments/{deployment_id}": {"get": {"tags": ["deployments"], "summary": "Read Status Deployment", "operationId": "read_status_deployment_deployments__deployment_id__get", "parameters": [{"required": true, "schema": {"title": "Deployment Id", "type": "string"}, "name": "deployment_id", "in": "path"}, {"required": false, "schema": {"title": "Start", "type": "integer", "default": 0}, "name": "start", "in": "query"}, {"required": false, "schema": {"title": "Count", "type": "integer", "default": 1}, "name": "count", "in": "query"}], "responses": {"200": {"description": "Successful Response", "content": {"application/json": {"schema": {"title": "Response Read Status Deployment Deployments Deployment Id Get", "type": "array", "items": {"$ref": "#/components/schemas/DeploymentResponse"}}}}}, "422": {"description": "Validation Error", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/HTTPValidationError"}}}}}, "security": [{"APIKeyHeader": []}]}}, "/undeploy/": {"post": {"tags": ["deployments"], "summary": "Undeploy", "operationId": "undeploy_undeploy__post", "requestBody": {"content": {"application/json": {"schema": {"$ref": "#/components/schemas/DeleteDeploymentRequest"}}}, "required": true}, "responses": {"202": {"description": "Successful Response", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/BaseResponse"}}}}, "422": {"description": "Validation Error", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/HTTPValidationError"}}}}}, "security": [{"APIKeyHeader": []}]}}}, "components": {"schemas": {"Aws": {"title": "Aws", "required": ["access_key_id", "secret_access_key"], "type": "object", "properties": {"access_key_id": {"title": "Access Key Id", "type": "string"}, "secret_access_key": {"title": "Secret Access Key", "type": "string"}}}, "Azure": {"title": "Azure", "required": ["arm_client_id", "arm_client_secret", "arm_subscription_id", "arm_tenant_id"], "type": "object", "properties": {"arm_client_id": {"title": "Arm Client Id", "type": "string"}, "arm_client_secret": {"title": "Arm Client Secret", "type": "string"}, "arm_subscription_id": {"title": "Arm Subscription Id", "type": "string"}, "arm_tenant_id": {"title": "Arm Tenant Id", "type": "string"}}}, "BaseResponse": {"title": "BaseResponse", "required": ["message"], "type": "object", "properties": {"message": {"title": "Message", "type": "string"}}}, "Credentials": {"title": "Credentials", "type": "object", "properties": {"aws": {"$ref": "#/components/schemas/Aws"}, "azure": {"$ref": "#/components/schemas/Azure"}, "openstack": {"$ref": "#/components/schemas/Openstack"}}}, "DeleteDeploymentRequest": {"title": "DeleteDeploymentRequest", "required": ["deployment_id", "credentials"], "type": "object", "properties": {"deployment_id": {"title": "Deployment Id", "type": "string"}, "credentials": {"$ref": "#/components/schemas/Credentials"}}}, "DeploymentRequest": {"title": "DeploymentRequest", "required": ["deployment_id", "repository", "commit", "credentials"], "type": "object", "properties": {"deployment_id": {"title": "Deployment Id", "type": "string"}, "repository": {"title": "Repository", "type": "string"}, "commit": {"title": "Commit", "type": "string"}, "credentials": {"$ref": "#/components/schemas/Credentials"}}}, "DeploymentResponse": {"title": "DeploymentResponse", "required": ["status_time", "deployment_id", "status"], "type": "object", "properties": {"status_time": {"title": "Status Time", "type": "string", "format": "date-time"}, "deployment_id": {"title": "Deployment Id", "type": "string"}, "status": {"title": "Status", "type": "string"}, "stdout": {"title": "Stdout", "type": "string"}, "stderr": {"title": "Stderr", "type": "string"}}}, "HTTPValidationError": {"title": "HTTPValidationError", "type": "object", "properties": {"detail": {"title": "Detail", "type": "array", "items": {"$ref": "#/components/schemas/ValidationError"}}}}, "Openstack": {"title": "Openstack", "required": ["user_name", "password", "auth_url", "project_name"], "type": "object", "properties": {"user_name": {"title": "User Name", "type": "string"}, "password": {"title": "Password", "type": "string"}, "auth_url": {"title": "Auth Url", "type": "string"}, "project_name": {"title": "Project Name", "type": "string"}, "region_name": {"title": "Region Name", "type": "string"}, "domain_name": {"title": "Domain Name", "type": "string"}, "project_domain_name": {"title": "Project Domain Name", "type": "string"}, "user_domain_name": {"title": "User Domain Name", "type": "string"}}}, "ValidationError": {"title": "ValidationError", "required": ["loc", "msg", "type"], "type": "object", "properties": {"loc": {"title": "Location", "type": "array", "items": {"type": "string"}}, "msg": {"title": "Message", "type": "string"}, "type": {"title": "Error Type", "type": "string"}}}}, "securitySchemes": {"APIKeyHeader": {"type": "apiKey", "in": "header", "name": "x-api-key"}}}}
\ No newline at end of file
+{"openapi": "3.0.2", "info": {"title": "IaC Execution Manager", "description": "IaC Execution Manager", "version": "3.0.1.17"}, "paths": {"/": {"get": {"tags": ["greeting"], "summary": "Read Root", "operationId": "read_root__get", "responses": {"200": {"description": "Successful Response", "content": {"application/json": {"schema": {}}}}}, "security": [{"APIKeyHeader": []}]}}, "/deployments/": {"get": {"tags": ["deployments"], "summary": "Read Status", "operationId": "read_status_deployments__get", "responses": {"200": {"description": "Successful Response", "content": {"application/json": {"schema": {"title": "Response Read Status Deployments Get", "type": "array", "items": {"$ref": "#/components/schemas/DeploymentResponse"}}}}}}, "security": [{"APIKeyHeader": []}]}, "post": {"tags": ["deployments"], "summary": "Deploy", "operationId": "deploy_deployments__post", "requestBody": {"content": {"application/json": {"schema": {"$ref": "#/components/schemas/DeploymentRequest"}}}, "required": true}, "responses": {"201": {"description": "Successful Response", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/BaseResponse"}}}}, "422": {"description": "Validation Error", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/HTTPValidationError"}}}}}, "security": [{"APIKeyHeader": []}]}}, "/deployments/{deployment_id}": {"get": {"tags": ["deployments"], "summary": "Read Status Deployment", "operationId": "read_status_deployment_deployments__deployment_id__get", "parameters": [{"required": true, "schema": {"title": "Deployment Id", "type": "string"}, "name": "deployment_id", "in": "path"}], "responses": {"200": {"description": "Successful Response", "content": {"application/json": {"schema": {"title": "Response Read Status Deployment Deployments Deployment Id Get", "type": "array", "items": {"$ref": "#/components/schemas/DeploymentResponse"}}}}}, "422": {"description": "Validation Error", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/HTTPValidationError"}}}}}, "security": [{"APIKeyHeader": []}]}}, "/deployments/{deployment_id}/{stage_id}/outputs": {"get": {"tags": ["deployments"], "summary": "Read Deployment Outputs", "operationId": "read_deployment_outputs_deployments__deployment_id___stage_id__outputs_get", "parameters": [{"required": true, "schema": {"title": "Deployment Id", "type": "string"}, "name": "deployment_id", "in": "path"}, {"required": true, "schema": {"title": "Stage Id", "type": "string"}, "name": "stage_id", "in": "path"}], "responses": {"200": {"description": "Successful Response", "content": {"application/json": {"schema": {"title": "Response Read Deployment Outputs Deployments Deployment Id Stage Id Outputs Get", "type": "object"}}}}, "422": {"description": "Validation Error", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/HTTPValidationError"}}}}}, "security": [{"APIKeyHeader": []}]}}, "/undeploy/": {"post": {"tags": ["deployments"], "summary": "Undeploy", "operationId": "undeploy_undeploy__post", "requestBody": {"content": {"application/json": {"schema": {"$ref": "#/components/schemas/DeleteDeploymentRequest"}}}, "required": true}, "responses": {"202": {"description": "Successful Response", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/BaseResponse"}}}}, "422": {"description": "Validation Error", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/HTTPValidationError"}}}}}, "security": [{"APIKeyHeader": []}]}}, "/deployments/{deployment_id}/self-healing": {"post": {"tags": ["deployments"], "summary": "Self Healing Strategy", "operationId": "self_healing_strategy_deployments__deployment_id__self_healing_post", "parameters": [{"required": true, "schema": {"title": "Deployment Id", "type": "string"}, "name": "deployment_id", "in": "path"}], "requestBody": {"content": {"application/json": {"schema": {"$ref": "#/components/schemas/SelfHealingRequest"}}}, "required": true}, "responses": {"201": {"description": "Successful Response", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/BaseResponse"}}}}, "422": {"description": "Validation Error", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/HTTPValidationError"}}}}}, "security": [{"APIKeyHeader": []}]}}, "/update-iac-bundle/": {"post": {"tags": ["deployments"], "summary": "Self Healing Bundle", "operationId": "self_healing_bundle_update_iac_bundle__post", "requestBody": {"content": {"application/json": {"schema": {"$ref": "#/components/schemas/DeploymentRequest"}}}, "required": true}, "responses": {"201": {"description": "Successful Response", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/BaseResponse"}}}}, "422": {"description": "Validation Error", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/HTTPValidationError"}}}}}, "security": [{"APIKeyHeader": []}]}}}, "components": {"schemas": {"Aws": {"title": "Aws", "required": ["access_key_id", "secret_access_key"], "type": "object", "properties": {"access_key_id": {"title": "Access Key Id", "type": "string"}, "secret_access_key": {"title": "Secret Access Key", "type": "string"}, "region": {"title": "Region", "type": "string", "default": "us-west-2"}}}, "Azure": {"title": "Azure", "required": ["arm_client_id", "arm_client_secret", "arm_subscription_id", "arm_tenant_id"], "type": "object", "properties": {"arm_client_id": {"title": "Arm Client Id", "type": "string"}, "arm_client_secret": {"title": "Arm Client Secret", "type": "string"}, "arm_subscription_id": {"title": "Arm Subscription Id", "type": "string"}, "arm_tenant_id": {"title": "Arm Tenant Id", "type": "string"}}}, "BaseResponse": {"title": "BaseResponse", "required": ["message"], "type": "object", "properties": {"message": {"title": "Message", "type": "string"}}}, "Bundle": {"title": "Bundle", "required": ["base64"], "type": "object", "properties": {"base64": {"title": "Base64", "type": "string"}}}, "Credentials": {"title": "Credentials", "type": "object", "properties": {"aws": {"$ref": "#/components/schemas/Aws"}, "azure": {"$ref": "#/components/schemas/Azure"}, "openstack": {"$ref": "#/components/schemas/Openstack"}, "vmware": {"$ref": "#/components/schemas/Vmware"}, "docker": {"$ref": "#/components/schemas/Docker"}, "custom": {"title": "Custom", "type": "object"}}}, "DeleteDeploymentRequest": {"title": "DeleteDeploymentRequest", "required": ["deployment_id", "credentials"], "type": "object", "properties": {"deployment_id": {"title": "Deployment Id", "type": "string"}, "credentials": {"$ref": "#/components/schemas/Credentials"}}}, "DeploymentRequest": {"title": "DeploymentRequest", "required": ["deployment_id", "credentials", "bundle"], "type": "object", "properties": {"deployment_id": {"title": "Deployment Id", "type": "string"}, "credentials": {"$ref": "#/components/schemas/Credentials"}, "bundle": {"$ref": "#/components/schemas/Bundle"}}}, "DeploymentResponse": {"title": "DeploymentResponse", "required": ["status_time", "deployment_id", "status"], "type": "object", "properties": {"status_time": {"title": "Status Time", "type": "string", "format": "date-time"}, "deployment_id": {"title": "Deployment Id", "type": "string"}, "status": {"title": "Status", "type": "string"}, "stdout": {"title": "Stdout", "type": "string"}, "stderr": {"title": "Stderr", "type": "string"}}}, "Docker": {"title": "Docker", "required": ["server", "user_name", "password"], "type": "object", "properties": {"server": {"title": "Server", "type": "string"}, "user_name": {"title": "User Name", "type": "string"}, "password": {"title": "Password", "type": "string"}}}, "HTTPValidationError": {"title": "HTTPValidationError", "type": "object", "properties": {"detail": {"title": "Detail", "type": "array", "items": {"$ref": "#/components/schemas/ValidationError"}}}}, "Openstack": {"title": "Openstack", "required": ["user_name", "password", "auth_url", "project_name"], "type": "object", "properties": {"user_name": {"title": "User Name", "type": "string"}, "password": {"title": "Password", "type": "string"}, "auth_url": {"title": "Auth Url", "type": "string"}, "project_name": {"title": "Project Name", "type": "string"}, "region_name": {"title": "Region Name", "type": "string"}, "domain_name": {"title": "Domain Name", "type": "string"}, "project_domain_name": {"title": "Project Domain Name", "type": "string"}, "user_domain_name": {"title": "User Domain Name", "type": "string"}}}, "SelfHealingRequest": {"title": "SelfHealingRequest", "required": ["credentials", "playbook"], "type": "object", "properties": {"credentials": {"$ref": "#/components/schemas/Credentials"}, "playbook": {"title": "Playbook", "type": "string"}}}, "ValidationError": {"title": "ValidationError", "required": ["loc", "msg", "type"], "type": "object", "properties": {"loc": {"title": "Location", "type": "array", "items": {"type": "string"}}, "msg": {"title": "Message", "type": "string"}, "type": {"title": "Error Type", "type": "string"}}}, "Vmware": {"title": "Vmware", "required": ["user_name", "password", "server"], "type": "object", "properties": {"user_name": {"title": "User Name", "type": "string"}, "password": {"title": "Password", "type": "string"}, "server": {"title": "Server", "type": "string"}, "allow_unverified_ssl": {"title": "Allow Unverified Ssl", "type": "string"}}}}, "securitySchemes": {"APIKeyHeader": {"type": "apiKey", "in": "header", "name": "x-api-key"}}}}
\ No newline at end of file
diff --git a/sonar-project.properties b/sonar-project.properties
new file mode 100644
index 0000000..9f5f8e0
--- /dev/null
+++ b/sonar-project.properties
@@ -0,0 +1,5 @@
+sonar.projectKey=piacere_private_t51-iem_AXlg6OYJGykB3kuTt_u4
+sonar.qualitygate.wait=true
+sonar.sources=iem-api/src
+sonar.python.coverage.reportPaths=iem-api/coverage.xml
+
--
GitLab