y1 files

.gitignore

+__pycache__
+env/
+*.png
+.coverage
+coverage.xml
+.vscode
+**/.env
+*.swp
+iem-api/iem.db
+iem-api/.venv

README.md

+# T51 IaC Executor Manager
+
+Running the server
+```bash
+uvicorn main:app --reload
+```
+
+###### Containers
+
+Containerize the IEM
+```bash
+docker build --build-arg API_KEY=$API_KEY -t optima-piacere-docker-dev.artifact.tecnalia.com/wp5/iem-api:y1 .
+```
+
+Similarly, docker compose can be used to build both
+```bash
+docker-compose build
+```
+
+It can also be used to push them to the registry
+```bash
+docker-compose push
+```
+
+Run the IEM
+```bash
+docker run -p 8000:8000 optima-piacere-docker-dev.artifact.tecnalia.com/wp5/iem-api:y1
+```
+
+###### Tests
+
+Run all tests
+
+```bash
+nose2 -v
+```
+
+Run a single test
+
+```bash
+nose2 -v tests.core.test_iem.TestIem.test_deploy_destroy_openstack
+```
+
+

doc/sequence-diagrams/51-request-deployment-status.puml

+@startuml
+
+title Request the Current Status of a Deployment
+
+participant "PRC" as DESIDE
+
+box "IaC Execution Manager" #LightBlue
+participant "Rest API" as RTIEM_api #99FF99
+participant Core as RTIEM_core #99FF99
+participant Persistence as RTIEM_db #99FF99
+end box
+
+
+DESIDE -> RTIEM_api: Deployment Status Request
+
+RTIEM_api -> RTIEM_core: Deployment Status Request
+
+RTIEM_core -> RTIEM_db: Deployment Status Request
+
+RTIEM_core <-- RTIEM_db: Deployment Status Response
+
+RTIEM_api <-- RTIEM_core: Deployment Status Response
+
+DESIDE <-- RTIEM_api: Deployment Status Response
+
+@enduml

doc/sequence-diagrams/51-start-deployment-custom-configuration.puml

+@startuml
+
+title Initiate Deployment (Custom Configuration)
+
+participant "GUI/IDE via Eclipse" as DESIDE
+
+participant "Runtime Controller (PRC)" as RTPRC
+
+box "IaC Execution Manager" #LightBlue
+participant "Rest API" as RTIEM_api #99FF99
+participant Core as RTIEM_core #99FF99
+participant Persistence as RTIEM_db #99FF99
+end box
+
+participant "DOML & IaC\nrepository" as DBDOMLIAC
+
+participant "Executor" as executor
+
+collections "Resource Provider" as infraresource
+
+DESIDE -> RTPRC: Deployment Request, including: \n - custom configuration \n - custom configuration reference 
+
+note right of DESIDE: custom configuration ,may include: \n - environment variables \n - json or yaml structures \n - base64 encoded file content \n - etc \ncustom configuration reference, \n are references to previously \n uploaded configurations
+
+RTPRC -> RTIEM_api: Deployment Request
+RTPRC <-- RTIEM_api: Deployment Response
+
+DESIDE <-- RTPRC: Deployment Response
+
+RTIEM_api -> RTIEM_core: Deployment Request
+
+RTIEM_core -> RTIEM_db: Save Deployment Started
+
+RTIEM_core -> DBDOMLIAC: IaC Deployment Request
+
+RTIEM_core <-- DBDOMLIAC: IaC Deployment Response
+
+RTIEM_core -> executor: Deployment Request
+
+executor -> infraresource: Deploy Commands
+executor -> infraresource: ...
+executor -> infraresource: Deploy Commands
+
+executor -> RTIEM_core: Deployment Response
+
+RTIEM_core -> RTIEM_db: Save Deployment Status
+
+@enduml

doc/sequence-diagrams/51-start-deployment-secrets.puml

+@startuml
+
+title Initiate Deployment (Infrastructure Provider Secrets inyection to be used by the executor)
+
+participant "GUI/IDE via Eclipse" as DESIDE
+
+participant "Runtime Controller (PRC)" as RTPRC
+
+box "IaC Execution Manager" #LightBlue
+participant "Rest API" as RTIEM_api #99FF99
+participant Core as RTIEM_core #99FF99
+participant Persistence as RTIEM_db #99FF99
+end box
+
+participant "DOML & IaC\nrepository" as DBDOMLIAC
+
+participant "Executor" as executor
+
+collections "Resource Provider" as infraresource
+
+DESIDE -> infraresource: configure api tokens, \n credentials, etc
+DESIDE <-- infraresource: new api tokens
+
+DESIDE -> RTPRC: Deployment Request \n including api tokens
+
+RTPRC -> RTIEM_api: Deployment Request
+RTPRC <-- RTIEM_api: Deployment Response
+
+DESIDE <-- RTPRC: Deployment Response
+
+RTIEM_api -> RTIEM_core: Deployment Request
+
+RTIEM_core -> RTIEM_db: Save Deployment Started
+
+RTIEM_core -> DBDOMLIAC: IaC Deployment Request
+
+RTIEM_core <-- DBDOMLIAC: IaC Deployment Response
+
+RTIEM_core -> executor: Deployment Request
+
+executor -> infraresource: Deploy Commands
+executor -> infraresource: ...
+executor -> infraresource: Deploy Commands
+
+executor -> RTIEM_core: Deployment Response
+
+RTIEM_core -> RTIEM_db: Save Deployment Status
+
+@enduml

doc/sequence-diagrams/51-start-deployment.puml

+@startuml
+
+title Initiate Deployment
+
+participant "Runtime Controller (PRC)" as RTPRC
+
+box "IaC Execution Manager" #LightBlue
+participant "Rest API" as RTIEM_api #99FF99
+participant Core as RTIEM_core #99FF99
+participant Persistence as RTIEM_db #99FF99
+end box
+
+participant "DOML & IaC\nrepository" as DBDOMLIAC
+
+participant "Executor" as executor
+
+collections "Resource Provider" as infraresource
+
+RTPRC -> RTIEM_api: Deployment Request
+RTPRC <-- RTIEM_api: Deployment Response
+
+RTIEM_api -> RTIEM_core: Deployment Request
+
+RTIEM_core -> RTIEM_db: Save Deployment Started
+
+RTIEM_core -> DBDOMLIAC: IaC Deployment Request
+
+RTIEM_core <-- DBDOMLIAC: IaC Deployment Response
+
+RTIEM_core -> executor: Deployment Request
+
+executor -> infraresource: Deploy Commands
+executor -> infraresource: ...
+executor -> infraresource: Deploy Commands
+
+executor -> RTIEM_core: Deployment Response
+
+RTIEM_core -> RTIEM_db: Save Deployment Status
+
+@enduml

doc/sequence-diagrams/README.md

+# T51 IaC Executor Manager Secuence diagrams
+
+This folder contains the sequence diagrams developed for the T51 IEM. They have been developed using plantuml
+* https://plantuml.com
+
+These files follow a very simple text based syntax. ie
+```
+Bob->Alice : Hello!
+```
+which renders (providing plantuml is enabled in gitlab https://docs.gitlab.com/ee/administration/integration/plantuml.html) as 
+
+```plantuml 
+Bob->Alice : Hello!
+```  
+we can also specify a file
+
+```plantuml source="51-start-deployment.puml"
+```  
+
+To be able to edit them and check the rendering there are several options:
+* Edit and generate the file using the jar, which is not very user friendly  
+``` java -jar plantuml.jar sequenceDiagram.txt ```
+* Use an IDE and a plugin. There are plugins available for different IDEs,i.e.
+  * eclipse https://plantuml.com/eclipse
+  * visual code https://marketplace.visualstudio.com/items?itemName=jebbs.plantuml
+
+

docker-compose.yml

+version: '3'
+services:
+  iem-api:
+    image: optima-piacere-docker-dev.artifact.tecnalia.com/wp5/iem-api:y1
+    build: iem-api/.
+    ports:
+    - "8000:8000"
+    volumes:
+    - iem-db:/opt/iem/iem.db 
+
+volumes:
+  iem-db:

docs/.gitignore

+/docs_files/_build/

docs/README.md

+# IEM Documentation
+
+This directory contains the IaC Executor Manager documentation (in the `docs_files` directory), in the form of a Sphinx project with reStructuredText source files.
+
+## TODO: fill the skeleton
+
+The files provided in the `docs_files` directory are merely a skeleton of the target documentation. The authors of this component should fill this skeleton with proper content.
+
+The main task to accomplish here is to provide adequate `*.rst` (*reStructuredText*) files, corresponding to specific subpages of the target documentation. The names of the new docs files can be added by the authors to the `toctree`s in the `index.rst` file in order to add links to the corresponding subpages to the table of contents (in the sidebar). However, simply providing the `*.rst` files with documentation content is fine as well - 7bulls will take care of integrating them in such case.
+
+Some information needs to be added to the already provided files as well. The files include:
+- `conf.py` - the configuration of the Sphinx docs generator
+- `index.rst` - the home page of the documentation
+- `01-intro.rst` - the page with an outline of the component
+- `contact.rst` - the page with contact info
+
+Some of the information to provide is outlined and indicated with `TODO` comments in these files.
+
+*reStructuredText* is a markup language designed for writing technical documentation. A primer on its most crucial elements can be found [here](https://www.sphinx-doc.org/en/master/usage/restructuredtext/basics.html).
+
+Some PIACERE components with already prepared documentation following this or a similar style include:
+- [IOP](https://git.code.tecnalia.com/piacere/private/t53-iop-optimizer/-/tree/docs/add-docs/docs)
+- [ISR](https://github.com/xlab-si/iac-scanner-docs)
+- [IDE](https://git.code.tecnalia.com/piacere/private/ide_tool/-/tree/main/docs) (this one uses Markdown as its documentation source language)
+
+## Documentation building
+
+The documentation can be built using Sphinx. First, install the requirements using:
+```
+pip3 install -r requirements.txt
+```
+
+or, if you wish to use a virtual environment:
+```
+python3 -m venv venv
+source ./venv/bin/activate
+pip3 install -r requirements.txt
+```
+
+Next, in order to generate the result HTML documentation, run the following commands:
+```
+cd docs_files
+make clean html
+```
+
+The result documentation can then be found in the `docs_files/_build` directory.
+
+## TODO: documentation building and deployment in CI/CD
+
+The documentation will eventually be automatically built and deployed to a server (so that it is available to end users) as part of the CI/CD process. 7bulls will take care of this aspect.

docs/docs_files/01-intro.rst

+.. _Introduction:
+
+************
+Introduction
+************
+
+..
+    TODO Provide a brief description of the component here. Outline its goals, functionalities, etc.;
+    Mention subcomponents or extra delivered tools etc., with rst references to adequate sections.
+    
\ No newline at end of file

docs/docs_files/Makefile

+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = .
+BUILDDIR      = _build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/docs_files/conf.py

+# -*- coding: utf-8 -*-
+#
+# Configuration file for the Sphinx documentation builder.
+#
+# This file does only contain a selection of the most common options. For a
+# full list see the documentation:
+# http://www.sphinx-doc.org/en/master/config
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+# import os
+# import sys
+# sys.path.insert(0, os.path.abspath('.'))
+
+# -- Project information -----------------------------------------------------
+
+project = u'IaC Executor Manager'
+copyright = u'' # TODO
+author = u'' # TODO
+
+# The short X.Y version
+version = u''
+# The full version, including alpha/beta/rc tags
+release = u''
+
+# -- General configuration ---------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#
+# needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+
+extensions = ['sphinx_rtd_theme',
+              'sphinx_copybutton']
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix(es) of source filenames.
+# You can specify multiple suffix as a list of string:
+#
+# source_suffix = ['.rst', '.md']
+source_suffix = '.rst'
+
+# The master toctree document.
+master_doc = 'index'
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#
+# This is also used if you do content translation via gettext catalogs.
+# Usually you set "language" from the command line for these cases.
+language = None
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = []
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = None
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+html_theme = 'sphinx_rtd_theme'
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+#
+# html_theme_options = {}
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = []
+
+# Custom sidebar templates, must be a dictionary that maps document names
+# to template names.
+#
+# The default sidebars (for documents that don't match any pattern) are
+# defined by theme itself.  Builtin themes are using these templates by
+# default: ``['localtoc.html', 'relations.html', 'sourcelink.html',
+# 'searchbox.html']``.
+#
+# html_sidebars = {}

docs/docs_files/contact.rst

+.. _Get in touch:
+
+************
+Get in touch
+************
+
+.. TODO Provide contact info here.

docs/docs_files/index.rst

+************************************************
+Welcome to IaC Executor Manager's documentation!
+************************************************
+
+The following documentation explains the **IaC Executor Manager** (IEM).
+
+.. toctree::
+    :caption: Table of Contents
+    :maxdepth: 2
+
+    01-intro
+..
+   TODO Provide the names of your files with sections here (as additional
+   elements of the above list).
+   They will be displayed in the sidebar as references to those files,
+   as well as in the form of a table of contents on the index page.
+
+.. toctree::
+    :caption: More info
+    :maxdepth: 2
+
+    contact
+..
+   TODO Provide the names of any extra files or links to resources here
+   (as additional elements of the above list).
+
+.. _Acknowledgments:
+
+===============
+Acknowledgments
+===============
+
+..
+   TODO Provide info on acknowledgements here.
+
+This work is being supported by the European Union’s Horizon 2020 research and innovation programme
+(grant no. 101000162, `PIACERE`_).
+
+.. _PIACERE: https://www.piacere-project.eu/

docs/docs_files/make.bat

+@ECHO OFF
+
+pushd %~dp0
+
+REM Command file for Sphinx documentation
+
+if "%SPHINXBUILD%" == "" (
+	set SPHINXBUILD=sphinx-build
+)
+set SOURCEDIR=.
+set BUILDDIR=_build
+
+%SPHINXBUILD% >NUL 2>NUL
+if errorlevel 9009 (
+	echo.
+	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
+	echo.installed, then set the SPHINXBUILD environment variable to point
+	echo.to the full path of the 'sphinx-build' executable. Alternatively you
+	echo.may add the Sphinx directory to PATH.
+	echo.
+	echo.If you don't have Sphinx installed, grab it from
+	echo.https://www.sphinx-doc.org/
+	exit /b 1
+)
+
+if "%1" == "" goto help
+
+%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+goto end
+
+:help
+%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+
+:end
+popd

docs/requirements.txt

+sphinx==4.4.0
+sphinx-rtd-theme==1.0.0
+sphinx-copybutton==0.5.0

iem-api/.python-version

+3.9.7

iem-api/Dockerfile

+FROM hashicorp/terraform:1.1.4
+
+ARG API_KEY
+
+ENV API_KEY=$API_KEY
+ENV IEM_HOME=/opt/iem/
+
+COPY src/resources/ansible.cfg /etc/ansible/ansible.cfg
+COPY requirements.txt /tmp/requirements.txt
+COPY src ${IEM_HOME}src
+COPY main.py ${IEM_HOME}main.py
+
+RUN apk add py3-pip cargo g++ python3-dev file libffi-dev openssl-dev bash python3=3.9.13-r1 gnupg
+RUN pip3 install -r /tmp/requirements.txt
+
+# RUN adduser -h ${IEM_HOME} -S -D iem
+COPY certs/config ${IEM_HOME}.ssh/config
+COPY certs/id_rsa ${IEM_HOME}.ssh/id_rsa
+COPY certs/id_rsa.pub ${IEM_HOME}.ssh/id_rsa.pub
+RUN adduser -h ${IEM_HOME} -S -D iem && \
+    chown -R iem ${IEM_HOME} && \
+    chmod 0700 ${IEM_HOME}.ssh && \
+    chmod 0644 ${IEM_HOME}.ssh/config && \
+    chmod 0600 ${IEM_HOME}.ssh/id_rsa && \
+    chmod 0644 ${IEM_HOME}.ssh/id_rsa.pub
+USER iem
+RUN ansible-galaxy collection install community.general
+COPY roles.yml /tmp/roles.yml
+RUN ansible-galaxy install -r /tmp/roles.yml
+
+ENTRYPOINT ["/usr/bin/env"]
+WORKDIR ${IEM_HOME}
+CMD /usr/bin/uvicorn main:app --host 0.0.0.0
+EXPOSE 8000

iem-api/main.py

+import json
+import logging
+import os
+
+from fastapi import FastAPI, BackgroundTasks, status, Security, Depends, HTTPException
+from fastapi.openapi.utils import get_openapi
+from fastapi.security.api_key import APIKeyHeader, APIKey
+from typing import List
+
+from src.core.iem import Iem
+from src.core.persistence import Sqlite
+from src.core.utils import (
+    BaseResponse,
+    DeploymentResponse,
+    DeploymentRequest,
+    DeleteDeploymentRequest,
+)
+
+LOGGER = logging.getLogger("iem")
+
+api_key_header = APIKeyHeader(name="x-api-key", auto_error=False)
+
+app = FastAPI(
+    title="IaC Execution Manager", version="0.1.15", description="IaC Execution Manager"
+)
+
+
+async def get_api_key(api_key_query: str = Security(api_key_header)):
+
+    if Sqlite().valid_api_key(api_key_query=api_key_query):
+        return api_key_query
+    else:
+        raise HTTPException(
+            status_code=status.HTTP_403_FORBIDDEN,
+            detail="Could not validate credentials",
+        )
+
+
+@app.get("/", tags=["greeting"])
+async def read_root(api_key: APIKey = Depends(get_api_key)):
+    return {
+        "message": "Hello from the IaC Execution Manager!",
+        "version": app.version,
+        "terraform": "1.1.4",
+        "ansible": "5.5.0",
+    }
+
+
+@app.get("/deployments/", response_model=List[DeploymentResponse], tags=["deployments"])
+async def read_status(
+    start: int = 0,
+    count: int = 25,
+    start_date: str = "1970-01-01",
+    end_date: str = "2100-01-01",
+    api_key: APIKey = Depends(get_api_key),
+):
+    all_deployments = Iem(credentials=None).get_all_deployments()
+    return list(all_deployments)
+
+
+@app.get(
+    "/deployments/{deployment_id}",
+    response_model=List[DeploymentResponse],
+    tags=["deployments"],
+)
+async def read_status_deployment(
+    deployment_id: str,
+    start: int = 0,
+    count: int = 1,
+    api_key: APIKey = Depends(get_api_key),
+):
+    deployment = Iem().get_deployment(deployment_id=deployment_id)
+    return list(deployment)
+
+
+@app.post(
+    "/deployments/",
+    status_code=status.HTTP_201_CREATED,
+    response_model=BaseResponse,
+    tags=["deployments"],
+)
+async def deploy(
+    d: DeploymentRequest,
+    background_tasks: BackgroundTasks,
+    api_key: APIKey = Depends(get_api_key),
+):
+    logging.warning(d)
+    i = Iem(credentials=d.credentials)
+    background_tasks.add_task(i.deploy, d.deployment_id, d.repository, d.commit)
+    return BaseResponse(message="Deployment Request Created")
+
+
+@app.post(
+    "/undeploy/",
+    status_code=status.HTTP_202_ACCEPTED,
+    response_model=BaseResponse,
+    tags=["deployments"],
+)
+async def undeploy(
+    d: DeleteDeploymentRequest,
+    background_tasks: BackgroundTasks,
+    api_key: APIKey = Depends(get_api_key),
+):
+    logging.warning(d)
+    i = Iem(credentials=d.credentials)
+    background_tasks.add_task(i.destroy, d.deployment_id)
+    return BaseResponse(message="Undeployment Request Created")
+
+
+if os.getenv("STAGE") == "dev":
+    with open("../openapi.json", "w") as f:
+        json.dump(
+            get_openapi(
+                title=app.title,
+                version=app.version,
+                openapi_version=app.openapi_version,
+                description=app.description,
+                routes=app.routes,
+            ),
+            f,
+        )