Skip to content
Snippets Groups Projects
Commit 1c700deb authored by Diaz de Arcaya Serrano, Josu's avatar Diaz de Arcaya Serrano, Josu
Browse files

updating main branch

parent 9872248e
Branches main
No related tags found
No related merge requests found
Show whitespace changes
Inline Side-by-side
Showing
with 769 additions and 181 deletions
iem-api/src/core/iem.py
+ 246
− 96
View file @ 1c700deb
import git
import base64
import binascii
import json
import logging
import os
import subprocess
from git import GitCommandError, InvalidGitRepositoryError
from omegaconf import OmegaConf
import shutil
from io import BytesIO
from subprocess import CalledProcessError
from zipfile import BadZipFile, ZipFile
from omegaconf import OmegaConf
from src.core.engine import Factory
from src.core.persistence import Sqlite
from src.core.utils import DeploymentResponse, Credentials
from src.core.persistence import Persistence
from src.core.utils import Credentials, DeploymentResponse
LOGGER = logging.getLogger("iem")
LOGGER = logging.getLogger(__name__)
class Iem:
def __init__(self, credentials: Credentials = None):
logging.basicConfig(
level=logging.INFO, format="%(asctime)s %(levelname)-8s %(message)s"
)
self._credentials = credentials
# if credentials.aws:
# self._aws_access_key_id = credentials.aws.access_key_id
# self._aws_secret_access_key = credentials.aws.secret_access_key
# if credentials.openstack:
# self._os_username = credentials.openstack.user_name
# self._os_password = credentials.openstack.password
# self._os_auth_url = credentials.openstack.auth_url
# self._os_project_name = credentials.openstack.project_name
# Check IEM_HOME variable
if os.getenv("IEM_HOME") is None:
LOGGER.error("Please define IEM_HOME environment variable.")
exit(-1)
self._iem_home = os.getenv("IEM_HOME")
self._path_deployments = f"{os.environ['IEM_HOME']}deployments/"
self._persistence = Sqlite()
self._persistence = Persistence()
def get_all_deployments(self):
rows = self._persistence.get_all_deployments()
for r in rows:
d = DeploymentResponse(
status_time=r[0],
deployment_id=r[1],
status=r[2],
stdout=r[3],
stderr=r[4],
status_time=r.status_time,
deployment_id=r.deployment_id,
status=r.status,
stdout=r.stdout,
stderr=r.stderr,
)
yield d
......@@ -56,67 +40,139 @@ class Iem:
row = self._persistence.get_deployment(deployment_id=deployment_id)
if row:
d = DeploymentResponse(
status_time=row[0],
deployment_id=row[1],
status=row[2],
stdout=row[3],
stderr=row[4],
status_time=row.status_time,
deployment_id=row.deployment_id,
status=row.status,
stdout=row.stdout,
stderr=row.stderr,
)
yield d
def deploy(self, deployment_id: str, repository: str, commit: str):
function_name = "deploy"
LOGGER.info(f"Running {function_name} method")
def get_deployment_outputs(self, deployment_id: str, stage_id: str):
function_name = "get_deployment_outputs"
LOGGER.info(f"Running {function_name} method.")
self._persistence.insert_deployment(
deployment_id=deployment_id, status="STARTED", stdout=None, stderr=None
repo_path = f"{self._path_deployments}{deployment_id}"
LOGGER.info(f"Reading credentials.")
my_env = self._get_env(
deployment_id=deployment_id, credentials=self._credentials
)
try:
repo_path = f"{self._iem_home}{deployment_id}"
self.get_repo(repo_path=repo_path, repository=repository, commit=commit)
LOGGER.info(f"About to read outputs for project {repo_path}.")
conf = OmegaConf.load(f"{repo_path}/{stage_id}/config.yaml")
self.validate(env=my_env, io=conf.input)
LOGGER.info(f"About to deploy project {repo_path}")
my_eng = Factory().get_engine(conf.engine)(
repo_path=f"{repo_path}/{stage_id}",
my_env=my_env,
)
output = my_eng.output()
return dict(json.loads(output))
def _get_env(self, deployment_id: str, credentials: Credentials) -> dict:
my_env = os.environ.copy()
if self._credentials.aws:
my_env["AWS_ACCESS_KEY_ID"] = self._credentials.aws.access_key_id
if credentials.aws:
my_env["AWS_ACCESS_KEY_ID"] = credentials.aws.access_key_id
my_env["AWS_SECRET_ACCESS_KEY"] = credentials.aws.secret_access_key
my_env["AWS_REGION"] = credentials.aws.region
if credentials.openstack:
my_env["OS_USERNAME"] = credentials.openstack.user_name
my_env["OS_PASSWORD"] = credentials.openstack.password
my_env["OS_AUTH_URL"] = credentials.openstack.auth_url
my_env["OS_PROJECT_NAME"] = credentials.openstack.project_name
if credentials.azure:
my_env["ARM_CLIENT_ID"] = credentials.azure.arm_client_id
my_env["ARM_CLIENT_SECRET"] = credentials.azure.arm_client_secret
my_env["ARM_SUBSCRIPTION_ID"] = credentials.azure.arm_subscription_id
my_env["ARM_TENANT_ID"] = credentials.azure.arm_tenant_id
if credentials.vmware:
my_env["VSPHERE_USER"] = credentials.vmware.user_name
my_env["VSPHERE_PASSWORD"] = credentials.vmware.password
my_env["VSPHERE_SERVER"] = credentials.vmware.server
my_env[
"AWS_SECRET_ACCESS_KEY"
] = self._credentials.aws.secret_access_key
if self._credentials.openstack:
my_env["OS_USERNAME"] = self._credentials.openstack.user_name
my_env["OS_PASSWORD"] = self._credentials.openstack.password
my_env["OS_AUTH_URL"] = self._credentials.openstack.auth_url
my_env["OS_PROJECT_NAME"] = self._credentials.openstack.project_name
"VSPHERE_ALLOW_UNVERIFIED_SSL"
] = credentials.vmware.allow_unverified_ssl
if credentials.custom:
for key, value in credentials.custom.items():
my_env[key] = value
project_conf = OmegaConf.load(f"{repo_path}/config.yaml")
for stage in project_conf.iac:
my_env["DEPLOYMENT_ID"] = deployment_id
return my_env
def _apply_stage(
self, my_env: dict, repo_path: str, stage: str, skip_inventory=False
):
conf = OmegaConf.load(f"{repo_path}/{stage}/config.yaml")
self.validate(env=my_env, io=conf.input)
self.validate(env=my_env, io=conf.input) if not skip_inventory else None
LOGGER.info(f"About to run stage: {stage}")
my_eng = Factory().get_engine(conf.engine)(
repo_path=f"{repo_path}/{stage}", my_env=my_env
repo_path=f"{repo_path}/{stage}",
my_env=my_env,
skip_inventory=skip_inventory,
)
status, stdout, stderr = my_eng.apply()
returncode, stdout, stderr = my_eng.apply()
if conf.output:
if returncode == 0 and conf.output:
output = my_eng.output()
self.update_env(output=output, env=my_env, io=conf.output)
self.validate(env=my_env, io=conf.output)
return returncode, stdout, stderr
def deploy(self, deployment_id: str, bundle: str) -> int:
function_name = "deploy"
LOGGER.info(f"Running {function_name} method.")
self._persistence.insert_deployment(
deployment_id=deployment_id, status="STARTED", stdout=None, stderr=None
)
repo_path = f"{self._path_deployments}{deployment_id}"
if not os.path.exists(repo_path):
os.makedirs(repo_path)
try:
LOGGER.info(f"Decompressing base64 bundle.")
bundle_decode = base64.b64decode(bundle)
ZipFile(BytesIO(bundle_decode)).extractall(repo_path)
LOGGER.info(f"Reading credentials.")
my_env = self._get_env(
deployment_id=deployment_id, credentials=self._credentials
)
LOGGER.info(f"About to deploy project {repo_path}.")
project_conf = OmegaConf.load(f"{repo_path}/config.yaml")
for stage in project_conf.iac:
returncode, stdout, stderr = self._apply_stage(
my_env=my_env, repo_path=repo_path, stage=stage
)
if returncode != 0:
LOGGER.error(f"Deployment failed for project {repo_path}.")
break
self._persistence.insert_deployment(
deployment_id=deployment_id,
status=status,
status="CREATED" if returncode == 0 else "FAILED",
stdout=stdout,
stderr=stderr,
)
LOGGER.info(f"Deployment completed for project {repo_path}")
return returncode
except (
binascii.Error,
BadZipFile,
CalledProcessError,
GitCommandError,
FileNotFoundError,
InvalidGitRepositoryError,
NameError,
KeyError,
) as e:
LOGGER.exception(e)
self._persistence.insert_deployment(
......@@ -124,23 +180,15 @@ class Iem:
)
raise e
LOGGER.info(f"The {function_name} method finished successfully")
def destroy(self, deployment_id: str):
function_name = "destroy"
LOGGER.info(f"Running {function_name} method")
repo_path = f"{self._iem_home}{deployment_id}"
repo_path = f"{self._path_deployments}{deployment_id}"
my_env = os.environ.copy()
if self._credentials.aws:
my_env["AWS_ACCESS_KEY_ID"] = self._credentials.aws.access_key_id
my_env["AWS_SECRET_ACCESS_KEY"] = self._credentials.aws.secret_access_key
if self._credentials.openstack:
my_env["OS_USERNAME"] = self._credentials.openstack.user_name
my_env["OS_PASSWORD"] = self._credentials.openstack.password
my_env["OS_AUTH_URL"] = self._credentials.openstack.auth_url
my_env["OS_PROJECT_NAME"] = self._credentials.openstack.project_name
my_env = self._get_env(
deployment_id=deployment_id, credentials=self._credentials
)
LOGGER.info(f"About to destroy project {repo_path}")
project_conf = OmegaConf.load(f"{repo_path}/config.yaml")
......@@ -151,11 +199,19 @@ class Iem:
my_eng = Factory().get_engine(conf.engine)(
repo_path=f"{repo_path}/{stage}", my_env=my_env
)
status, stdout, stderr = my_eng.destroy()
returncode, stdout, stderr = my_eng.destroy()
if returncode != 0:
LOGGER.error(f"Undeployment failed for project {repo_path}.")
break
if "/home" in repo_path:
LOGGER.exception(f"Cannot delete {repo_path}")
elif os.getenv("DOCKERIZED") == "true":
shutil.rmtree(repo_path)
self._persistence.insert_deployment(
deployment_id=deployment_id,
status=status,
status="DESTROYED" if returncode == 0 else "FAILED",
stdout=stdout,
stderr=stderr,
)
......@@ -174,20 +230,114 @@ class Iem:
LOGGER.error(f"Variable {variable} not found in environment.")
raise NameError
def get_repo(self, repo_path: str, repository: str, commit: str):
LOGGER.info(
f"About to download the project {repo_path} with repository {repository}"
def self_healing_strategy(self, deployment_id: str, playbook: str):
function_name = "self_healing_strategy"
LOGGER.info(f"Running {function_name} method")
self._persistence.insert_deployment(
deployment_id=deployment_id, status="SHS-STARTED", stdout=None, stderr=None
)
if not os.path.isdir(repo_path):
os.makedirs(repo_path)
repo = git.Repo.clone_from(
url=repository, to_path=repo_path, no_checkout=True
LOGGER.info(f"The {function_name} method finished successfully")
repo_path = f"{self._path_deployments}{deployment_id}"
if not os.path.exists(repo_path):
LOGGER.error(
f"The deployment_id = {deployment_id} does not correspond to any active deployment."
)
return
try:
LOGGER.info(f"Updating main.yml with new playbook.")
with open(f"{repo_path}/self_healing_monitoring/main.yml", "w") as f:
f.write(playbook)
LOGGER.info(f"Reading credentials.")
my_env = self._get_env(
deployment_id=deployment_id, credentials=self._credentials
)
returncode, stdout, stderr = self._apply_stage(
my_env=my_env,
repo_path=repo_path,
stage=f"self_healing_monitoring",
skip_inventory=True,
)
if returncode != 0:
LOGGER.error(f"Update failed for project {repo_path}.")
self._persistence.insert_deployment(
deployment_id=deployment_id,
status="UPDATED" if returncode == 0 else "FAILED",
stdout=stdout,
stderr=stderr,
)
LOGGER.info(f"Update completed for project {repo_path}")
except (
binascii.Error,
CalledProcessError,
FileNotFoundError,
NameError,
KeyError,
) as e:
LOGGER.exception(e)
self._persistence.insert_deployment(
deployment_id=deployment_id, status="ERROR", stdout=None, stderr=None
)
raise e
def self_healing_bundle(self, deployment_id: str, bundle: str) -> int:
function_name = "self_healing_bundle"
LOGGER.info(f"Running {function_name} method")
self._persistence.insert_deployment(
deployment_id=deployment_id, status="UPDATING", stdout=None, stderr=None
)
repo_path = f"{self._path_deployments}{deployment_id}"
if not os.path.exists(repo_path):
LOGGER.error(
f"The deployment_id = {deployment_id} does not correspond to any active deployment."
)
else:
repo = git.Repo(repo_path)
return -1
repo.git.checkout(commit)
try:
LOGGER.info(f"Decompressing base64 bundle.")
bundle_decode = base64.b64decode(bundle)
ZipFile(BytesIO(bundle_decode)).extractall(repo_path)
LOGGER.info(f"Reading credentials.")
my_env = self._get_env(
deployment_id=deployment_id, credentials=self._credentials
)
for submodule in repo.submodules:
submodule.update(init=True)
submodule.module().git.checkout()
LOGGER.info(f"About to update project {repo_path}.")
project_conf = OmegaConf.load(f"{repo_path}/config.yaml")
for stage in project_conf.iac:
returncode, stdout, stderr = self._apply_stage(
my_env=my_env, repo_path=repo_path, stage=stage, skip_inventory=True
)
if returncode != 0:
LOGGER.error(f"Update failed for project {repo_path}.")
break
self._persistence.insert_deployment(
deployment_id=deployment_id,
status="UPDATED" if returncode == 0 else "FAILED",
stdout=stdout,
stderr=stderr,
)
LOGGER.info(f"Update completed for project {repo_path}")
return returncode
except (
binascii.Error,
BadZipFile,
CalledProcessError,
FileNotFoundError,
NameError,
KeyError,
) as e:
LOGGER.exception(e)
self._persistence.insert_deployment(
deployment_id=deployment_id, status="ERROR", stdout=None, stderr=None
)
raise e
iem-api/src/core/persistence.py
+ 8
− 78
View file @ 1c700deb
import logging
import os
import sqlite3
# from dataclasses import dataclass
from datetime import datetime
from ratelimiter import RateLimiter
from sqlalchemy import (Column, DateTime, Integer, String, create_engine, desc,
func, select)
from sqlalchemy.orm import Session, declarative_base
from sqlalchemy import create_engine
from sqlalchemy import Column
from sqlalchemy import DateTime, Integer, String
from sqlalchemy import select, desc
from sqlalchemy.orm import declarative_base, Session
LOGGER = logging.getLogger(__name__)
Base = declarative_base()
class Sqlite:
def __init__(self):
self._db_name = "iem.db"
# create database if it does not exist
if not os.path.isfile(self._db_name):
self.__create_database()
def __create_database(self):
conn = sqlite3.connect(self._db_name)
with conn:
sql = """CREATE TABLE deployments (id INTEGER PRIMARY KEY AUTOINCREMENT,
status_time DATETIME DEFAULT CURRENT_TIMESTAMP,
deployment_id TEXT NOT NULL, status TEXT NOT NULL, stdout TEXT, stderr TEXT);"""
conn.execute(sql)
def insert_deployment(
self, deployment_id: str, status: str, stdout: str, stderr: str
):
conn = sqlite3.connect(self._db_name)
with conn:
sql = """INSERT INTO deployments (deployment_id, status, stdout, stderr) VALUES (?,?,?,?)"""
cursor = conn.cursor()
cursor.execute(
sql,
(
deployment_id,
status,
stdout,
stderr,
),
)
conn.commit()
def get_deployment(self, deployment_id: str):
conn = sqlite3.connect(self._db_name)
with conn:
sql = """SELECT status_time, deployment_id, status, stdout, stderr FROM deployments
WHERE deployment_id=? ORDER BY id DESC LIMIT 1"""
cursor = conn.cursor()
cursor.execute(sql, (deployment_id,))
row = cursor.fetchone()
return row
def get_all_deployments(self):
conn = sqlite3.connect(self._db_name)
with conn:
sql = """SELECT status_time, deployment_id, status, stdout, stderr
FROM deployments ORDER BY id DESC LIMIT 25 OFFSET 0"""
cursor = conn.cursor()
cursor.execute(sql)
rows = cursor.fetchall()
return rows
@RateLimiter(max_calls=10, period=1)
def valid_api_key(self, api_key_query: str):
if api_key_query == os.getenv("API_KEY"):
return True
else:
return False
class Deployment(Base):
__tablename__ = "deployments"
id = Column(Integer, primary_key=True)
status_time = Column(DateTime, default=datetime.now())
status_time = Column(DateTime, default=func.now())
deployment_id = Column(String, nullable=False)
status = Column(String, nullable=False)
stdout = Column(String)
......@@ -96,8 +26,8 @@ class Deployment(Base):
class Persistence:
def __init__(self):
self._engine = create_engine("sqlite:///:memory:", future=True)
def __init__(self, engine_url="sqlite:///db/iem.db"):
self._engine = create_engine(url=engine_url, future=True)
Base.metadata.create_all(self._engine)
def insert_deployment(
......
iem-api/src/core/utils.py
+ 32
− 7
View file @ 1c700deb
import logging
from datetime import datetime
from pydantic import BaseModel
from typing import Dict, Optional
from typing import Any, Dict, Optional
LOGGER = logging.getLogger("iem")
from pydantic import BaseModel
class BaseResponse(BaseModel):
......@@ -22,6 +19,7 @@ class DeploymentResponse(BaseModel):
class Aws(BaseModel):
access_key_id: str
secret_access_key: str
region: Optional[str] = "us-west-2"
class Azure(BaseModel):
......@@ -42,19 +40,46 @@ class Openstack(BaseModel):
user_domain_name: Optional[str]
class Docker(BaseModel):
server: str
user_name: str
password: str
class Vmware(BaseModel):
user_name: str
password: str
server: str
allow_unverified_ssl: Optional[str]
class Credentials(BaseModel):
aws: Optional[Aws] = None
azure: Optional[Azure] = None
openstack: Optional[Openstack] = None
vmware: Optional[Vmware] = None
docker: Optional[Docker] = None
custom: Optional[dict[str, Any]] = None
class Bundle(BaseModel):
base64: str
class DeploymentRequest(BaseModel):
deployment_id: str
repository: str
commit: str
credentials: Credentials
bundle: Bundle
class SelfHealingRequest(BaseModel):
credentials: Credentials
playbook: str
class DeleteDeploymentRequest(BaseModel):
deployment_id: str
credentials: Credentials
class DeploymentStatusRequest(BaseModel):
credentials: Credentials
iem-api/src/resources/id_iem 0 → 100644
+ 38
− 0
View file @ 1c700deb
-----BEGIN OPENSSH PRIVATE KEY-----
b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAABlwAAAAdzc2gtcn
NhAAAAAwEAAQAAAYEA1d1XmDw9VikKatj87TVENCAJ5uCQpgjyaQi3HpvZDtw7zW1QTeUN
CRdJ36oHPG9cD+7gR9W9xvI2l+2bKugqTbuDr1ysPxAcTt6XYgMer9JkWTCwcdqnAXdUwf
dLMUPo7mOv/9OnOdjoEo0NwnsvGvPxinbUzMUSofqwLgHIdlo3GQZk2T3vvRN3zoYmx4OV
AN7C4RXAD7hLhEEW0wwUrAqnQqh13yHuDYi6iWFSzh5B7h3700Wi0Toe85n9tBZiD+7NYG
FNxqQykYGXrqNmTPe/dTffAdCYEp3qW8dvzA7htlrwBcByCu1kjVcRrsXb17KVFnwxmpvx
uGRrsc2sVR7cGIEssTaAFF3L/NhkvzDS16qX2qbz0fyxnFgBHINRYM45AvMuDAWerUA6MA
GK/wRJ1CO8ZrSJtIsQVnXdsxrcjKEm76hzdA0rETPmA5A+FzHH3fSqfZIiFh5dTfUUnldw
9STXwxjRgBNasptuTYtM1jW0shFMaS5F4+E9sBAzAAAFkHZrcNl2a3DZAAAAB3NzaC1yc2
EAAAGBANXdV5g8PVYpCmrY/O01RDQgCebgkKYI8mkItx6b2Q7cO81tUE3lDQkXSd+qBzxv
XA/u4EfVvcbyNpftmyroKk27g69crD8QHE7el2IDHq/SZFkwsHHapwF3VMH3SzFD6O5jr/
/TpznY6BKNDcJ7Lxrz8Yp21MzFEqH6sC4ByHZaNxkGZNk9770Td86GJseDlQDewuEVwA+4
S4RBFtMMFKwKp0Kodd8h7g2IuolhUs4eQe4d+9NFotE6HvOZ/bQWYg/uzWBhTcakMpGBl6
6jZkz3v3U33wHQmBKd6lvHb8wO4bZa8AXAcgrtZI1XEa7F29eylRZ8MZqb8bhka7HNrFUe
3BiBLLE2gBRdy/zYZL8w0teql9qm89H8sZxYARyDUWDOOQLzLgwFnq1AOjABiv8ESdQjvG
a0ibSLEFZ13bMa3IyhJu+oc3QNKxEz5gOQPhcxx930qn2SIhYeXU31FJ5XcPUk18MY0YAT
WrKbbk2LTNY1tLIRTGkuRePhPbAQMwAAAAMBAAEAAAGAB5BynrHSwY9mDO1r1MADj4xqjT
34H8dFO63RPEXq4Xmsq9Fn+7lUQrQOKtkKtHqD2RRr3l6S/cxnXexLhrL7fBBb0gIHHZvm
RGvfEtplZXadkgIE26IOMiEUYF/syutJ+9SOzw+fZI5ldvKCQBS3T869Bla5pBx8UjpZrO
bnPjhmpn3xZzWnmxprLGTWTkw7IvK+FdP9HRE5qo3aztAokwU1cUggEypSDyx83IsSsLOl
RVTOKWTXI2tY2OjjblE0SiFimH7btHH2VhdAWEtOujs04cKU1mQ7pPnZS0uK5PfkC89I7v
zeslnsMsZ73jtBMtCWnNwh3Om7saP4GK0G58pQqyC4SAN/4Clf6PGB+/dtQsJG1XDQ+Idc
IjlSly43tX6Q3t+M7VtfGbLYy/vUF01r2PaG5G+BUucH11DQrGxw/gMQhpqHaXu5C7PpxG
4yBxF58BaMqRPPTJWgTFuMw1Ib2Ogda5dRuC6Qi2hJ3T2S2IkFXDwGWabRGIpqKXMhAAAA
wChxJsE8SV5uxbYtgPKGFcHvZLqsX7Hg3q9GXDjqCiFmJCl2GpFnqacxqfhTlTGShHBB6q
O71zUOktX/Dbje7VTTBKhHLrpqsp7JXWao7v9w007LxhKAR3XEencoynmWTL9lGBvbQ8mX
X5Q4zqMt+1GqH5ae4BuT4BtlqD+R9WO4ZvnsEKUuCFXbndZlWsMaRD7J/PQXM/vrsDg0yF
9dHUH8INayw4utgQ/FwmdkFx6YAxnwqgcoYX0KzoX3FJhN+QAAAMEA+fiPdeB4tVLGLQyF
k/4SDjE1pyba0XUwl0hoc8IWILaiwmncn1uRLCbRSYZp+gikFheFY7zIzbdnZXPv4vpX8a
0ydCdksE9iwPxuhzMb5EbyoWCffUuDmiWaVu5H3Q9rveJwFPB5HJXmT4S0ZTS5JZHZi6nb
h55CTKFAoCd9pUBEZUATWUBfml+WurF72C9VzxiSqyd9S8XVwE2Oq72sxyI3S4VUGKvJ+t
wZZFU1BE7qoJWyV0RiV3eiez9Mum0rAAAAwQDbBdgafpny35V4Yc5YFr+k1FSv3Xw9fduH
+uqyskdHpB3anUe366ZsSpnQcLjbDFsW45K9tZxe8GacTwyiTfbdrT/m5zWzLV3cMBnVyv
Pf1oYSjrV4H1BDShikc5LCWGje+FWxONgtB2oOcUCQo7pPq/YYD984DsurBY9l1VWJM+T4
U1KTMp9x9b+xoINP0+9uXZHLmL7LsxOGhygXhUCmE4XBCmJYZdEEodwlUco8RBraapFYqN
inyHJxviLktRkAAAAUdmFncmFudEB1YnVudHUtZm9jYWwBAgMEBQYH
-----END OPENSSH PRIVATE KEY-----
iem-api/src/resources/id_iem.pub 0 → 100644
+ 1
− 0
View file @ 1c700deb
ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDV3VeYPD1WKQpq2PztNUQ0IAnm4JCmCPJpCLcem9kO3DvNbVBN5Q0JF0nfqgc8b1wP7uBH1b3G8jaX7Zsq6CpNu4OvXKw/EBxO3pdiAx6v0mRZMLBx2qcBd1TB90sxQ+juY6//06c52OgSjQ3Cey8a8/GKdtTMxRKh+rAuAch2WjcZBmTZPe+9E3fOhibHg5UA3sLhFcAPuEuEQRbTDBSsCqdCqHXfIe4NiLqJYVLOHkHuHfvTRaLROh7zmf20FmIP7s1gYU3GpDKRgZeuo2ZM9791N98B0JgSnepbx2/MDuG2WvAFwHIK7WSNVxGuxdvXspUWfDGam/G4ZGuxzaxVHtwYgSyxNoAUXcv82GS/MNLXqpfapvPR/LGcWAEcg1FgzjkC8y4MBZ6tQDowAYr/BEnUI7xmtIm0ixBWdd2zGtyMoSbvqHN0DSsRM+YDkD4XMcfd9Kp9kiIWHl1N9RSeV3D1JNfDGNGAE1qym25Ni0zWNbSyEUxpLkXj4T2wEDM=
iem-api/tests/__init__.py 0 → 100644
+ 0
− 0
View file @ 1c700deb
iem-api/tests/it/__init__.py 0 → 100644
+ 0
− 0
View file @ 1c700deb
iem-api/tests/it/test_it_iem.py 0 → 100644
+ 149
− 0
View file @ 1c700deb
import base64
import logging
import os
import unittest
import uuid
from src.core.iem import Iem
from src.core.utils import Aws, Credentials, Openstack
LOGGER = logging.getLogger(__name__)
class TestIem(unittest.TestCase):
def __init__(self, *args, **kwargs):
super(TestIem, self).__init__(*args, **kwargs)
# Check IEM_HOME variable
self._iem_home = os.environ["IEM_HOME"]
@unittest.skipUnless(os.getenv("AWS"), "Define AWS variable to execute")
def test_deploy_destroy_aws(self):
deployment_id = str(uuid.uuid4())
a = Iem(
Credentials(
aws=Aws(
access_key_id=os.environ["AWS_ACCESS_KEY_ID"],
secret_access_key=os.environ["AWS_SECRET_ACCESS_KEY"],
)
)
)
with open("tests/resources/aws.zip", "rb") as binary_file:
bundle = base64.b64encode(binary_file.read())
a.deploy(deployment_id=deployment_id, bundle=bundle)
a.destroy(deployment_id=deployment_id)
@unittest.skipUnless(os.getenv("OS"), "Define OS variable to execute")
def test_deploy_destroy_openstack(self):
deployment_id = str(uuid.uuid4())
a = Iem(
Credentials(
openstack=Openstack(
user_name=os.environ["OS_USERNAME"],
password=os.environ["OS_PASSWORD"],
auth_url=os.environ["OS_AUTH_URL"],
project_name=os.environ["OS_PROJECT_NAME"],
)
)
)
with open("tests/resources/openstack.zip", "rb") as binary_file:
bundle = base64.b64encode(binary_file.read())
output = a.deploy(deployment_id=deployment_id, bundle=bundle)
a.destroy(deployment_id=deployment_id)
self.assertEqual(output, 0)
@unittest.skipUnless(os.getenv("ANSIBLE"), "Define ANSIBLE variable to execute")
def test_deploy_destroy_ansible(self):
deployment_id = str(uuid.uuid4())
a = Iem(
Credentials(
openstack=Openstack(
user_name=os.environ["OS_USERNAME"],
password=os.environ["OS_PASSWORD"],
auth_url=os.environ["OS_AUTH_URL"],
project_name=os.environ["OS_PROJECT_NAME"],
)
)
)
with open("tests/resources/ansible.zip", "rb") as binary_file:
bundle = base64.b64encode(binary_file.read())
output = a.deploy(deployment_id=deployment_id, bundle=bundle)
a.destroy(deployment_id=deployment_id)
self.assertEqual(output, 0)
@unittest.skipUnless(
os.getenv("INTEGRATION"), "Define INTEGRATION variable to execute"
)
def test_deploy_destroy_docker(self):
deployment_id = str(uuid.uuid4())
a = Iem(
Credentials(
openstack=Openstack(
user_name=os.environ["OS_USERNAME"],
password=os.environ["OS_PASSWORD"],
auth_url=os.environ["OS_AUTH_URL"],
project_name=os.environ["OS_PROJECT_NAME"],
)
)
)
with open("tests/resources/docker.zip", "rb") as binary_file:
bundle = base64.b64encode(binary_file.read())
a.deploy(deployment_id=deployment_id, bundle=bundle)
a.destroy(deployment_id=deployment_id)
@unittest.skipUnless(os.getenv("SHS"), "Define SHS variable to execute")
def test_self_healing_strategy(self):
deployment_id = str(uuid.uuid4())
a = Iem(
Credentials(
custom={
"instance_usr": "vagrant",
"instance_pwd": "vagrant",
"instance_ip": "192.168.56.201",
}
)
)
with open("tests/resources/shs.zip", "rb") as binary_file:
bundle = base64.b64encode(binary_file.read())
a.deploy(deployment_id=deployment_id, bundle=bundle)
a.self_healing_strategy(deployment_id=deployment_id, strategy="25")
@unittest.skipUnless(os.getenv("SHS"), "Define SHS variable to execute")
def test_self_healing_bundle(self):
deployment_id = str(uuid.uuid4())
a = Iem(
Credentials(
aws=Aws(
access_key_id=os.environ["AWS_ACCESS_KEY_ID"],
secret_access_key=os.environ["AWS_SECRET_ACCESS_KEY"],
)
)
)
with open("tests/resources/shs.zip", "rb") as binary_file:
bundle = base64.b64encode(binary_file.read())
returncode = a.deploy(deployment_id=deployment_id, bundle=bundle)
self.assertEqual(returncode, 0)
with open("tests/resources/shs-bundle.zip", "rb") as binary_file:
bundle = base64.b64encode(binary_file.read())
returncode = a.self_healing_bundle(deployment_id=deployment_id, bundle=bundle)
self.assertEqual(returncode, 0)
a.destroy(deployment_id=deployment_id)
iem-api/tests/resources/ansible.zip 0 → 100644
+ 0
− 0
View file @ 1c700deb
File added
iem-api/tests/resources/aws.zip 0 → 100644
+ 0
− 0
View file @ 1c700deb
File added
iem-api/tests/resources/docker.zip 0 → 100644
+ 0
− 0
View file @ 1c700deb
File added
iem-api/tests/resources/dummy.zip 0 → 100644
+ 0
− 0
View file @ 1c700deb
File added
iem-api/tests/resources/main.yml 0 → 100644
+ 8
− 0
View file @ 1c700deb
---
- hosts: all
become: yes
tasks:
- name: install nmap
apt:
name: nmap
state: latest
iem-api/tests/resources/openstack.zip 0 → 100644
+ 0
− 0
View file @ 1c700deb
File added
iem-api/tests/resources/shs-bundle.zip 0 → 100644
+ 0
− 0
View file @ 1c700deb
File added
iem-api/tests/resources/shs.zip 0 → 100644
+ 0
− 0
View file @ 1c700deb
File added
iem-api/tests/unit/__init__.py 0 → 100644
+ 0
− 0
View file @ 1c700deb
iem-api/tests/unit/test_iem.py 0 → 100644
+ 31
− 0
View file @ 1c700deb
import logging
import os
import unittest
import uuid
from src.core.iem import Iem
from src.core.utils import Aws, Credentials, Openstack
LOGGER = logging.getLogger(__name__)
class TestIem(unittest.TestCase):
def __init__(self, *args, **kwargs):
super(TestIem, self).__init__(*args, **kwargs)
# Check IEM_HOME variable
self._iem_home = os.environ["IEM_HOME"]
def test_get_all_deployments(self):
i = Iem()
all_deployments = i.get_all_deployments()
self.assertGreaterEqual(len(list(all_deployments)), 0)
def test_get_deployment_ok(self):
i = Iem()
i.get_deployment(deployment_id="deployment2")
def test_get_deployment_not_ok(self):
i = Iem()
deployment_id = str(uuid.uuid4())
i.get_deployment(deployment_id=deployment_id)
iem-api/tests/unit/test_main.py 0 → 100644
+ 224
− 0
View file @ 1c700deb
import base64
import os
import unittest
import uuid
from unittest.mock import Mock, patch
from fastapi.testclient import TestClient
from main import app
class TestMain(unittest.TestCase):
def __init__(self, *args, **kwargs):
super(TestMain, self).__init__(*args, **kwargs)
self.client = TestClient(app)
# Check API_KEY variable
self._api_key = os.environ["API_KEY"]
def test_root_no_apikey(self):
response = self.client.get("/")
assert response.status_code == 403
def test_root(self):
response = self.client.get("/", headers={"x-api-key": self._api_key})
assert response.status_code == 200
def test_get_all_deployments(self):
response = self.client.get(
f"/deployments/", headers={"x-api-key": self._api_key}
)
assert response.status_code == 200
def test_get_one_deployment(self):
response = self.client.get(
f"/deployments/25/", headers={"x-api-key": self._api_key}
)
assert response.status_code == 200
@patch("subprocess.run")
def test_deployments(self, mock_run):
mock_run.return_value = Mock(returncode=0, stdout=b"{}", stderr=b"{}")
with open("tests/resources/dummy.zip", "rb") as binary_file:
bundle = base64.b64encode(binary_file.read())
response = self.client.post(
f"/deployments/",
headers={"x-api-key": self._api_key},
json={
"deployment_id": str(uuid.uuid4()),
"credentials": {
"openstack": {
"user_name": "string",
"password": "string",
"auth_url": "string",
"project_name": "string",
"region_name": "string",
"domain_name": "string",
"project_domain_name": "string",
"user_domain_name": "string",
}
},
"bundle": {"base64": bundle.decode("utf-8")},
},
)
assert response.status_code == 201
@patch("subprocess.run")
def test_undeploy(self, mock_run):
mock_run.return_value = Mock(returncode=0, stdout=b"{}", stderr=b"{}")
deployment_id = str(uuid.uuid4())
with open("tests/resources/dummy.zip", "rb") as binary_file:
bundle = base64.b64encode(binary_file.read())
response = self.client.post(
f"/deployments/",
headers={"x-api-key": self._api_key},
json={
"deployment_id": deployment_id,
"credentials": {
"openstack": {
"user_name": "string",
"password": "string",
"auth_url": "string",
"project_name": "string",
"region_name": "string",
"domain_name": "string",
"project_domain_name": "string",
"user_domain_name": "string",
}
},
"bundle": {"base64": bundle.decode("utf-8")},
},
)
assert response.status_code == 201
response = self.client.post(
f"/undeploy/",
headers={"x-api-key": self._api_key},
json={
"deployment_id": deployment_id,
"credentials": {},
},
)
assert response.status_code == 202
@patch("subprocess.run")
def test_custom(self, mock_run):
mock_run.return_value = Mock(returncode=0, stdout=b"{}", stderr=b"{}")
with open("tests/resources/dummy.zip", "rb") as binary_file:
bundle = base64.b64encode(binary_file.read())
response = self.client.post(
f"/deployments/",
headers={"x-api-key": self._api_key},
json={
"deployment_id": str(uuid.uuid4()),
"credentials": {
"openstack": {
"user_name": "string",
"password": "string",
"auth_url": "string",
"project_name": "string",
"region_name": "string",
"domain_name": "string",
"project_domain_name": "string",
"user_domain_name": "string",
},
"custom": {
"CUSTOM_VAR1": "string",
"CUSTOM_VAR2": "string",
"CUSTOM_VAR3": "string",
},
},
"bundle": {"base64": bundle.decode("utf-8")},
},
)
assert response.status_code == 201
@patch("subprocess.run")
def test_self_healing_strategy(self, mock_run):
mock_run.return_value = Mock(returncode=0, stdout=b"{}", stderr=b"{}")
deployment_id = str(uuid.uuid4())
with open("tests/resources/shs.zip", "rb") as binary_file:
bundle = base64.b64encode(binary_file.read())
response = self.client.post(
f"/deployments/",
headers={"x-api-key": self._api_key},
json={
"deployment_id": deployment_id,
"credentials": {},
"bundle": {"base64": bundle.decode("utf-8")},
},
)
assert response.status_code == 201
response = self.client.post(
f"/self-healing/{25}",
headers={"x-api-key": self._api_key},
json={
"deployment_id": deployment_id,
"credentials": {
"custom": {
"instance_usr": "vagrant",
"instance_pwd": "vagrant",
"instance_ip": "192.168.56.201",
}
},
},
)
assert response.status_code == 201
@patch("subprocess.run")
def test_self_healing_bundle(self, mock_run):
mock_run.return_value = Mock(returncode=0, stdout=b"{}", stderr=b"{}")
deployment_id = str(uuid.uuid4())
with open("tests/resources/dummy.zip", "rb") as binary_file:
bundle = base64.b64encode(binary_file.read())
response = self.client.post(
f"/deployments/",
headers={"x-api-key": self._api_key},
json={
"deployment_id": deployment_id,
"credentials": {
"openstack": {
"user_name": "string",
"password": "string",
"auth_url": "string",
"project_name": "string",
"region_name": "string",
"domain_name": "string",
"project_domain_name": "string",
"user_domain_name": "string",
}
},
"bundle": {"base64": bundle.decode("utf-8")},
},
)
assert response.status_code == 201
response = self.client.post(
f"/update-iac-bundle/",
headers={"x-api-key": self._api_key},
json={
"deployment_id": deployment_id,
"credentials": {
"openstack": {
"user_name": "string",
"password": "string",
"auth_url": "string",
"project_name": "string",
"region_name": "string",
"domain_name": "string",
"project_domain_name": "string",
"user_domain_name": "string",
}
},
"bundle": {"base64": bundle.decode("utf-8")},
},
)
assert response.status_code == 201
iem-api/tests/unit/test_persistence.py 0 → 100644
+ 32
− 0
View file @ 1c700deb
import unittest
import uuid
from src.core.persistence import Persistence
class TestPersistence(unittest.TestCase):
@classmethod
def setUpClass(self):
self.engine_url = "sqlite:///:memory:"
def test_insert_deployment(self):
Persistence(engine_url=self.engine_url).insert_deployment(
"deployment1", "STARTED", "stdout", "stderr"
)
def test_get_deployment(self):
p = Persistence(engine_url=self.engine_url)
p.insert_deployment("deployment2", "STOPPED", "stdout", "stderr")
p.insert_deployment("deployment2", "STARTED", "stdout", "stderr")
d = p.get_deployment("deployment2")
self.assertEqual(d.status, "STARTED")
def test_get_deployment_not_ok(self):
row = Persistence(engine_url=self.engine_url).get_deployment(str(uuid.uuid4()))
self.assertIsNone(row)
def test_get_all_deployments(self):
persistence = Persistence(engine_url=self.engine_url)
persistence.insert_deployment("deployment3", "STARTED", "stdout", "stderr")
rows = persistence.get_all_deployments()
self.assertGreaterEqual(len(list(rows)), 1)
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment