Skip to content

D
DECIDE_Components
Commit fbcd9dc8 authored by Escalante Martinez, Marisa's avatar Escalante Martinez, Marisa
Browse files
Options

Merge branch 'master' of https://git.code.tecnalia.com/DECIDE_Public/DECIDE_Components

parents 7facf9de 3b986164
Hide whitespace changes
Inline Side-by-side
Showing with 0 additions and 54283 deletions

Too many changes to show.
Plain diff Email patch

To preserve performance only 517 of 517+ files are displayed.

ADAPT/DeploymentOrchestrator/adapt-do/Dockerfile deleted 100644 → 0
View file @ 7facf9de
FROM tiangolo/uwsgi-nginx-flask:python3.6
RUN apt-get update && apt-get install unzip && apt-get install -y vim && wget https://releases.hashicorp.com/terraform/0.10.7/terraform_0.10.7_linux_amd64.zip?_ga=2.121414664.102068769.1507033863-2054770415.1501495729 -O temp.zip && unzip temp.zip -d /usr/local/bin && rm temp.zip && mkdir -p /app/repo && mkdir -p /home/ubuntu/terraform/certs && mkdir -p /home/ubuntu/terraform/scripts && mkdir /home/ubuntu/terraform/keypairs && pip install flask-restplus && pip install -U flask-cors && pip install pymongo==3.7 && pip install jsonschema==3.0.0a3 && wget http://security.debian.org/debian-security/pool/updates/main/o/openssl/libssl1.0.0_1.0.1t-1+deb8u9_amd64.deb && dpkg -i libssl1.0.0_1.0.1t-1+deb8u9_amd64.deb && wget https://fastdl.mongodb.org/linux/mongodb-linux-x86_64-ubuntu1604-4.0.0.tgz && tar -zxvf mongodb-linux-*-4.0.0.tgz && mkdir -p /data/db && export PATH=mongodb-linux-x86_64-ubuntu1604-4.0.0/bin:$PATH && cp mongodb-linux-x86_64-ubuntu1604-4.0.0/bin/* /usr/local/bin
COPY app/ /app/
COPY tfplugin/terraform-provider-cloudbroker /usr/local/bin
COPY scripts /home/ubuntu/terraform/scripts
COPY mongo.conf /etc/supervisor/conf.d
ENV STATIC_INDEX 1
ADAPT/DeploymentOrchestrator/adapt-do/app/clean-mongodb.js deleted 100644 → 0
View file @ 7facf9de
conn = new Mongo();
print("Getting db 'adapt-do-db'")
db = conn.getDB("adapt-do-db");
print("Removing vm list");
db.vmList.remove({})
print("Removing vm status");
db.vmStatus.remove({})
\ No newline at end of file
ADAPT/DeploymentOrchestrator/adapt-do/app/cleanup.sh deleted 100644 → 0
View file @ 7facf9de
#!/bin/bash
rm -r ExampleApp*
rm -r vm*
rm -r operations*
mongo clean-mongodb.js
ADAPT/DeploymentOrchestrator/adapt-do/app/main.py deleted 100644 → 0
View file @ 7facf9de
import werkzeug, os, sys
import subprocess
import json
import uuid
import time
import urllib.parse
import http.client
from jinja2 import Environment, FileSystemLoader
from flask import Flask, send_file, make_response, request, copy_current_request_context
from threading import Thread
from flask_restplus import Resource, Api, fields, reqparse
from stat import *
from flask_cors import CORS
from jsonschema import validate
from jsonschema import Draft7Validator
from pymongo import MongoClient
import bson.objectid
from bson.json_util import dumps
app = Flask(__name__)
CORS(app)
app.config.SWAGGER_UI_JSONEDITOR = True
app.config.SWAGGER_UI_DOC_EXPANSION = 'list'
api = Api(app, doc='/doc/')
REGISTRY_CERTS_UPLOAD_FOLDER = '/home/ubuntu/terraform/scripts/certs/registry'
parser = reqparse.RequestParser()
parser.add_argument('file', type=werkzeug.datastructures.FileStorage, location='files', required=True)
env_fields = api.model('EnvModel', {
'cloudbroker_endpoint': fields.String(description='The Cloudbroker REST Api endpoint', required=True),
'cloudbroker_username': fields.String(description='The username', required=True),
'cloudbroker_password': fields.String(description='The password', required=True),
'repository_user': fields.String(description='The Git repo username', required=True),
'repository_pwd': fields.String(description='The Git repo password', required=True),
'repository_url': fields.String(description='The Git repo http(s) url', required=True),
'revision': fields.String(description='The code revision for the configuration', required=False),
'filepath': fields.String(description='The filepath for the configuration file in the repo', required=True),
'monitoring_host': fields.String(description='The ADAPT Monitoring REST Api endpoint url', required=True),
'monitoring_port': fields.String(description='The ADAPT Monitoring REST Api endpoint port', required=True),
})
schemaFile = open('schemas/application_description.schema.json', 'r')
schemaContent = schemaFile.read()
appDescriptionSchema = json.loads(schemaContent)
@app.route("/")
def main():
index_path = os.path.join(app.static_folder, 'index.html')
return send_file(index_path)
@api.route("/schema")
class DumpSchema(Resource):
def get(self):
print(json.dumps(api.__schema__, indent=4, sort_keys=False))
return json.dumps(api.__schema__)
@api.route("/certs/registry/<registryIp>")
@api.doc(params={'registryIp': 'The ip of the docker registry for which we want to upload a certificate'})
class RegistryCertsUpload(Resource):
decorators=[]
def post(self, registryIp):
print("Got request to create certs for ip " + registryIp)
respData = {}
data = parser.parse_args()
if data['file'] == "":
respData["content"] = "No file found"
respData["resultCode"] = "400"
respData["result"] = "error"
response = make_response(json.dumps(respData))
response.status_code = 400
return response
cert = data['file']
if cert:
filename = 'ca.crt'
if uploadRegistryCerts(cert, registryIp, filename):
respData["resultCode"] = "200"
respData["result"] = "success"
response = make_response(json.dumps(respData))
response.status_code = 200
return response
respData["content"] = "Something went wrong"
respData["resultCode"] = "400"
respData["result"] = "error"
response = make_response(json.dumps(respData))
response.status_code = 400
return response
def uploadRegistryCerts(cert, registryIp, filename):
try:
destinationFolder = os.path.join(REGISTRY_CERTS_UPLOAD_FOLDER,registryIp)
if not os.path.exists(destinationFolder):
os.makedirs(destinationFolder)
print("Saving cert to " + destinationFolder)
cert.save(os.path.join(destinationFolder,filename))
return True
except:
print("Unexpected error:", sys.exc_info()[0])
return False
@api.route("/terraform/<environment>")
@api.doc(params={'environment': 'The environment [infrastructure | services] for which we want to create the configuration files'})
class Configuration(Resource):
@api.expect(env_fields)
def post(self, environment):
data = {}
body = request.get_json()
print(body)
repo_user = body["repository_user"]
repo_pwd = body["repository_pwd"]
repo_url = body["repository_url"]
repo_url = repo_url[8:]
repo_url = "https://"+urllib.parse.quote(repo_user)+":"+urllib.parse.quote(repo_pwd)+"@"+repo_url
if not "repository_url" in body or not "filepath" in body or not body["repository_user"] or not body["repository_pwd"]:
data["result"] = "error"
data["resultCode"] = "400"
data["content"] = "Missing parameters"
response = make_response(json.dumps(data))
response.status_code = 400
return response
repository = body["repository_url"].split('/')[-1]
repository = repository[:repository.index('.git')]
filePath = body["filepath"]
if not os.path.exists("repo/"+repository):
print("Cloning repository: " + body["repository_url"])
if "branch" in body:
branch = body["branch"]
result = subprocess.check_output(['git', 'clone', repo_url, '-b', branch], cwd="repo/")
else:
result = subprocess.check_output(['git', 'clone', repo_url], cwd="repo/")
print("Pulling repository: " + body["repository_url"])
result = subprocess.check_output(['git', 'pull'], cwd="repo/"+repository)
if "revision" in body:
revision = body["revision"]
else:
revision = "HEAD"
print("Getting revision " + revision + " for file " + filePath + " from repository " + repository + " (" + body["repository_url"] + ")")
result = subprocess.check_output(['git', 'checkout', '-m', revision, filePath], cwd="repo/"+repository)
file = open('repo/'+repository+"/"+filePath, 'r')
content = file.read()
print("Creating config for " + environment + " with input file:")
print(json.loads(content))
jsonContent = json.loads(content)
try:
jsonContent = json.loads(content)
print(">>>>>>>>>>>>>>>>Validating content against schema")
#validate(jsonContent, appDescriptionSchema)
Draft7Validator(appDescriptionSchema).is_valid(jsonContent)
except:
print("*************Validation error!****************", sys.exc_info()[0], sys.exc_info()[1])
if "cloudbroker_username" in body and "cloudbroker_password" in body and "cloudbroker_endpoint" in body and body["cloudbroker_username"] and body["cloudbroker_password"] and body["cloudbroker_endpoint"]:
jsonContent["cloudbrokerEndpoint"] = body["cloudbroker_endpoint"]
jsonContent["cloudbrokerUsername"] = body["cloudbroker_username"]
jsonContent["cloudbrokerPassword"] = body["cloudbroker_password"]
print("Changed credentials!")
if "privateClouds" in body and body["privateClouds"]:
jsonContent["privateClouds"] = body["privateClouds"]
data["result"] = "success"
data["resultCode"] = "202"
if environment == "infrastructure":
create_infrastructure_tf(jsonContent)
data["content"] = "Request for infrastructure configuration sent"
return make_response(json.dumps(data))
elif environment == "services":
create_services_tf(jsonContent)
data["content"] = "Request for services configuration sent"
return make_response(json.dumps(data))
elif environment == "all":
create_infrastructure_tf(jsonContent)
create_services_tf(jsonContent)
data["content"] = "Request for infrastructure and services configuration sent"
return make_response(json.dumps(data))
else:
data["result"] = "error"
data["resultCode"] = "400"
data["content"] = "Wrong environment parameter. Accepted values are: 'infrastructure'|'services'|'all'"
response = make_response(json.dumps(data))
response.status_code = 400
return response
@api.route("/terraform/<operation>/<appName>/<folder>")
@api.doc(params={'operation': 'The Terraform operation to perform: [init | plan | apply]', 'appName':'The name of the application','folder':'The logical application area we want to manage: [infrastructure | services]'})
class terraformOperation(Resource):
@api.expect(env_fields)
def post(self, operation, appName, folder):
op_id = str(uuid.uuid4())
body = request.get_json()
status_data = {}
@copy_current_request_context
def terraformThread(operation, appName, folder, op_id, status_data):
print("Starting thread for operation " + op_id)
status_data["operation"] = operation
status_data["folder"] = folder
status_data["appName"] = appName
status_data["operation_id"] = op_id
log_dir = appName+"/logs"
if not os.path.exists(log_dir):
os.makedirs(log_dir)
status_data["result"] = "running"
try:
client = MongoClient()
adaptDoDb = client['adapt-do-db']
operations = adaptDoDb.operations
jdata = json.loads(json.dumps(status_data))
db_result = operations.update_one({'operation_id': op_id}, {'$set': jdata}, upsert=True)
print("POST operation: %s record(s) matched, %s record(s) modified, upserted id: %s" % (
db_result.matched_count, db_result.modified_count, db_result.upserted_id))
except:
print("Unexpected error:", sys.exc_info()[0])
finally:
client.close()
#result = make_response(subprocess.run(['terraform', operation, '-no-color'], cwd=appName+"/"+folder, stdout=subprocess.PIPE, stderr=subprocess.STDOUT).stdout.decode('utf-8'))
with open(log_dir+"/"+folder+"-"+operation+"-"+op_id+".log","w") as out:
if operation == "destroy":
myproc = subprocess.run(['terraform', operation, '-no-color', '-force'], cwd=appName+"/"+folder, stdout=out, stderr=out)
else:
print("Starting subprocess")
myproc = subprocess.run(['terraform', operation, '-no-color'], cwd=appName+"/"+folder, stdout=out, stderr=out)
print("myproc return code: " + str(myproc.returncode))
#section to manage the update of the app description file upon completion of 'apply' operations
if myproc.returncode == 0:
status_data["result"] = "success"
elif myproc.returncode > 0:
status_data["result"] = "failed"
try:
client = MongoClient()
adaptDoDb = client['adapt-do-db']
operations = adaptDoDb.operations
jdata = json.loads(json.dumps(status_data))
db_result = operations.update_one({'operation_id': op_id}, {'$set': jdata}, upsert=True)
print("POST operation: %s record(s) matched, %s record(s) modified, upserted id: %s" % (
db_result.matched_count, db_result.modified_count, db_result.upserted_id))
except:
print("Unexpected error:", sys.exc_info()[0])
finally:
client.close()
if operation == "apply":
data = {}
if body:
print("Body is present in request ")
else:
print("Body is not present in request ")
if body:
repo_user = body["repository_user"]
repo_pwd = body["repository_pwd"]
repo_url = body["repository_url"]
repo_url = repo_url[8:]
repo_url = "https://"+urllib.parse.quote(repo_user)+":"+urllib.parse.quote(repo_pwd)+"@"+repo_url
print("repo_url for clone: " + repo_url)
if not "repository_url" in body or not "filepath" in body or not body["repository_user"] or not body["repository_pwd"]:
data["result"] = "error"
data["resultCode"] = "400"
data["content"] = "Missing parameters"
response = make_response(json.dumps(data))
response.status_code = 400
return response
repository = body["repository_url"].split('/')[-1]
repository = repository[:repository.index('.git')]
filePath = body["filepath"]
if not os.path.exists("repo/"+repository):
print("Cloning repository: " + body["repository_url"])
if "branch" in body:
branch = body["branch"]
result = subprocess.check_output(['git', 'clone', repo_url, '-b', branch], cwd="repo/")
else:
result = subprocess.check_output(['git', 'clone', repo_url], cwd="repo/")
print("Pulling repository: " + body["repository_url"])
result = subprocess.check_output(['git', 'pull'], cwd="repo/"+repository)
if "revision" in body:
revision = body["revision"]
else:
revision = "HEAD"
print("Getting revision " + revision + " for file " + filePath + " from repository " + repository + " (" + body["repository_url"] + ")")
result = subprocess.check_output(['git', 'checkout', '-m', revision, filePath], cwd="repo/"+repository)
with open('repo/'+repository+"/"+filePath, 'r') as ad:
content = ad.read()
print("Got application descriptor:")
print(json.loads(content))
jsonContent = json.loads(content)
if folder == "infrastructure":
#Generating a unique instance id for the application, to be saved in the app descriptor
print("Generating unique instance id for application")
jsonContent['applicationInstanceId'] = str(uuid.uuid4())
print("Generated unique instance id for application: " + jsonContent['applicationInstanceId'])
status_urls = []
for p in jsonContent['virtualMachines']:
nodeName = p['dockerHostNodeName']
nodeIpLabel = nodeName+".external_ip_address"
vm_ip = subprocess.check_output(['terraform', 'output', nodeIpLabel], cwd=appName+"/"+folder).decode(encoding='UTF-8').strip()
if vm_ip:
print("Got ip: " + vm_ip + " for node " + nodeName )
p['dockerHostPublicIp'] = vm_ip
status_urls.append(request.url_root+"vm/"+vm_ip+"/status")
try:
client = MongoClient()
adaptDoDb = client['adapt-do-db']
operations = adaptDoDb.operations
json_op = operations.find_one({'operation_id': op_id})
status_data = json.loads(json_op)
status_data["status_urls"] = status_urls
jdata = json.loads(json.dumps(status_data))
db_result = operations.update_one({'operation_id': op_id}, {'$set': jdata}, upsert=True)
print("POST operation: %s record(s) matched, %s record(s) modified, upserted id: %s" % (
db_result.matched_count, db_result.modified_count, db_result.upserted_id))
except:
print("Unexpected error:", sys.exc_info()[0])
finally:
client.close()
with open('repo/'+repository+"/"+filePath, 'w') as ad:
json.dump(jsonContent, ad, indent=4, sort_keys=False)
print("Updated locally application description:")
print(json.dumps(jsonContent, indent=4, sort_keys=False))
result = subprocess.check_output(['git', 'config', 'user.email', 'fakeemail@fakeemail.com'], cwd="repo/"+repository)
result = subprocess.check_output(['git', 'config', 'user.name', repo_user], cwd="repo/"+repository)
print("Committing application description file:")
try:
result = subprocess.check_output(['git', 'commit', '-a', '-m', 'Added node ip'], cwd="repo/"+repository)
except subprocess.CalledProcessError as commitError:
output = commitError.output.decode(encoding='UTF-8')
if "Your branch is up-to-date" not in output:
print("Commit error: " + output)