From 97c3d6c4aafb5d98a1a77dee19dc9c275e1a0d6a Mon Sep 17 00:00:00 2001
From: Debora Benedetto <debora.benedetto@hpe.com>
Date: Fri, 10 Jun 2022 14:13:28 +0200
Subject: [PATCH] add aws templates and integration
---
controller/Orchestrator.py | 17 ++++++++++---
icgparser/DomlParserUtilities.py | 9 +++++--
icgparser/ModelParser.py | 6 ++++-
plugin/TemplateUtils.py | 6 +++++
template-location.properties | 6 +++++
templates/terraform/aws/init.tpl | 15 +++++++++++
templates/terraform/aws/network.tpl | 25 +++++++++++++------
templates/terraform/aws/port_rule.tpl | 20 +++++++++------
templates/terraform/aws/virtual_machine.tpl | 23 +++--------------
templates/terraform/open_stack/init.tpl | 8 +++---
.../terraform/open_stack/virtual_machine.tpl | 17 +++++++++++++
utility/FileParsingUtility.py | 7 ++++++
12 files changed, 112 insertions(+), 47 deletions(-)
diff --git a/controller/Orchestrator.py b/controller/Orchestrator.py
index 8d532b8..a35389e 100644
--- a/controller/Orchestrator.py
+++ b/controller/Orchestrator.py
@@ -1,11 +1,13 @@
import json
import logging
import tarfile
+import time
import uuid
import yaml
from icgparser import ModelParser
from plugin import AnsiblePlugin, TerraformPlugin
+from utility.FileParsingUtility import replace_none_with_empty_str
class CompressFolder:
@@ -48,11 +50,13 @@ def save_file(data, file_path, output_extensions="json"):
logging.info(f"Saving data at: {file_path}")
file = open(file_path, "w")
if isinstance(data, dict) and output_extensions == "YAML":
+ logging.info("Converting python dict into yaml data")
data = yaml.dump(data)
data = "---\n" + data + "..."
if isinstance(data, dict):
- data = json.dumps(data, indent=2, sort_keys=True)
- print(data)
+ data_without_none_value = replace_none_with_empty_str(data)
+ logging.info("Converting python dict into json data")
+ data = json.dumps(data_without_none_value, indent=2, sort_keys=True)
file.write(data)
file.close()
@@ -96,8 +100,13 @@ def create_intermediate_representation(model_path, is_multiecore_metamodel, meta
intermediate_representation = ModelParser.parse_model(model_path=model_path,
is_multiecore_metamodel=is_multiecore_metamodel,
metamodel_directory=metamodel_directory)
- intermediate_representation = reorganize_info(intermediate_representation)
- logging.info("Successfully created intermediate representation")
+ ## TODO remove, introduced because sg sometimes not created
+ if "computingGroup" in intermediate_representation["steps"][0]["data"].keys():## TODO remove
+ logging.debug("security group found")## TODO remove
+ else:
+ logging.debug("no security group found!")
+ # intermediate_representation = reorganize_info(intermediate_representation)
+ logging.info(f"Successfully created intermediate representation {intermediate_representation}")
intermediate_representation_path = "input_file_generated/ir.json"
save_file(intermediate_representation, intermediate_representation_path)
logging.info(f"Saved intermediate representation at {intermediate_representation_path}")
diff --git a/icgparser/DomlParserUtilities.py b/icgparser/DomlParserUtilities.py
index ba1d8d4..b546d76 100644
--- a/icgparser/DomlParserUtilities.py
+++ b/icgparser/DomlParserUtilities.py
@@ -90,10 +90,15 @@ def save_inner_components(from_object, to_object):
inner_components = from_object.eAllContents()
for obj in inner_components:
if not isinstance(obj, EOrderedSet): # TODO espandere info
- print(f'Saving information from object {obj.name}')
+ if obj.name is not None:
+ object_name = obj.name
+ else:
+ logging.warning(f'Object name not available, changing it using class name: {obj.eClass.name}')
+ object_name = obj.eClass.name
+ print(f'Saving information from object {object_name}')
inner_component = save_attributes(obj, {})
save_references_info(obj, inner_component)
- to_object[obj.name] = inner_component
+ to_object[object_name] = inner_component
return to_object
diff --git a/icgparser/ModelParser.py b/icgparser/ModelParser.py
index 6b5b072..68b4627 100644
--- a/icgparser/ModelParser.py
+++ b/icgparser/ModelParser.py
@@ -22,6 +22,7 @@ doml_layers = {
"active_infrastructure_layer": "activeInfrastructure",
}
+
def to_camel_case(content):
return content[0].lower() + content[1:]
@@ -30,9 +31,12 @@ def include_missing_objects_from_infrastructure_layer(to_step):
for obj_name in DomlParserUtilities.retrieve_missing_parsed_resources():
obj = DomlParserUtilities.retrieve_missing_parsed_resources()[obj_name]
infra_object_representation = {}
- infra_object_representation = DomlParserUtilities.save_attributes(obj["resource"], infra_object_representation)
+ infra_object_representation = DomlParserUtilities.save_attributes(obj["resource"], infra_object_representation,
+ skip_component_name=True)
infra_object_representation = DomlParserUtilities.save_inner_components(obj["resource"],
infra_object_representation)
+ infra_object_representation = DomlParserUtilities.add_infrastructure_information(obj["resource"],
+ infra_object_representation)
## TODO fix attenzione che sovrascrive
ir_key_name = to_camel_case(obj["reference"].eType.name)
to_step["data"][ir_key_name] = [infra_object_representation]
diff --git a/plugin/TemplateUtils.py b/plugin/TemplateUtils.py
index 1bda9f2..6eeb561 100644
--- a/plugin/TemplateUtils.py
+++ b/plugin/TemplateUtils.py
@@ -6,6 +6,10 @@ import jinja2
from jinja2 import Template
+@jinja2.pass_context
+def get_context(c):
+ return c
+
def find_template_path(iac_language, key, resource_name):
try:
properties_reader = configparser.ConfigParser()
@@ -20,6 +24,8 @@ def find_template_path(iac_language, key, resource_name):
def edit_template(template, parameters: dict):
logging.info("Starting editing template")
+ template.globals['context'] = get_context
+ template.globals['callable'] = callable
render = template.render(parameters)
template_with_custom_params = ""+render+"\n"
return template_with_custom_params
diff --git a/template-location.properties b/template-location.properties
index ccfa042..b644572 100644
--- a/template-location.properties
+++ b/template-location.properties
@@ -10,6 +10,12 @@ vm = templates/terraform/azure/virtual_machine.tpl
net = templates/terraform/azure/network.tpl
rg = templates/terraform/azure/resource_group.tpl
+[terraform.aws]
+init = templates/terraform/aws/init.tpl
+vms = templates/terraform/aws/virtual_machine.tpl
+networks = templates/terraform/aws/network.tpl
+computingGroup = templates/terraform/aws/port_rule.tpl
+
[ansible.ubuntu]
nginx = templates/ansible/ubuntu/nginx.tpl
mysql = templates/ansible/ubuntu/mysql.tpl
diff --git a/templates/terraform/aws/init.tpl b/templates/terraform/aws/init.tpl
index e69de29..0298118 100644
--- a/templates/terraform/aws/init.tpl
+++ b/templates/terraform/aws/init.tpl
@@ -0,0 +1,15 @@
+terraform {
+ required_providers {
+ aws = {
+ source = "hashicorp/aws"
+ version = "~> 3.0"
+ }
+ }
+}
+
+# Configure the AWS Provider
+provider "aws" {
+ # region = var.region_name
+ # access_key = var.aws_access_key
+ # secret_key = var.aws_secret_key
+}
\ No newline at end of file
diff --git a/templates/terraform/aws/network.tpl b/templates/terraform/aws/network.tpl
index c565119..b43dba4 100644
--- a/templates/terraform/aws/network.tpl
+++ b/templates/terraform/aws/network.tpl
@@ -1,14 +1,23 @@
-resource "aws_subnet" "aws_subnet" {
- vpc_id = aws_vpc.aws_vpc.id
- cidr_block = "{{ subnet_cidrblock }}"
+resource "aws_vpc" "{{infra_element_name}}" {
+ cidr_block = "{{ addressRange }}"
tags = {
- Name = "{{ subnetname }}"
+ Name = "{{name}}"
}
+}
+resource "aws_subnet" "{{infra_element_name ~ "_subnet"}}" {
+ vpc_id = aws_vpc.{{infra_element_name}}.id
+ cidr_block = "{{vpc_subnet.addressRange}}"
+ # map_public_ip_on_launch = true
+ tags = {
+ Name = "{{name}}"
+ }
}
-resource "aws_vpc" "aws_vpc" {
- cidr = "{{ vpc_cidr }}"
+
+resource "aws_network_interface" {{infra_element_name ~ "_network_interface"}} {
+ subnet_id = aws_subnet.{{infra_element_name ~ "_subnet"}}.id
+ security_groups = [aws_security_group.{{ name ~ "_security_group_rule" }}.id] ##TOBECHANGED
tags = {
- Name = "{{ vpcname }}"
+ Name = "{{infra_element_name ~ "_network_interface"}}"
}
-}
\ No newline at end of file
+}
diff --git a/templates/terraform/aws/port_rule.tpl b/templates/terraform/aws/port_rule.tpl
index 0e54a7e..0f3d507 100644
--- a/templates/terraform/aws/port_rule.tpl
+++ b/templates/terraform/aws/port_rule.tpl
@@ -1,10 +1,14 @@
-resource "openstack_compute_secgroup_v2" "{{ name }}" {
- name = "{{ name }}"
- description = "Security group rule for port {{ fromPort }}"
- rule {
- from_port = {{ fromPort }}
- to_port = {{ toPort }}
- ip_protocol = "{{ protocol }}"
- cidr = {% for range in cidr %}"{{ range }}"{% endfor %}
+# CREATING SECURITY_GROUP
+resource "aws_security_group" "{{ infra_element_name ~ "_security_group_rule" }}" { ## TOBECHANGE
+ name = "{{ infra_element_name }}"
+ # description = "Security group rule for port {{ fromPort }}"
+ vpc_id = aws_vpc.{{vpc_name}}.id ##ADD VPC NAME REFERENCE
+ {% for key, value in context().items() %}{% if not callable(value)%} {%if value.kind and value.kind is defined %}
+ {% if value == "INGRESS" %} ingress {% else %} egress {% endif %} {
+ from_port = {{ value.fromPort }}
+ to_port = {{ value.toPort }}
+ protocol = "{{ value.protocol }}"
+ cidr_blocks = [{% for range in value.cidr %}"{{ range }}"{% endfor %}]
}
+ {% endif %}{% endif %}{% endfor %}
}
\ No newline at end of file
diff --git a/templates/terraform/aws/virtual_machine.tpl b/templates/terraform/aws/virtual_machine.tpl
index 4b27da6..fcb6b4d 100644
--- a/templates/terraform/aws/virtual_machine.tpl
+++ b/templates/terraform/aws/virtual_machine.tpl
@@ -1,27 +1,10 @@
-data "{{ vm }}" "ami{{ id }}" {
- #executable_users = {{ executable_users }}
- most_recent = {{ mostrecent }}
- name_regex = "{{ name_regex }}"
- #owners = {{ owners }}
- {{ filters }}
- owners = ["099720109477"] # Canonical
-}
-resource "aws_instance" "instance{{ id }}" {
- ami = data.aws_ami.ami{{ id }}.id
+resource "aws_instance" "{{name}}" {
+ ami = "{{ os }}"
instance_type = "{{ instance_type }}"
- tags = {
- Name = "{{ name }}"
- }
-}
-
-resource "aws_instance" "nginx-app" {
- ami = {{ os }}
- instance_type = {{ instance_type }}
key_name = "{{ssh_key_name}}"
network_interface {
- network_interface_id = aws_network_interface.nginx-network_interface.id
+ network_interface_id = aws_network_interface.{{i1.belongsTo ~ "_network_interface"}}.id
device_index = 0
}
}
-
diff --git a/templates/terraform/open_stack/init.tpl b/templates/terraform/open_stack/init.tpl
index 040b726..e72aeca 100644
--- a/templates/terraform/open_stack/init.tpl
+++ b/templates/terraform/open_stack/init.tpl
@@ -10,10 +10,10 @@ required_version = ">= 0.14.0"
# Configure the OpenStack Provider
provider "openstack" {
- user_name = var.openstack_username
- tenant_name = "admin"
- password = var.openstack_password
- auth_url = var.openstack_auth_url
+ #user_name = var.openstack_username
+ #tenant_name = "admin"
+ #password = var.openstack_password
+ #auth_url = var.openstack_auth_url
insecure = true
}
diff --git a/templates/terraform/open_stack/virtual_machine.tpl b/templates/terraform/open_stack/virtual_machine.tpl
index 9a3e8dc..6af46b2 100644
--- a/templates/terraform/open_stack/virtual_machine.tpl
+++ b/templates/terraform/open_stack/virtual_machine.tpl
@@ -7,6 +7,23 @@ resource "openstack_compute_instance_v2" "{{ infra_element_name }}" {
network {
port = openstack_networking_port_v2.{{ i1.belongsTo }}.id
}
+
+ ## AGENTS TO ADD
+ # this is subject to be moved to IEM as part of its baseline
+ provisioner "local-exec" {
+ command = "ansible-galaxy collection install community.general"
+ }
+
+ # this is subject to be moved to IEM as part of its baseline
+ provisioner "local-exec" {
+ command = "ansible-playbook ansible/playbooks/pma/site_requirements.yaml"
+ }
+
+ # secrets can be taken from environment variables at IEM but these security issues I will leave them to y2, the user can also be problematic ubuntu/root/centos/...
+ provisioner "local-exec" {
+ command = "ansible-playbook -u root -i '${openstack_networking_floatingip_v2.{{ infra_element_name ~ "_floating_ip_association" }}.address},' ansible/playbooks/pma/site.yaml --extra-vars '{\"pma_deployment_id\": \"123e4567-e89b-12d3-a456-426614174002\", \"pma_influxdb_bucket\": \"bucket\", \"pma_influxdb_token\": \"piacerePassword\", \"pma_influxdb_org\": \"piacere\", \"pma_influxdb_addr\": \"https://influxdb.pm.ci.piacere.digital.tecnalia.dev\" }'"
+ }
+
}
# Create ssh keys
diff --git a/utility/FileParsingUtility.py b/utility/FileParsingUtility.py
index e69de29..667ae8a 100644
--- a/utility/FileParsingUtility.py
+++ b/utility/FileParsingUtility.py
@@ -0,0 +1,7 @@
+import logging
+
+
+def replace_none_with_empty_str(some_dict):
+ logging.info("Replacing dictionary None values with empty string")
+ print(some_dict.items())
+ return {('REPLACE' if k is None else k): ('' if v is None else v) for k, v in some_dict.items()}
\ No newline at end of file
--
GitLab