diff --git a/controller/Orchestrator.py b/controller/Orchestrator.py index 8d532b89c8c6992b4939b724d3a3aa2d69774de3..a35389e1b4db0449d522c0494f65d44d20db6686 100644 --- a/controller/Orchestrator.py +++ b/controller/Orchestrator.py @@ -1,11 +1,13 @@ import json import logging import tarfile +import time import uuid import yaml from icgparser import ModelParser from plugin import AnsiblePlugin, TerraformPlugin +from utility.FileParsingUtility import replace_none_with_empty_str class CompressFolder: @@ -48,11 +50,13 @@ def save_file(data, file_path, output_extensions="json"): logging.info(f"Saving data at: {file_path}") file = open(file_path, "w") if isinstance(data, dict) and output_extensions == "YAML": + logging.info("Converting python dict into yaml data") data = yaml.dump(data) data = "---\n" + data + "..." if isinstance(data, dict): - data = json.dumps(data, indent=2, sort_keys=True) - print(data) + data_without_none_value = replace_none_with_empty_str(data) + logging.info("Converting python dict into json data") + data = json.dumps(data_without_none_value, indent=2, sort_keys=True) file.write(data) file.close() @@ -96,8 +100,13 @@ def create_intermediate_representation(model_path, is_multiecore_metamodel, meta intermediate_representation = ModelParser.parse_model(model_path=model_path, is_multiecore_metamodel=is_multiecore_metamodel, metamodel_directory=metamodel_directory) - intermediate_representation = reorganize_info(intermediate_representation) - logging.info("Successfully created intermediate representation") + ## TODO remove, introduced because sg sometimes not created + if "computingGroup" in intermediate_representation["steps"][0]["data"].keys():## TODO remove + logging.debug("security group found")## TODO remove + else: + logging.debug("no security group found!") + # intermediate_representation = reorganize_info(intermediate_representation) + logging.info(f"Successfully created intermediate representation {intermediate_representation}") intermediate_representation_path = "input_file_generated/ir.json" save_file(intermediate_representation, intermediate_representation_path) logging.info(f"Saved intermediate representation at {intermediate_representation_path}") diff --git a/icgparser/DomlParserUtilities.py b/icgparser/DomlParserUtilities.py index ba1d8d4e105820e9c7ef17dd7e1b2a9955cb0777..b546d76a9aa31994e4a33c77094534c0ab0a8b0f 100644 --- a/icgparser/DomlParserUtilities.py +++ b/icgparser/DomlParserUtilities.py @@ -90,10 +90,15 @@ def save_inner_components(from_object, to_object): inner_components = from_object.eAllContents() for obj in inner_components: if not isinstance(obj, EOrderedSet): # TODO espandere info - print(f'Saving information from object {obj.name}') + if obj.name is not None: + object_name = obj.name + else: + logging.warning(f'Object name not available, changing it using class name: {obj.eClass.name}') + object_name = obj.eClass.name + print(f'Saving information from object {object_name}') inner_component = save_attributes(obj, {}) save_references_info(obj, inner_component) - to_object[obj.name] = inner_component + to_object[object_name] = inner_component return to_object diff --git a/icgparser/ModelParser.py b/icgparser/ModelParser.py index 6b5b07230ceffcd094a66e2a7634f8926d8a15ac..68b4627f0e920b59a66a2b5a928aeeb5e5743068 100644 --- a/icgparser/ModelParser.py +++ b/icgparser/ModelParser.py @@ -22,6 +22,7 @@ doml_layers = { "active_infrastructure_layer": "activeInfrastructure", } + def to_camel_case(content): return content[0].lower() + content[1:] @@ -30,9 +31,12 @@ def include_missing_objects_from_infrastructure_layer(to_step): for obj_name in DomlParserUtilities.retrieve_missing_parsed_resources(): obj = DomlParserUtilities.retrieve_missing_parsed_resources()[obj_name] infra_object_representation = {} - infra_object_representation = DomlParserUtilities.save_attributes(obj["resource"], infra_object_representation) + infra_object_representation = DomlParserUtilities.save_attributes(obj["resource"], infra_object_representation, + skip_component_name=True) infra_object_representation = DomlParserUtilities.save_inner_components(obj["resource"], infra_object_representation) + infra_object_representation = DomlParserUtilities.add_infrastructure_information(obj["resource"], + infra_object_representation) ## TODO fix attenzione che sovrascrive ir_key_name = to_camel_case(obj["reference"].eType.name) to_step["data"][ir_key_name] = [infra_object_representation] diff --git a/plugin/TemplateUtils.py b/plugin/TemplateUtils.py index 1bda9f2b7d8252b5a4800e0c2bcbcd35a44ec98d..6eeb5613880037cd1202a1c386e2bb6c6618bc3b 100644 --- a/plugin/TemplateUtils.py +++ b/plugin/TemplateUtils.py @@ -6,6 +6,10 @@ import jinja2 from jinja2 import Template +@jinja2.pass_context +def get_context(c): + return c + def find_template_path(iac_language, key, resource_name): try: properties_reader = configparser.ConfigParser() @@ -20,6 +24,8 @@ def find_template_path(iac_language, key, resource_name): def edit_template(template, parameters: dict): logging.info("Starting editing template") + template.globals['context'] = get_context + template.globals['callable'] = callable render = template.render(parameters) template_with_custom_params = ""+render+"\n" return template_with_custom_params diff --git a/template-location.properties b/template-location.properties index ccfa0422c1422d2ae4a80b257bafcec71ddf7ad8..b644572d0d3fcbeadd7dc46568d983c4f3a44c03 100644 --- a/template-location.properties +++ b/template-location.properties @@ -10,6 +10,12 @@ vm = templates/terraform/azure/virtual_machine.tpl net = templates/terraform/azure/network.tpl rg = templates/terraform/azure/resource_group.tpl +[terraform.aws] +init = templates/terraform/aws/init.tpl +vms = templates/terraform/aws/virtual_machine.tpl +networks = templates/terraform/aws/network.tpl +computingGroup = templates/terraform/aws/port_rule.tpl + [ansible.ubuntu] nginx = templates/ansible/ubuntu/nginx.tpl mysql = templates/ansible/ubuntu/mysql.tpl diff --git a/templates/terraform/aws/init.tpl b/templates/terraform/aws/init.tpl index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..02981187f2e5449c04fd4d52dca691038151db68 100644 --- a/templates/terraform/aws/init.tpl +++ b/templates/terraform/aws/init.tpl @@ -0,0 +1,15 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 3.0" + } + } +} + +# Configure the AWS Provider +provider "aws" { + # region = var.region_name + # access_key = var.aws_access_key + # secret_key = var.aws_secret_key +} \ No newline at end of file diff --git a/templates/terraform/aws/network.tpl b/templates/terraform/aws/network.tpl index c565119c3099ea7ad0b77e6aca9bafa623c3e9d2..b43dba4bbbbf6165be5fbac8ae8d1aabaeeb52b5 100644 --- a/templates/terraform/aws/network.tpl +++ b/templates/terraform/aws/network.tpl @@ -1,14 +1,23 @@ -resource "aws_subnet" "aws_subnet" { - vpc_id = aws_vpc.aws_vpc.id - cidr_block = "{{ subnet_cidrblock }}" +resource "aws_vpc" "{{infra_element_name}}" { + cidr_block = "{{ addressRange }}" tags = { - Name = "{{ subnetname }}" + Name = "{{name}}" } +} +resource "aws_subnet" "{{infra_element_name ~ "_subnet"}}" { + vpc_id = aws_vpc.{{infra_element_name}}.id + cidr_block = "{{vpc_subnet.addressRange}}" + # map_public_ip_on_launch = true + tags = { + Name = "{{name}}" + } } -resource "aws_vpc" "aws_vpc" { - cidr = "{{ vpc_cidr }}" + +resource "aws_network_interface" {{infra_element_name ~ "_network_interface"}} { + subnet_id = aws_subnet.{{infra_element_name ~ "_subnet"}}.id + security_groups = [aws_security_group.{{ name ~ "_security_group_rule" }}.id] ##TOBECHANGED tags = { - Name = "{{ vpcname }}" + Name = "{{infra_element_name ~ "_network_interface"}}" } -} \ No newline at end of file +} diff --git a/templates/terraform/aws/port_rule.tpl b/templates/terraform/aws/port_rule.tpl index 0e54a7e0543d70b1021db34568d213c85589cbb3..0f3d50713cd0d5e89d46a94912c93c4eb02ae726 100644 --- a/templates/terraform/aws/port_rule.tpl +++ b/templates/terraform/aws/port_rule.tpl @@ -1,10 +1,14 @@ -resource "openstack_compute_secgroup_v2" "{{ name }}" { - name = "{{ name }}" - description = "Security group rule for port {{ fromPort }}" - rule { - from_port = {{ fromPort }} - to_port = {{ toPort }} - ip_protocol = "{{ protocol }}" - cidr = {% for range in cidr %}"{{ range }}"{% endfor %} +# CREATING SECURITY_GROUP +resource "aws_security_group" "{{ infra_element_name ~ "_security_group_rule" }}" { ## TOBECHANGE + name = "{{ infra_element_name }}" + # description = "Security group rule for port {{ fromPort }}" + vpc_id = aws_vpc.{{vpc_name}}.id ##ADD VPC NAME REFERENCE + {% for key, value in context().items() %}{% if not callable(value)%} {%if value.kind and value.kind is defined %} + {% if value == "INGRESS" %} ingress {% else %} egress {% endif %} { + from_port = {{ value.fromPort }} + to_port = {{ value.toPort }} + protocol = "{{ value.protocol }}" + cidr_blocks = [{% for range in value.cidr %}"{{ range }}"{% endfor %}] } + {% endif %}{% endif %}{% endfor %} } \ No newline at end of file diff --git a/templates/terraform/aws/virtual_machine.tpl b/templates/terraform/aws/virtual_machine.tpl index 4b27da64868ed27ad6508363f7708b0be64bd3ba..fcb6b4ddf9b6198cf4024c37bb5fea69d3f06b25 100644 --- a/templates/terraform/aws/virtual_machine.tpl +++ b/templates/terraform/aws/virtual_machine.tpl @@ -1,27 +1,10 @@ -data "{{ vm }}" "ami{{ id }}" { - #executable_users = {{ executable_users }} - most_recent = {{ mostrecent }} - name_regex = "{{ name_regex }}" - #owners = {{ owners }} - {{ filters }} - owners = ["099720109477"] # Canonical -} -resource "aws_instance" "instance{{ id }}" { - ami = data.aws_ami.ami{{ id }}.id +resource "aws_instance" "{{name}}" { + ami = "{{ os }}" instance_type = "{{ instance_type }}" - tags = { - Name = "{{ name }}" - } -} - -resource "aws_instance" "nginx-app" { - ami = {{ os }} - instance_type = {{ instance_type }} key_name = "{{ssh_key_name}}" network_interface { - network_interface_id = aws_network_interface.nginx-network_interface.id + network_interface_id = aws_network_interface.{{i1.belongsTo ~ "_network_interface"}}.id device_index = 0 } } - diff --git a/templates/terraform/open_stack/init.tpl b/templates/terraform/open_stack/init.tpl index 040b726778f9a3579dac9b9efd445916d60acfff..e72aeca9fdce777fc4861e007f96a84ca83842d5 100644 --- a/templates/terraform/open_stack/init.tpl +++ b/templates/terraform/open_stack/init.tpl @@ -10,10 +10,10 @@ required_version = ">= 0.14.0" # Configure the OpenStack Provider provider "openstack" { - user_name = var.openstack_username - tenant_name = "admin" - password = var.openstack_password - auth_url = var.openstack_auth_url + #user_name = var.openstack_username + #tenant_name = "admin" + #password = var.openstack_password + #auth_url = var.openstack_auth_url insecure = true } diff --git a/templates/terraform/open_stack/virtual_machine.tpl b/templates/terraform/open_stack/virtual_machine.tpl index 9a3e8dc08c822209cbd70c6cc97d81c49c9290d2..6af46b2a8078191f3e4ce3cf85a07393ac0f9144 100644 --- a/templates/terraform/open_stack/virtual_machine.tpl +++ b/templates/terraform/open_stack/virtual_machine.tpl @@ -7,6 +7,23 @@ resource "openstack_compute_instance_v2" "{{ infra_element_name }}" { network { port = openstack_networking_port_v2.{{ i1.belongsTo }}.id } + + ## AGENTS TO ADD + # this is subject to be moved to IEM as part of its baseline + provisioner "local-exec" { + command = "ansible-galaxy collection install community.general" + } + + # this is subject to be moved to IEM as part of its baseline + provisioner "local-exec" { + command = "ansible-playbook ansible/playbooks/pma/site_requirements.yaml" + } + + # secrets can be taken from environment variables at IEM but these security issues I will leave them to y2, the user can also be problematic ubuntu/root/centos/... + provisioner "local-exec" { + command = "ansible-playbook -u root -i '${openstack_networking_floatingip_v2.{{ infra_element_name ~ "_floating_ip_association" }}.address},' ansible/playbooks/pma/site.yaml --extra-vars '{\"pma_deployment_id\": \"123e4567-e89b-12d3-a456-426614174002\", \"pma_influxdb_bucket\": \"bucket\", \"pma_influxdb_token\": \"piacerePassword\", \"pma_influxdb_org\": \"piacere\", \"pma_influxdb_addr\": \"https://influxdb.pm.ci.piacere.digital.tecnalia.dev\" }'" + } + } # Create ssh keys diff --git a/utility/FileParsingUtility.py b/utility/FileParsingUtility.py index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..667ae8a2364049e25de01291b9383e4ec4d8bcbc 100644 --- a/utility/FileParsingUtility.py +++ b/utility/FileParsingUtility.py @@ -0,0 +1,7 @@ +import logging + + +def replace_none_with_empty_str(some_dict): + logging.info("Replacing dictionary None values with empty string") + print(some_dict.items()) + return {('REPLACE' if k is None else k): ('' if v is None else v) for k, v in some_dict.items()} \ No newline at end of file