add aws templates and integration

controller/Orchestrator.py

 import json
 import logging
 import tarfile
+import time
 import uuid
 import yaml
 
 from icgparser import ModelParser
 from plugin import AnsiblePlugin, TerraformPlugin
+from utility.FileParsingUtility import replace_none_with_empty_str
 
 
 class CompressFolder:
@@ -48,11 +50,13 @@ def save_file(data, file_path, output_extensions="json"):
     logging.info(f"Saving data at: {file_path}")
     file = open(file_path, "w")
     if isinstance(data, dict) and output_extensions == "YAML":
+        logging.info("Converting python dict into yaml data")
         data = yaml.dump(data)
         data = "---\n" + data + "..."
     if isinstance(data, dict):
-        data = json.dumps(data, indent=2, sort_keys=True)
-    print(data)
+        data_without_none_value = replace_none_with_empty_str(data)
+        logging.info("Converting python dict into json data")
+        data = json.dumps(data_without_none_value, indent=2, sort_keys=True)
     file.write(data)
     file.close()
 
@@ -96,8 +100,13 @@ def create_intermediate_representation(model_path, is_multiecore_metamodel, meta
     intermediate_representation = ModelParser.parse_model(model_path=model_path,
                                                           is_multiecore_metamodel=is_multiecore_metamodel,
                                                           metamodel_directory=metamodel_directory)
-    intermediate_representation = reorganize_info(intermediate_representation)
-    logging.info("Successfully created intermediate representation")
+    ## TODO remove, introduced because sg sometimes not created
+    if "computingGroup" in intermediate_representation["steps"][0]["data"].keys():## TODO remove
+        logging.debug("security group found")## TODO remove
+    else:
+        logging.debug("no security group found!")
+    # intermediate_representation = reorganize_info(intermediate_representation)
+    logging.info(f"Successfully created intermediate representation {intermediate_representation}")
     intermediate_representation_path = "input_file_generated/ir.json"
     save_file(intermediate_representation, intermediate_representation_path)
     logging.info(f"Saved intermediate representation at {intermediate_representation_path}")

icgparser/DomlParserUtilities.py

@@ -90,10 +90,15 @@ def save_inner_components(from_object, to_object):
     inner_components = from_object.eAllContents()
     for obj in inner_components:
         if not isinstance(obj, EOrderedSet):  # TODO espandere info
-            print(f'Saving information from object {obj.name}')
+            if obj.name is not None:
+                object_name = obj.name
+            else:
+                logging.warning(f'Object name not available, changing it using class name: {obj.eClass.name}')
+                object_name = obj.eClass.name
+            print(f'Saving information from object {object_name}')
             inner_component = save_attributes(obj, {})
             save_references_info(obj, inner_component)
-            to_object[obj.name] = inner_component
+            to_object[object_name] = inner_component
     return to_object
 
 

icgparser/ModelParser.py

@@ -22,6 +22,7 @@ doml_layers = {
     "active_infrastructure_layer": "activeInfrastructure",
 }
 
+
 def to_camel_case(content):
     return content[0].lower() + content[1:]
 
@@ -30,9 +31,12 @@ def include_missing_objects_from_infrastructure_layer(to_step):
     for obj_name in DomlParserUtilities.retrieve_missing_parsed_resources():
         obj = DomlParserUtilities.retrieve_missing_parsed_resources()[obj_name]
         infra_object_representation = {}
-        infra_object_representation = DomlParserUtilities.save_attributes(obj["resource"], infra_object_representation)
+        infra_object_representation = DomlParserUtilities.save_attributes(obj["resource"], infra_object_representation,
+                                                                          skip_component_name=True)
         infra_object_representation = DomlParserUtilities.save_inner_components(obj["resource"],
                                                                                 infra_object_representation)
+        infra_object_representation = DomlParserUtilities.add_infrastructure_information(obj["resource"],
+                                                                                         infra_object_representation)
         ## TODO fix attenzione che sovrascrive
         ir_key_name = to_camel_case(obj["reference"].eType.name)
         to_step["data"][ir_key_name] = [infra_object_representation]

plugin/TemplateUtils.py

@@ -6,6 +6,10 @@ import jinja2
 from jinja2 import Template
 
 
+@jinja2.pass_context
+def get_context(c):
+    return c
+
 def find_template_path(iac_language, key, resource_name):
     try:
         properties_reader = configparser.ConfigParser()
@@ -20,6 +24,8 @@ def find_template_path(iac_language, key, resource_name):
 
 def edit_template(template, parameters: dict):
     logging.info("Starting editing template")
+    template.globals['context'] = get_context
+    template.globals['callable'] = callable
     render = template.render(parameters)
     template_with_custom_params = ""+render+"\n"
     return template_with_custom_params

template-location.properties

@@ -10,6 +10,12 @@ vm = templates/terraform/azure/virtual_machine.tpl
 net = templates/terraform/azure/network.tpl
 rg = templates/terraform/azure/resource_group.tpl
 
+[terraform.aws]
+init = templates/terraform/aws/init.tpl
+vms = templates/terraform/aws/virtual_machine.tpl
+networks = templates/terraform/aws/network.tpl
+computingGroup = templates/terraform/aws/port_rule.tpl
+
 [ansible.ubuntu]
 nginx = templates/ansible/ubuntu/nginx.tpl
 mysql = templates/ansible/ubuntu/mysql.tpl

templates/terraform/aws/init.tpl

+terraform {
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "~> 3.0"
+    }
+  }
+}
+
+# Configure the AWS Provider
+provider "aws" {
+  # region = var.region_name
+  # access_key = var.aws_access_key
+  # secret_key = var.aws_secret_key
+}
\ No newline at end of file

templates/terraform/aws/network.tpl

-resource "aws_subnet" "aws_subnet" {
-  vpc_id = aws_vpc.aws_vpc.id
-  cidr_block = "{{ subnet_cidrblock }}"
+resource "aws_vpc" "{{infra_element_name}}" {
+  cidr_block = "{{ addressRange }}"
   tags = {
-    Name = "{{ subnetname }}"
+    Name = "{{name}}"
+  }
 }
 
+resource "aws_subnet" "{{infra_element_name ~ "_subnet"}}" {
+  vpc_id = aws_vpc.{{infra_element_name}}.id
+  cidr_block = "{{vpc_subnet.addressRange}}"
+  # map_public_ip_on_launch = true
+  tags = {
+    Name = "{{name}}"
   }
-resource "aws_vpc" "aws_vpc" {
-  cidr = "{{ vpc_cidr }}"
+}
+
+resource "aws_network_interface" {{infra_element_name ~ "_network_interface"}} {
+  subnet_id = aws_subnet.{{infra_element_name ~ "_subnet"}}.id
+  security_groups = [aws_security_group.{{ name ~ "_security_group_rule" }}.id] ##TOBECHANGED
   tags = {
-    Name = "{{ vpcname }}"
+    Name = "{{infra_element_name ~ "_network_interface"}}"
   }
 }

templates/terraform/aws/port_rule.tpl

-resource "openstack_compute_secgroup_v2" "{{ name }}" {
-  name        = "{{ name }}"
-  description  = "Security group rule for port {{ fromPort }}"
-  rule {
-    from_port   = {{ fromPort }}
-    to_port     = {{ toPort }}
-    ip_protocol = "{{ protocol }}"
-    cidr        = {% for range in cidr %}"{{ range }}"{% endfor %}
+# CREATING SECURITY_GROUP
+resource "aws_security_group" "{{ infra_element_name ~ "_security_group_rule" }}" { ## TOBECHANGE
+  name        = "{{ infra_element_name }}"
+  # description  = "Security group rule for port {{ fromPort }}"
+  vpc_id      = aws_vpc.{{vpc_name}}.id ##ADD VPC NAME REFERENCE
+  {% for key, value in context().items() %}{% if not callable(value)%} {%if value.kind and value.kind is defined %}
+  {% if value == "INGRESS" %} ingress {% else %} egress {% endif %}  {
+    from_port   = {{ value.fromPort }}
+    to_port     = {{ value.toPort }}
+    protocol = "{{ value.protocol }}"
+    cidr_blocks = [{% for range in value.cidr %}"{{ range }}"{% endfor %}]
   }
+  {% endif %}{% endif %}{% endfor %}
 }
\ No newline at end of file

templates/terraform/aws/virtual_machine.tpl

-data "{{ vm }}" "ami{{ id }}" {
-  #executable_users = {{ executable_users }}
-  most_recent = {{ mostrecent }}
-  name_regex = "{{ name_regex }}"
-  #owners = {{ owners }}
-  {{ filters }}
-  owners = ["099720109477"] # Canonical
-}
-resource "aws_instance" "instance{{ id }}" {
-  ami = data.aws_ami.ami{{ id }}.id
+resource "aws_instance" "{{name}}" {
+  ami = "{{ os }}"
   instance_type = "{{ instance_type }}"
-    tags = {
-     Name = "{{ name }}"
-   }
-}
-
-resource "aws_instance" "nginx-app" {
-  ami = {{ os }}
-  instance_type = {{ instance_type }}
   key_name = "{{ssh_key_name}}"
 
   network_interface {
-    network_interface_id = aws_network_interface.nginx-network_interface.id
+    network_interface_id = aws_network_interface.{{i1.belongsTo ~ "_network_interface"}}.id
     device_index = 0
   }
 }
-

templates/terraform/open_stack/init.tpl

@@ -10,10 +10,10 @@ required_version = ">= 0.14.0"
 
 # Configure the OpenStack Provider
 provider "openstack" {
-  user_name   = var.openstack_username
-  tenant_name = "admin"
-  password    = var.openstack_password
-  auth_url    = var.openstack_auth_url
+  #user_name   = var.openstack_username
+  #tenant_name = "admin"
+  #password    = var.openstack_password
+  #auth_url    = var.openstack_auth_url
   insecure    = true
 }
 

templates/terraform/open_stack/virtual_machine.tpl

@@ -7,6 +7,23 @@ resource "openstack_compute_instance_v2" "{{ infra_element_name }}" {
   network {
     port = openstack_networking_port_v2.{{ i1.belongsTo }}.id
   }
+
+  ## AGENTS TO ADD
+  # this is subject to be moved to IEM as part of its baseline
+    provisioner "local-exec" {
+    command = "ansible-galaxy collection install community.general"
+  }
+
+  # this is subject to be moved to IEM as part of its baseline
+  provisioner "local-exec" {
+    command = "ansible-playbook ansible/playbooks/pma/site_requirements.yaml"
+  }
+
+  # secrets can be taken from environment variables at IEM but these security issues I will leave them to y2, the user can also be problematic ubuntu/root/centos/...
+  provisioner "local-exec" {
+    command = "ansible-playbook -u root -i '${openstack_networking_floatingip_v2.{{ infra_element_name ~ "_floating_ip_association" }}.address},' ansible/playbooks/pma/site.yaml --extra-vars '{\"pma_deployment_id\": \"123e4567-e89b-12d3-a456-426614174002\", \"pma_influxdb_bucket\": \"bucket\", \"pma_influxdb_token\": \"piacerePassword\", \"pma_influxdb_org\": \"piacere\", \"pma_influxdb_addr\": \"https://influxdb.pm.ci.piacere.digital.tecnalia.dev\" }'"
+  }
+
 }
 
 # Create ssh keys

utility/FileParsingUtility.py

+import logging
+
+
+def replace_none_with_empty_str(some_dict):
+    logging.info("Replacing dictionary None values with empty string")
+    print(some_dict.items())
+    return {('REPLACE' if k is None else k): ('' if v is None else v) for k, v in some_dict.items()}
\ No newline at end of file