add template for ssh_key ansible generation, bugfix template replacement

controller/Orchestrator.py

@@ -21,7 +21,7 @@ import time
 import uuid
 import yaml
 
-from icgparser import ModelParser, PiacereInternalToolsIntegrator
+from icgparser import ModelParser, PiacereInternalToolsIntegrator, IntermediateRepresentationUtility
 from icgparser.IntermediateRepresentationUtility import IntermediateRepresentationResources
 from plugin import AnsiblePlugin, TerraformPlugin
 from utility.FileParsingUtility import replace_none_with_empty_str
@@ -123,6 +123,11 @@ def create_intermediate_representation(model_path, is_multiecore_metamodel, meta
     logging.info(f"Successfully created intermediate representation {intermediate_representation}")
     logging.info("Calling ICG PiacereInternalToolsIntegrator to add info for PIACERE internal tools")
     intermediate_representation = PiacereInternalToolsIntegrator.add_internal_tool_information(intermediate_representation)
+    logging.warning("Force adding sg information in network") ## TODO fix from doml
+    intermediate_representation = IntermediateRepresentationUtility.force_add_resources_name(
+        IntermediateRepresentationResources.NETWORKS,
+        IntermediateRepresentationResources.SECURITY_GROUPS,
+        intermediate_representation)
     intermediate_representation_path = "input_file_generated/ir.json"
     save_file(intermediate_representation, intermediate_representation_path)
     logging.info(f"Saved intermediate representation at {intermediate_representation_path}")
@@ -184,5 +189,6 @@ def create_iac_from_doml_path(model_path, is_multiecore_metamodel, metamodel_dir
     intermediate_representation = create_intermediate_representation(model_path, is_multiecore_metamodel,
                                                                      metamodel_directory)
     template_generated_folder = create_iac_from_intermediate_representation(intermediate_representation)
+    PiacereInternalToolsIntegrator.add_files_for_piacere_internal_tools(template_generated_folder)
     compress_folder_info = compress_iac_folder(template_generated_folder)
     return compress_folder_info

icgparser/IntermediateRepresentationUtility.py

@@ -13,6 +13,8 @@ class IntermediateRepresentationResources(NoValue):
     DATA = 'data'
     LANGUAGE = "programming_language"
     VIRTUAL_MACHINES = 'vms'
+    NETWORKS = "networks"
+    SECURITY_GROUPS = "computingGroup"
 
 
 def find_objects(object_name, intermediate_representation):
@@ -21,12 +23,26 @@ def find_objects(object_name, intermediate_representation):
     for step in steps:
         data = step[IntermediateRepresentationResources.DATA.value]
         if object_name.value in data.keys():
-            return data[IntermediateRepresentationResources.VIRTUAL_MACHINES.value]
+            return data[object_name.value]
     return []
 
 
-def add_step(step, intermediate_representation):
+def add_step(step, intermediate_representation, step_number):
     logging.info("Adding step into intermediate representation")
     steps = intermediate_representation[IntermediateRepresentationResources.STEPS.value]
+    if step_number:
+        steps.insert(step_number, step)
+    else:
         steps.append(step)
     return intermediate_representation
+
+
+def force_add_resources_name(to_resource, from_resource, intermediate_representation):
+    sec_groups = find_objects(from_resource, intermediate_representation)
+    sec_groups_names = []
+    for key, sg in sec_groups[0].items():
+        if isinstance(sg, dict) and sg["name"]:
+            sec_groups_names.append(sg["name"])
+    for resource in find_objects(to_resource, intermediate_representation):
+        resource["infra_sgs"] = sec_groups_names
+    return intermediate_representation

icgparser/PiacereInternalToolsIntegrator.py

@@ -24,7 +24,8 @@ def extract_info_for_monitoring_agents(intermediate_representation):
 def add_internal_tool_information(intermediate_representation):
     monitoring_step = extract_info_for_monitoring_agents(intermediate_representation)
     intermediate_representation = IntermediateRepresentationUtility.add_step(monitoring_step,
-                                                                             intermediate_representation)
+                                                                             intermediate_representation,
+                                                                             1)
     return intermediate_representation
 
 

icgparser/doml/v1/nginx-openstack_v1.domlx

@@ -12,7 +12,7 @@
       </machineDefinition>
     </nodes>
     <networks name="net1" protocol="tcp/ip" addressRange="16.0.0.0/24" connectedIfaces="//@infrastructure/@nodes.0/@machineDefinition/@ifaces.0"/>
-    <credentials xsi:type="infra:KeyPair" name="ssh_key" user="ubuntu"/>
+    <credentials xsi:type="infra:KeyPair" name="ssh_key" user="user1"/>
     <groups xsi:type="infra:SecurityGroup" name="sg" groupedNodes="//@infrastructure/@nodes.0/@machineDefinition" ifaces="//@infrastructure/@nodes.0/@machineDefinition/@ifaces.0">
       <rules name="icmp" protocol="icmp" fromPort="-1" toPort="-1">
         <cidr>0.0.0.0/0</cidr>
@@ -32,7 +32,7 @@
     <providers name="openstack">
       <vms name="concrete_vm" maps="//@infrastructure/@nodes.0/@machineDefinition">
         <annotations xsi:type="commons:SProperty" key="vm_name" value="nginx-host"/>
-        <annotations xsi:type="commons:SProperty" key="vm_flavor" value="small"/>
+        <annotations xsi:type="commons:SProperty" key="vm_flavor" value="ubuntu"/>
       </vms>
       <networks name="concrete_net" maps="//@infrastructure/@networks.0">
         <annotations xsi:type="commons:SProperty" key="name" value="ostack2"/>

input_file_generated/ir.json

@@ -51,13 +51,19 @@
         "credentials": [
           {
             "infra_element_name": "ssh_key",
-            "user": "ubuntu"
+            "user": "user1"
           }
         ],
         "networks": [
           {
             "addressRange": "16.0.0.0/24",
             "infra_element_name": "net1",
+            "infra_sgs": [
+              "icmp",
+              "http",
+              "https",
+              "ssh"
+            ],
             "name": "concrete_net",
             "protocol": "tcp/ip"
           }
@@ -76,7 +82,7 @@
             "infra_element_name": "vm1",
             "name": "concrete_vm",
             "os": "Ubuntu-Focal-20.04-Daily-2022-04-19",
-            "vm_flavor": "small",
+            "vm_flavor": "ubuntu",
             "vm_name": "nginx-host"
           }
         ]
@@ -85,8 +91,7 @@
     },
     {
       "data": {
-        "nginx": {
-          "name": "nginx",
+        "piacere_monitoring": {
           "node": {
             "credentials": "ssh_key",
             "group": "sg",
@@ -99,18 +104,18 @@
             "infra_element_name": "vm1",
             "name": "concrete_vm",
             "os": "Ubuntu-Focal-20.04-Daily-2022-04-19",
-            "vm_flavor": "small",
+            "vm_flavor": "ubuntu",
             "vm_name": "nginx-host"
-          },
-          "source_code": "/usr/share/nginx/html/index.html"
+          }
         }
       },
       "programming_language": "ansible",
-      "step_name": "nginx"
+      "step_name": "piacere_monitoring"
     },
     {
       "data": {
-        "piacere_monitoring": {
+        "nginx": {
+          "name": "nginx",
           "node": {
             "credentials": "ssh_key",
             "group": "sg",
@@ -123,13 +128,14 @@
             "infra_element_name": "vm1",
             "name": "concrete_vm",
             "os": "Ubuntu-Focal-20.04-Daily-2022-04-19",
-            "vm_flavor": "small",
+            "vm_flavor": "ubuntu",
             "vm_name": "nginx-host"
-          }
+          },
+          "source_code": "/usr/share/nginx/html/index.html"
         }
       },
       "programming_language": "ansible",
-      "step_name": "piacere_monitoring"
+      "step_name": "nginx"
     }
   ]
 }
\ No newline at end of file

output_file_example/nginx_openstack/terraform/main.tf

@@ -68,7 +68,7 @@ resource "openstack_networking_port_v2" "nginx" {
   admin_state_up = true
   security_group_ids = [
     openstack_compute_secgroup_v2.ssh.id,
-    openstack_compute_secgroup_v2.http.id
+    openstack_compute_secgroup_v2.http.id,
   ]
   fixed_ip {
     subnet_id = openstack_networking_subnet_v2.nginx.id

output_files_generated/nginx_openstack/config.yaml

 ---
 iac:
 - terraform
-- nginx
 - piacere_monitoring
+- nginx
 ...
\ No newline at end of file

output_files_generated/nginx_openstack/nginx/inventory.j2

@@ -5,5 +5,5 @@
 
 [servers_for_nginx:vars]
 ansible_connection=ssh
-ansible_user=
-ansible_ssh_private_key_file={{ instance_server_private_key_ }}
+ansible_user=ubuntu
+ansible_ssh_private_key_file=ssh_key

output_files_generated/nginx_openstack/nginx/ssh_key.j2

-{{ instance_server_private_key }}
\ No newline at end of file
+{{ instance_server_private_key_ssh_key }}

output_files_generated/nginx_openstack/piacere_monitoring/inventory.j2

@@ -5,5 +5,5 @@
 
 [servers_for_:vars]
 ansible_connection=ssh
-ansible_user=
-ansible_ssh_private_key_file={{ instance_server_private_key_ }}
+ansible_user=ubuntu
+ansible_ssh_private_key_file=ssh_key

output_files_generated/nginx_openstack/piacere_monitoring/ssh_key.j2

-{{ instance_server_private_key }}
\ No newline at end of file
+{{ instance_server_private_key_ssh_key }}

output_files_generated/nginx_openstack/terraform/config.yaml

@@ -9,8 +9,8 @@ input:
   - OS_PROJECT_NAME
 output:
 
-  - instance_server_public_key_
-  - instance_server_private_key_
+  - instance_server_public_key_ssh_key
+  - instance_server_private_key_ssh_key
   - instance_ip_nginx-host
 
 ...

output_files_generated/nginx_openstack/terraform/main.tf

@@ -12,10 +12,6 @@ required_version = ">= 0.14.0"
 
 # Configure the OpenStack Provider
 provider "openstack" {
-  #user_name   = var.openstack_username
-  #tenant_name = "admin"
-  #password    = var.openstack_password
-  #auth_url    = var.openstack_auth_url
   insecure    = true
 }
 
@@ -24,42 +20,16 @@ data "openstack_networking_network_v2" "external" {
   name = "external"
 }
 
-data "openstack_identity_project_v3" "test_tenant" {
-  name = "admin"
-}
-
-data "openstack_networking_secgroup_v2" "default" {
-  name = "default"
-  tenant_id = data.openstack_identity_project_v3.test_tenant.id
-}
-
 
 # Create virtual machine
 resource "openstack_compute_instance_v2" "vm1" {
   name        = "nginx-host"
   image_name  = "Ubuntu-Focal-20.04-Daily-2022-04-19"
-  flavor_name = "small"
+  flavor_name = "ubuntu"
   key_pair    = openstack_compute_keypair_v2.ssh_key.name
   network {
     port = openstack_networking_port_v2.net1.id
   }
-
-  ## AGENTS TO ADD
-  # this is subject to be moved to IEM as part of its baseline
-    provisioner "local-exec" {
-    command = "ansible-galaxy collection install community.general"
-  }
-
-  # this is subject to be moved to IEM as part of its baseline
-  provisioner "local-exec" {
-    command = "ansible-playbook ansible/playbooks/pma/site_requirements.yaml"
-  }
-
-  # secrets can be taken from environment variables at IEM but these security issues I will leave them to y2, the user can also be problematic ubuntu/root/centos/...
-  provisioner "local-exec" {
-    command = "ansible-playbook -u root -i '${openstack_networking_floatingip_v2.vm1_floating_ip.address},' ansible/playbooks/pma/site.yaml --extra-vars '{\"pma_deployment_id\": \"123e4567-e89b-12d3-a456-426614174002\", \"pma_influxdb_bucket\": \"bucket\", \"pma_influxdb_token\": \"piacerePassword\", \"pma_influxdb_org\": \"piacere\", \"pma_influxdb_addr\": \"https://influxdb.pm.ci.piacere.digital.tecnalia.dev\" }'"
-  }
-
 }
 
 # Create floating ip
@@ -97,7 +67,11 @@ resource "openstack_networking_port_v2" "net1" {
   network_id     = openstack_networking_network_v2.net1.id
   admin_state_up = true
   security_group_ids = [
-    data.openstack_networking_secgroup_v2.default.id        #default flavour id
+  openstack_compute_secgroup_v2.icmp.id,
+  openstack_compute_secgroup_v2.http.id,
+  openstack_compute_secgroup_v2.https.id,
+  openstack_compute_secgroup_v2.ssh.id,
+  
   ]
   fixed_ip {
     subnet_id = openstack_networking_subnet_v2.net1_subnet.id
@@ -168,7 +142,7 @@ resource "openstack_compute_secgroup_v2" "ssh" {
 
 # Create ssh keys
 resource "openstack_compute_keypair_v2" "ssh_key" {
-  name       = "ubuntu"
-  # public_key = "ubuntu"
+  name       = "user1"
+  # public_key = "user1"
 }
 

plugin/AnsiblePlugin.py

@@ -38,7 +38,7 @@ def find_operating_system(parameters):
         raise PluginResourceNotFoundError(plugin_name="AnsiblePlugin", resource_name="operating system")
 
 
-def create_inventory_file(parameters, language, operating_system, template_name):
+def create_template_file(parameters, language, operating_system, template_name):
     inventory_template_path = TemplateUtils.find_template_path(language, operating_system, template_name)
     template = TemplateUtils.read_template(inventory_template_path)
     template_filled = TemplateUtils.edit_template(template, parameters)
@@ -65,12 +65,13 @@ def create_files(step, output_path):
             template = TemplateUtils.read_template(ansible_template_path)
             template_filled = TemplateUtils.edit_template(template, resource_params)
 
-            inventory_template_filled = create_inventory_file(resource_params, language, operating_system, "inventory")
-            config_template_filled = create_inventory_file(resource_params, language, operating_system, "config")
+            inventory_template_filled = create_template_file(resource_params, language, operating_system, "inventory")
+            config_template_filled = create_template_file(resource_params, language, operating_system, "config")
+            ssh_key_template_filled = create_template_file(resource_params, language, operating_system, "ssh_key")
 
             TemplateUtils.write_template(inventory_template_filled, inventory_output_file_path)
             TemplateUtils.write_template(template_filled, ansible_output_file_path)
             TemplateUtils.write_template(config_template_filled, config_output_file_path)
-            TemplateUtils.write_template("{{ instance_server_private_key }}", ssh_key_output_file_path)
+            TemplateUtils.write_template(ssh_key_template_filled, ssh_key_output_file_path)
 
     logging.info("File available at: {}".format(output_path))

template-location.properties

@@ -36,6 +36,7 @@ computingGroup = templates/terraform/aws/port_rule.tpl
 
 [ansible.ubuntu]
 inventory = templates/ansible/ubuntu/inventory.tpl
+ssh_key = templates/ansible/ubuntu/ssh_key.tpl
 config = templates/ansible/ubuntu/config.tpl
 nginx = templates/ansible/ubuntu/nginx.tpl
 mysql = templates/ansible/ubuntu/mysql.tpl

templates/ansible/ubuntu/inventory.tpl

@@ -19,5 +19,5 @@
 
 [{{ "servers_for_" ~ name }}:vars]
 ansible_connection=ssh
-ansible_user={{node.vm_key_name}}
-ansible_ssh_private_key_file={% raw %}{{ instance_server_private_key_{% endraw %}{{ node.vm_key_name }} {% raw %}}}{% endraw %}
+ansible_user=ubuntu
+ansible_ssh_private_key_file=ssh_key

templates/ansible/ubuntu/ssh_key.tpl

+{% raw %}{{ instance_server_private_key_{% endraw %}{{ node.credentials }} {% raw %}}}{% endraw %}
\ No newline at end of file

templates/terraform/open_stack/config.tpl

@@ -23,8 +23,8 @@ input:
   - OS_PROJECT_NAME
 output:
 {% for vm in vms %}
-  - instance_server_public_key_{{ vm.vm_key_name }}
-  - instance_server_private_key_{{ vm.vm_key_name }}
+  - instance_server_public_key_{{ vm.credentials }}
+  - instance_server_private_key_{{ vm.credentials }}
   - instance_ip_{{ vm.vm_name }}
 {% endfor %}
 ...

templates/terraform/open_stack/init.tpl

@@ -26,10 +26,6 @@ required_version = ">= 0.14.0"
 
 # Configure the OpenStack Provider
 provider "openstack" {
-  #user_name   = var.openstack_username
-  #tenant_name = "admin"
-  #password    = var.openstack_password
-  #auth_url    = var.openstack_auth_url
   insecure    = true
 }
 
@@ -37,12 +33,3 @@ provider "openstack" {
 data "openstack_networking_network_v2" "external" {
   name = "external"
 }
-
-data "openstack_identity_project_v3" "test_tenant" {
-  name = "admin"
-}
-
-data "openstack_networking_secgroup_v2" "default" {
-  name = "default"
-  tenant_id = data.openstack_identity_project_v3.test_tenant.id
-}
\ No newline at end of file

templates/terraform/open_stack/network.tpl

@@ -35,7 +35,8 @@ resource "openstack_networking_port_v2" "{{ infra_element_name }}" {
   network_id     = openstack_networking_network_v2.{{ infra_element_name }}.id
   admin_state_up = true
   security_group_ids = [
-    data.openstack_networking_secgroup_v2.default.id        #default flavour id
+  {% for sg in infra_sgs %}openstack_compute_secgroup_v2.{{sg}}.id,
+  {% endfor %}
   ]
   fixed_ip {
     subnet_id = openstack_networking_subnet_v2.{{ infra_element_name ~ "_subnet" }}.id