diff --git a/controller/Orchestrator.py b/controller/Orchestrator.py index 95cba247636c4462b4d616e73e20e30d90e43a0a..f80607303c22ffbe5306776c13db9c5bc643169a 100644 --- a/controller/Orchestrator.py +++ b/controller/Orchestrator.py @@ -21,7 +21,7 @@ import time import uuid import yaml -from icgparser import ModelParser, PiacereInternalToolsIntegrator +from icgparser import ModelParser, PiacereInternalToolsIntegrator, IntermediateRepresentationUtility from icgparser.IntermediateRepresentationUtility import IntermediateRepresentationResources from plugin import AnsiblePlugin, TerraformPlugin from utility.FileParsingUtility import replace_none_with_empty_str @@ -123,6 +123,11 @@ def create_intermediate_representation(model_path, is_multiecore_metamodel, meta logging.info(f"Successfully created intermediate representation {intermediate_representation}") logging.info("Calling ICG PiacereInternalToolsIntegrator to add info for PIACERE internal tools") intermediate_representation = PiacereInternalToolsIntegrator.add_internal_tool_information(intermediate_representation) + logging.warning("Force adding sg information in network") ## TODO fix from doml + intermediate_representation = IntermediateRepresentationUtility.force_add_resources_name( + IntermediateRepresentationResources.NETWORKS, + IntermediateRepresentationResources.SECURITY_GROUPS, + intermediate_representation) intermediate_representation_path = "input_file_generated/ir.json" save_file(intermediate_representation, intermediate_representation_path) logging.info(f"Saved intermediate representation at {intermediate_representation_path}") @@ -184,5 +189,6 @@ def create_iac_from_doml_path(model_path, is_multiecore_metamodel, metamodel_dir intermediate_representation = create_intermediate_representation(model_path, is_multiecore_metamodel, metamodel_directory) template_generated_folder = create_iac_from_intermediate_representation(intermediate_representation) + PiacereInternalToolsIntegrator.add_files_for_piacere_internal_tools(template_generated_folder) compress_folder_info = compress_iac_folder(template_generated_folder) return compress_folder_info diff --git a/icgparser/IntermediateRepresentationUtility.py b/icgparser/IntermediateRepresentationUtility.py index 404f973d3d34f1028ad5c314fd3d89e1409c5047..3d021bc06765ba58432c2c1a477ad960d565b084 100644 --- a/icgparser/IntermediateRepresentationUtility.py +++ b/icgparser/IntermediateRepresentationUtility.py @@ -13,6 +13,8 @@ class IntermediateRepresentationResources(NoValue): DATA = 'data' LANGUAGE = "programming_language" VIRTUAL_MACHINES = 'vms' + NETWORKS = "networks" + SECURITY_GROUPS = "computingGroup" def find_objects(object_name, intermediate_representation): @@ -21,12 +23,26 @@ def find_objects(object_name, intermediate_representation): for step in steps: data = step[IntermediateRepresentationResources.DATA.value] if object_name.value in data.keys(): - return data[IntermediateRepresentationResources.VIRTUAL_MACHINES.value] + return data[object_name.value] return [] -def add_step(step, intermediate_representation): +def add_step(step, intermediate_representation, step_number): logging.info("Adding step into intermediate representation") steps = intermediate_representation[IntermediateRepresentationResources.STEPS.value] - steps.append(step) + if step_number: + steps.insert(step_number, step) + else: + steps.append(step) + return intermediate_representation + + +def force_add_resources_name(to_resource, from_resource, intermediate_representation): + sec_groups = find_objects(from_resource, intermediate_representation) + sec_groups_names = [] + for key, sg in sec_groups[0].items(): + if isinstance(sg, dict) and sg["name"]: + sec_groups_names.append(sg["name"]) + for resource in find_objects(to_resource, intermediate_representation): + resource["infra_sgs"] = sec_groups_names return intermediate_representation diff --git a/icgparser/PiacereInternalToolsIntegrator.py b/icgparser/PiacereInternalToolsIntegrator.py index 4b314fdeebf7cf2bca6500e6d307ba5c22b98ec8..722ae5bb9a97084c9fc06c69492d44f6ed5820b9 100644 --- a/icgparser/PiacereInternalToolsIntegrator.py +++ b/icgparser/PiacereInternalToolsIntegrator.py @@ -24,7 +24,8 @@ def extract_info_for_monitoring_agents(intermediate_representation): def add_internal_tool_information(intermediate_representation): monitoring_step = extract_info_for_monitoring_agents(intermediate_representation) intermediate_representation = IntermediateRepresentationUtility.add_step(monitoring_step, - intermediate_representation) + intermediate_representation, + 1) return intermediate_representation diff --git a/icgparser/doml/v1/nginx-openstack_v1.domlx b/icgparser/doml/v1/nginx-openstack_v1.domlx index 1b352ff3b631d2a21a9b4d9570f882b60b02a261..d33ba1d5a7ab5c901296b0fc701edffbcf1eae96 100644 --- a/icgparser/doml/v1/nginx-openstack_v1.domlx +++ b/icgparser/doml/v1/nginx-openstack_v1.domlx @@ -12,7 +12,7 @@ </machineDefinition> </nodes> <networks name="net1" protocol="tcp/ip" addressRange="16.0.0.0/24" connectedIfaces="//@infrastructure/@nodes.0/@machineDefinition/@ifaces.0"/> - <credentials xsi:type="infra:KeyPair" name="ssh_key" user="ubuntu"/> + <credentials xsi:type="infra:KeyPair" name="ssh_key" user="user1"/> <groups xsi:type="infra:SecurityGroup" name="sg" groupedNodes="//@infrastructure/@nodes.0/@machineDefinition" ifaces="//@infrastructure/@nodes.0/@machineDefinition/@ifaces.0"> <rules name="icmp" protocol="icmp" fromPort="-1" toPort="-1"> <cidr>0.0.0.0/0</cidr> @@ -32,7 +32,7 @@ <providers name="openstack"> <vms name="concrete_vm" maps="//@infrastructure/@nodes.0/@machineDefinition"> <annotations xsi:type="commons:SProperty" key="vm_name" value="nginx-host"/> - <annotations xsi:type="commons:SProperty" key="vm_flavor" value="small"/> + <annotations xsi:type="commons:SProperty" key="vm_flavor" value="ubuntu"/> </vms> <networks name="concrete_net" maps="//@infrastructure/@networks.0"> <annotations xsi:type="commons:SProperty" key="name" value="ostack2"/> diff --git a/input_file_generated/ir.json b/input_file_generated/ir.json index 4d6918935c269c58eb10635fe40560f957bde7bf..c0f69cc5bb4f4a395ef0b71fc4ee4c3ffa6920a8 100644 --- a/input_file_generated/ir.json +++ b/input_file_generated/ir.json @@ -51,13 +51,19 @@ "credentials": [ { "infra_element_name": "ssh_key", - "user": "ubuntu" + "user": "user1" } ], "networks": [ { "addressRange": "16.0.0.0/24", "infra_element_name": "net1", + "infra_sgs": [ + "icmp", + "http", + "https", + "ssh" + ], "name": "concrete_net", "protocol": "tcp/ip" } @@ -76,7 +82,7 @@ "infra_element_name": "vm1", "name": "concrete_vm", "os": "Ubuntu-Focal-20.04-Daily-2022-04-19", - "vm_flavor": "small", + "vm_flavor": "ubuntu", "vm_name": "nginx-host" } ] @@ -85,8 +91,7 @@ }, { "data": { - "nginx": { - "name": "nginx", + "piacere_monitoring": { "node": { "credentials": "ssh_key", "group": "sg", @@ -99,18 +104,18 @@ "infra_element_name": "vm1", "name": "concrete_vm", "os": "Ubuntu-Focal-20.04-Daily-2022-04-19", - "vm_flavor": "small", + "vm_flavor": "ubuntu", "vm_name": "nginx-host" - }, - "source_code": "/usr/share/nginx/html/index.html" + } } }, "programming_language": "ansible", - "step_name": "nginx" + "step_name": "piacere_monitoring" }, { "data": { - "piacere_monitoring": { + "nginx": { + "name": "nginx", "node": { "credentials": "ssh_key", "group": "sg", @@ -123,13 +128,14 @@ "infra_element_name": "vm1", "name": "concrete_vm", "os": "Ubuntu-Focal-20.04-Daily-2022-04-19", - "vm_flavor": "small", + "vm_flavor": "ubuntu", "vm_name": "nginx-host" - } + }, + "source_code": "/usr/share/nginx/html/index.html" } }, "programming_language": "ansible", - "step_name": "piacere_monitoring" + "step_name": "nginx" } ] } \ No newline at end of file diff --git a/output_file_example/nginx_openstack/terraform/main.tf b/output_file_example/nginx_openstack/terraform/main.tf index 72457624dd545703ea180157375679f162888c9e..05a347fc6baf5294cefcaad8fac9a4641b7a8231 100644 --- a/output_file_example/nginx_openstack/terraform/main.tf +++ b/output_file_example/nginx_openstack/terraform/main.tf @@ -68,7 +68,7 @@ resource "openstack_networking_port_v2" "nginx" { admin_state_up = true security_group_ids = [ openstack_compute_secgroup_v2.ssh.id, - openstack_compute_secgroup_v2.http.id + openstack_compute_secgroup_v2.http.id, ] fixed_ip { subnet_id = openstack_networking_subnet_v2.nginx.id diff --git a/output_files_generated/nginx_openstack/config.yaml b/output_files_generated/nginx_openstack/config.yaml index 9aa58bc55e5cf8c9366dd3087f48e7c6fa92c6a4..082e5e5f4463eaa91ce35a61957caf2dd029c03d 100644 --- a/output_files_generated/nginx_openstack/config.yaml +++ b/output_files_generated/nginx_openstack/config.yaml @@ -1,6 +1,6 @@ --- iac: - terraform -- nginx - piacere_monitoring +- nginx ... \ No newline at end of file diff --git a/output_files_generated/nginx_openstack/nginx/inventory.j2 b/output_files_generated/nginx_openstack/nginx/inventory.j2 index 7bd4920470d5540d7083fb63b6a94727bbf04e81..b95b2c1b62b32c96567de07cb942b4482158a272 100644 --- a/output_files_generated/nginx_openstack/nginx/inventory.j2 +++ b/output_files_generated/nginx_openstack/nginx/inventory.j2 @@ -5,5 +5,5 @@ [servers_for_nginx:vars] ansible_connection=ssh -ansible_user= -ansible_ssh_private_key_file={{ instance_server_private_key_ }} +ansible_user=ubuntu +ansible_ssh_private_key_file=ssh_key diff --git a/output_files_generated/nginx_openstack/nginx/ssh_key.j2 b/output_files_generated/nginx_openstack/nginx/ssh_key.j2 index 1b9ad076e0fea507ee1cbf04472da28a893fc929..4d512f8d655d1284d5940d5f8f9c3bee1c36f52d 100644 --- a/output_files_generated/nginx_openstack/nginx/ssh_key.j2 +++ b/output_files_generated/nginx_openstack/nginx/ssh_key.j2 @@ -1 +1 @@ -{{ instance_server_private_key }} \ No newline at end of file +{{ instance_server_private_key_ssh_key }} diff --git a/output_files_generated/nginx_openstack/piacere_monitoring/inventory.j2 b/output_files_generated/nginx_openstack/piacere_monitoring/inventory.j2 index 9bd52bbf200909d93fa271990cfeb103f26d35a2..13bf63f916847df970ee5df23e6a75436e70812b 100644 --- a/output_files_generated/nginx_openstack/piacere_monitoring/inventory.j2 +++ b/output_files_generated/nginx_openstack/piacere_monitoring/inventory.j2 @@ -5,5 +5,5 @@ [servers_for_:vars] ansible_connection=ssh -ansible_user= -ansible_ssh_private_key_file={{ instance_server_private_key_ }} +ansible_user=ubuntu +ansible_ssh_private_key_file=ssh_key diff --git a/output_files_generated/nginx_openstack/piacere_monitoring/ssh_key.j2 b/output_files_generated/nginx_openstack/piacere_monitoring/ssh_key.j2 index 1b9ad076e0fea507ee1cbf04472da28a893fc929..4d512f8d655d1284d5940d5f8f9c3bee1c36f52d 100644 --- a/output_files_generated/nginx_openstack/piacere_monitoring/ssh_key.j2 +++ b/output_files_generated/nginx_openstack/piacere_monitoring/ssh_key.j2 @@ -1 +1 @@ -{{ instance_server_private_key }} \ No newline at end of file +{{ instance_server_private_key_ssh_key }} diff --git a/output_files_generated/nginx_openstack/terraform/config.yaml b/output_files_generated/nginx_openstack/terraform/config.yaml index 405c5565b8f41ad862f87fd7a9478d3817c990c0..3262fae672be5908e50e662a0bd77f002a9594c6 100644 --- a/output_files_generated/nginx_openstack/terraform/config.yaml +++ b/output_files_generated/nginx_openstack/terraform/config.yaml @@ -9,8 +9,8 @@ input: - OS_PROJECT_NAME output: - - instance_server_public_key_ - - instance_server_private_key_ + - instance_server_public_key_ssh_key + - instance_server_private_key_ssh_key - instance_ip_nginx-host ... diff --git a/output_files_generated/nginx_openstack/terraform/main.tf b/output_files_generated/nginx_openstack/terraform/main.tf index e9ef70a729bbeb8bb71c124cc9d6fe7f1e372443..30cac937f1e2b1df0b39e448b366bb84359040e2 100644 --- a/output_files_generated/nginx_openstack/terraform/main.tf +++ b/output_files_generated/nginx_openstack/terraform/main.tf @@ -12,10 +12,6 @@ required_version = ">= 0.14.0" # Configure the OpenStack Provider provider "openstack" { - #user_name = var.openstack_username - #tenant_name = "admin" - #password = var.openstack_password - #auth_url = var.openstack_auth_url insecure = true } @@ -24,42 +20,16 @@ data "openstack_networking_network_v2" "external" { name = "external" } -data "openstack_identity_project_v3" "test_tenant" { - name = "admin" -} - -data "openstack_networking_secgroup_v2" "default" { - name = "default" - tenant_id = data.openstack_identity_project_v3.test_tenant.id -} - # Create virtual machine resource "openstack_compute_instance_v2" "vm1" { name = "nginx-host" image_name = "Ubuntu-Focal-20.04-Daily-2022-04-19" - flavor_name = "small" + flavor_name = "ubuntu" key_pair = openstack_compute_keypair_v2.ssh_key.name network { port = openstack_networking_port_v2.net1.id } - - ## AGENTS TO ADD - # this is subject to be moved to IEM as part of its baseline - provisioner "local-exec" { - command = "ansible-galaxy collection install community.general" - } - - # this is subject to be moved to IEM as part of its baseline - provisioner "local-exec" { - command = "ansible-playbook ansible/playbooks/pma/site_requirements.yaml" - } - - # secrets can be taken from environment variables at IEM but these security issues I will leave them to y2, the user can also be problematic ubuntu/root/centos/... - provisioner "local-exec" { - command = "ansible-playbook -u root -i '${openstack_networking_floatingip_v2.vm1_floating_ip.address},' ansible/playbooks/pma/site.yaml --extra-vars '{\"pma_deployment_id\": \"123e4567-e89b-12d3-a456-426614174002\", \"pma_influxdb_bucket\": \"bucket\", \"pma_influxdb_token\": \"piacerePassword\", \"pma_influxdb_org\": \"piacere\", \"pma_influxdb_addr\": \"https://influxdb.pm.ci.piacere.digital.tecnalia.dev\" }'" - } - } # Create floating ip @@ -97,7 +67,11 @@ resource "openstack_networking_port_v2" "net1" { network_id = openstack_networking_network_v2.net1.id admin_state_up = true security_group_ids = [ - data.openstack_networking_secgroup_v2.default.id #default flavour id + openstack_compute_secgroup_v2.icmp.id, + openstack_compute_secgroup_v2.http.id, + openstack_compute_secgroup_v2.https.id, + openstack_compute_secgroup_v2.ssh.id, + ] fixed_ip { subnet_id = openstack_networking_subnet_v2.net1_subnet.id @@ -168,7 +142,7 @@ resource "openstack_compute_secgroup_v2" "ssh" { # Create ssh keys resource "openstack_compute_keypair_v2" "ssh_key" { - name = "ubuntu" - # public_key = "ubuntu" + name = "user1" + # public_key = "user1" } diff --git a/plugin/AnsiblePlugin.py b/plugin/AnsiblePlugin.py index 493eced74e546829a2fbb1da40ea11f6a93a8816..bac5a9134ae6cfc0a8490cb06626227b0a3378fd 100644 --- a/plugin/AnsiblePlugin.py +++ b/plugin/AnsiblePlugin.py @@ -38,7 +38,7 @@ def find_operating_system(parameters): raise PluginResourceNotFoundError(plugin_name="AnsiblePlugin", resource_name="operating system") -def create_inventory_file(parameters, language, operating_system, template_name): +def create_template_file(parameters, language, operating_system, template_name): inventory_template_path = TemplateUtils.find_template_path(language, operating_system, template_name) template = TemplateUtils.read_template(inventory_template_path) template_filled = TemplateUtils.edit_template(template, parameters) @@ -65,12 +65,13 @@ def create_files(step, output_path): template = TemplateUtils.read_template(ansible_template_path) template_filled = TemplateUtils.edit_template(template, resource_params) - inventory_template_filled = create_inventory_file(resource_params, language, operating_system, "inventory") - config_template_filled = create_inventory_file(resource_params, language, operating_system, "config") + inventory_template_filled = create_template_file(resource_params, language, operating_system, "inventory") + config_template_filled = create_template_file(resource_params, language, operating_system, "config") + ssh_key_template_filled = create_template_file(resource_params, language, operating_system, "ssh_key") TemplateUtils.write_template(inventory_template_filled, inventory_output_file_path) TemplateUtils.write_template(template_filled, ansible_output_file_path) TemplateUtils.write_template(config_template_filled, config_output_file_path) - TemplateUtils.write_template("{{ instance_server_private_key }}", ssh_key_output_file_path) + TemplateUtils.write_template(ssh_key_template_filled, ssh_key_output_file_path) logging.info("File available at: {}".format(output_path)) diff --git a/template-location.properties b/template-location.properties index b9fa63fa5af690e19ef6782a6f19b1d06ef9a4ad..d017685758deeaaba2b69537f9a5e13c3fd95f05 100644 --- a/template-location.properties +++ b/template-location.properties @@ -36,6 +36,7 @@ computingGroup = templates/terraform/aws/port_rule.tpl [ansible.ubuntu] inventory = templates/ansible/ubuntu/inventory.tpl +ssh_key = templates/ansible/ubuntu/ssh_key.tpl config = templates/ansible/ubuntu/config.tpl nginx = templates/ansible/ubuntu/nginx.tpl mysql = templates/ansible/ubuntu/mysql.tpl diff --git a/templates/ansible/ubuntu/inventory.tpl b/templates/ansible/ubuntu/inventory.tpl index 4c7e43e6ed7864f8890276cc1c92d6b83a099151..9f59b2a0f26e2a34b1a6dba600c0a2dd307bd2c8 100644 --- a/templates/ansible/ubuntu/inventory.tpl +++ b/templates/ansible/ubuntu/inventory.tpl @@ -19,5 +19,5 @@ [{{ "servers_for_" ~ name }}:vars] ansible_connection=ssh -ansible_user={{node.vm_key_name}} -ansible_ssh_private_key_file={% raw %}{{ instance_server_private_key_{% endraw %}{{ node.vm_key_name }} {% raw %}}}{% endraw %} +ansible_user=ubuntu +ansible_ssh_private_key_file=ssh_key diff --git a/templates/ansible/ubuntu/ssh_key.tpl b/templates/ansible/ubuntu/ssh_key.tpl new file mode 100644 index 0000000000000000000000000000000000000000..798ef200ade5df0158a3316e588d7ffb0f57b25f --- /dev/null +++ b/templates/ansible/ubuntu/ssh_key.tpl @@ -0,0 +1 @@ +{% raw %}{{ instance_server_private_key_{% endraw %}{{ node.credentials }} {% raw %}}}{% endraw %} \ No newline at end of file diff --git a/templates/terraform/open_stack/config.tpl b/templates/terraform/open_stack/config.tpl index c0f56cf15650523821796dc1c7099b8a5a9419a9..a50ebb71b9ec2dc84e8c3a0230ea26dcae6b767f 100644 --- a/templates/terraform/open_stack/config.tpl +++ b/templates/terraform/open_stack/config.tpl @@ -23,8 +23,8 @@ input: - OS_PROJECT_NAME output: {% for vm in vms %} - - instance_server_public_key_{{ vm.vm_key_name }} - - instance_server_private_key_{{ vm.vm_key_name }} + - instance_server_public_key_{{ vm.credentials }} + - instance_server_private_key_{{ vm.credentials }} - instance_ip_{{ vm.vm_name }} {% endfor %} ... diff --git a/templates/terraform/open_stack/init.tpl b/templates/terraform/open_stack/init.tpl index 515d44eed842e2f125918d4834038fb5807d08cc..ac2ff2434722f4ed77f094b412c7829bd87b1ca7 100644 --- a/templates/terraform/open_stack/init.tpl +++ b/templates/terraform/open_stack/init.tpl @@ -26,10 +26,6 @@ required_version = ">= 0.14.0" # Configure the OpenStack Provider provider "openstack" { - #user_name = var.openstack_username - #tenant_name = "admin" - #password = var.openstack_password - #auth_url = var.openstack_auth_url insecure = true } @@ -37,12 +33,3 @@ provider "openstack" { data "openstack_networking_network_v2" "external" { name = "external" } - -data "openstack_identity_project_v3" "test_tenant" { - name = "admin" -} - -data "openstack_networking_secgroup_v2" "default" { - name = "default" - tenant_id = data.openstack_identity_project_v3.test_tenant.id -} \ No newline at end of file diff --git a/templates/terraform/open_stack/network.tpl b/templates/terraform/open_stack/network.tpl index 09fa76496bad14fbad02d377989cd6318f0cf0d9..6ab4d6a3454ed4acf2050445973ca19d7388df2e 100644 --- a/templates/terraform/open_stack/network.tpl +++ b/templates/terraform/open_stack/network.tpl @@ -35,7 +35,8 @@ resource "openstack_networking_port_v2" "{{ infra_element_name }}" { network_id = openstack_networking_network_v2.{{ infra_element_name }}.id admin_state_up = true security_group_ids = [ - data.openstack_networking_secgroup_v2.default.id #default flavour id + {% for sg in infra_sgs %}openstack_compute_secgroup_v2.{{sg}}.id, + {% endfor %} ] fixed_ip { subnet_id = openstack_networking_subnet_v2.{{ infra_element_name ~ "_subnet" }}.id diff --git a/templates/terraform/open_stack/virtual_machine.tpl b/templates/terraform/open_stack/virtual_machine.tpl index 68e6f9be458aa4ca8f7b188ed26e7e58dbe8915c..9e3777265b91c5dae523c75ccadb7592977944bb 100644 --- a/templates/terraform/open_stack/virtual_machine.tpl +++ b/templates/terraform/open_stack/virtual_machine.tpl @@ -23,23 +23,6 @@ resource "openstack_compute_instance_v2" "{{ infra_element_name }}" { network { port = openstack_networking_port_v2.{{ i1.belongsTo }}.id } - - ## AGENTS TO ADD - # this is subject to be moved to IEM as part of its baseline - provisioner "local-exec" { - command = "ansible-galaxy collection install community.general" - } - - # this is subject to be moved to IEM as part of its baseline - provisioner "local-exec" { - command = "ansible-playbook ansible/playbooks/pma/site_requirements.yaml" - } - - # secrets can be taken from environment variables at IEM but these security issues I will leave them to y2, the user can also be problematic ubuntu/root/centos/... - provisioner "local-exec" { - command = "ansible-playbook -u root -i '${openstack_networking_floatingip_v2.{{ infra_element_name ~ "_floating_ip"}}.address},' ansible/playbooks/pma/site.yaml --extra-vars '{\"pma_deployment_id\": \"123e4567-e89b-12d3-a456-426614174002\", \"pma_influxdb_bucket\": \"bucket\", \"pma_influxdb_token\": \"piacerePassword\", \"pma_influxdb_org\": \"piacere\", \"pma_influxdb_addr\": \"https://influxdb.pm.ci.piacere.digital.tecnalia.dev\" }'" - } - } # Create floating ip