From 36a30b15e8fba6c909ec821f0878cef22fadc32f Mon Sep 17 00:00:00 2001 From: Debora Benedetto <debora.benedetto@hpe.com> Date: Thu, 28 Jul 2022 15:35:09 +0200 Subject: [PATCH] add template for ssh_key ansible generation, bugfix template replacement --- controller/Orchestrator.py | 8 +++- .../IntermediateRepresentationUtility.py | 22 ++++++++-- icgparser/PiacereInternalToolsIntegrator.py | 3 +- icgparser/doml/v1/nginx-openstack_v1.domlx | 4 +- input_file_generated/ir.json | 30 +++++++------ .../nginx_openstack/terraform/main.tf | 2 +- .../nginx_openstack/config.yaml | 2 +- .../nginx_openstack/nginx/inventory.j2 | 4 +- .../nginx_openstack/nginx/ssh_key.j2 | 2 +- .../piacere_monitoring/inventory.j2 | 4 +- .../piacere_monitoring/ssh_key.j2 | 2 +- .../nginx_openstack/terraform/config.yaml | 4 +- .../nginx_openstack/terraform/main.tf | 42 ++++--------------- plugin/AnsiblePlugin.py | 9 ++-- template-location.properties | 1 + templates/ansible/ubuntu/inventory.tpl | 4 +- templates/ansible/ubuntu/ssh_key.tpl | 1 + templates/terraform/open_stack/config.tpl | 4 +- templates/terraform/open_stack/init.tpl | 13 ------ templates/terraform/open_stack/network.tpl | 3 +- .../terraform/open_stack/virtual_machine.tpl | 17 -------- 21 files changed, 79 insertions(+), 102 deletions(-) create mode 100644 templates/ansible/ubuntu/ssh_key.tpl diff --git a/controller/Orchestrator.py b/controller/Orchestrator.py index 95cba24..f806073 100644 --- a/controller/Orchestrator.py +++ b/controller/Orchestrator.py @@ -21,7 +21,7 @@ import time import uuid import yaml -from icgparser import ModelParser, PiacereInternalToolsIntegrator +from icgparser import ModelParser, PiacereInternalToolsIntegrator, IntermediateRepresentationUtility from icgparser.IntermediateRepresentationUtility import IntermediateRepresentationResources from plugin import AnsiblePlugin, TerraformPlugin from utility.FileParsingUtility import replace_none_with_empty_str @@ -123,6 +123,11 @@ def create_intermediate_representation(model_path, is_multiecore_metamodel, meta logging.info(f"Successfully created intermediate representation {intermediate_representation}") logging.info("Calling ICG PiacereInternalToolsIntegrator to add info for PIACERE internal tools") intermediate_representation = PiacereInternalToolsIntegrator.add_internal_tool_information(intermediate_representation) + logging.warning("Force adding sg information in network") ## TODO fix from doml + intermediate_representation = IntermediateRepresentationUtility.force_add_resources_name( + IntermediateRepresentationResources.NETWORKS, + IntermediateRepresentationResources.SECURITY_GROUPS, + intermediate_representation) intermediate_representation_path = "input_file_generated/ir.json" save_file(intermediate_representation, intermediate_representation_path) logging.info(f"Saved intermediate representation at {intermediate_representation_path}") @@ -184,5 +189,6 @@ def create_iac_from_doml_path(model_path, is_multiecore_metamodel, metamodel_dir intermediate_representation = create_intermediate_representation(model_path, is_multiecore_metamodel, metamodel_directory) template_generated_folder = create_iac_from_intermediate_representation(intermediate_representation) + PiacereInternalToolsIntegrator.add_files_for_piacere_internal_tools(template_generated_folder) compress_folder_info = compress_iac_folder(template_generated_folder) return compress_folder_info diff --git a/icgparser/IntermediateRepresentationUtility.py b/icgparser/IntermediateRepresentationUtility.py index 404f973..3d021bc 100644 --- a/icgparser/IntermediateRepresentationUtility.py +++ b/icgparser/IntermediateRepresentationUtility.py @@ -13,6 +13,8 @@ class IntermediateRepresentationResources(NoValue): DATA = 'data' LANGUAGE = "programming_language" VIRTUAL_MACHINES = 'vms' + NETWORKS = "networks" + SECURITY_GROUPS = "computingGroup" def find_objects(object_name, intermediate_representation): @@ -21,12 +23,26 @@ def find_objects(object_name, intermediate_representation): for step in steps: data = step[IntermediateRepresentationResources.DATA.value] if object_name.value in data.keys(): - return data[IntermediateRepresentationResources.VIRTUAL_MACHINES.value] + return data[object_name.value] return [] -def add_step(step, intermediate_representation): +def add_step(step, intermediate_representation, step_number): logging.info("Adding step into intermediate representation") steps = intermediate_representation[IntermediateRepresentationResources.STEPS.value] - steps.append(step) + if step_number: + steps.insert(step_number, step) + else: + steps.append(step) + return intermediate_representation + + +def force_add_resources_name(to_resource, from_resource, intermediate_representation): + sec_groups = find_objects(from_resource, intermediate_representation) + sec_groups_names = [] + for key, sg in sec_groups[0].items(): + if isinstance(sg, dict) and sg["name"]: + sec_groups_names.append(sg["name"]) + for resource in find_objects(to_resource, intermediate_representation): + resource["infra_sgs"] = sec_groups_names return intermediate_representation diff --git a/icgparser/PiacereInternalToolsIntegrator.py b/icgparser/PiacereInternalToolsIntegrator.py index 4b314fd..722ae5b 100644 --- a/icgparser/PiacereInternalToolsIntegrator.py +++ b/icgparser/PiacereInternalToolsIntegrator.py @@ -24,7 +24,8 @@ def extract_info_for_monitoring_agents(intermediate_representation): def add_internal_tool_information(intermediate_representation): monitoring_step = extract_info_for_monitoring_agents(intermediate_representation) intermediate_representation = IntermediateRepresentationUtility.add_step(monitoring_step, - intermediate_representation) + intermediate_representation, + 1) return intermediate_representation diff --git a/icgparser/doml/v1/nginx-openstack_v1.domlx b/icgparser/doml/v1/nginx-openstack_v1.domlx index 1b352ff..d33ba1d 100644 --- a/icgparser/doml/v1/nginx-openstack_v1.domlx +++ b/icgparser/doml/v1/nginx-openstack_v1.domlx @@ -12,7 +12,7 @@ </machineDefinition> </nodes> <networks name="net1" protocol="tcp/ip" addressRange="16.0.0.0/24" connectedIfaces="//@infrastructure/@nodes.0/@machineDefinition/@ifaces.0"/> - <credentials xsi:type="infra:KeyPair" name="ssh_key" user="ubuntu"/> + <credentials xsi:type="infra:KeyPair" name="ssh_key" user="user1"/> <groups xsi:type="infra:SecurityGroup" name="sg" groupedNodes="//@infrastructure/@nodes.0/@machineDefinition" ifaces="//@infrastructure/@nodes.0/@machineDefinition/@ifaces.0"> <rules name="icmp" protocol="icmp" fromPort="-1" toPort="-1"> <cidr>0.0.0.0/0</cidr> @@ -32,7 +32,7 @@ <providers name="openstack"> <vms name="concrete_vm" maps="//@infrastructure/@nodes.0/@machineDefinition"> <annotations xsi:type="commons:SProperty" key="vm_name" value="nginx-host"/> - <annotations xsi:type="commons:SProperty" key="vm_flavor" value="small"/> + <annotations xsi:type="commons:SProperty" key="vm_flavor" value="ubuntu"/> </vms> <networks name="concrete_net" maps="//@infrastructure/@networks.0"> <annotations xsi:type="commons:SProperty" key="name" value="ostack2"/> diff --git a/input_file_generated/ir.json b/input_file_generated/ir.json index 4d69189..c0f69cc 100644 --- a/input_file_generated/ir.json +++ b/input_file_generated/ir.json @@ -51,13 +51,19 @@ "credentials": [ { "infra_element_name": "ssh_key", - "user": "ubuntu" + "user": "user1" } ], "networks": [ { "addressRange": "16.0.0.0/24", "infra_element_name": "net1", + "infra_sgs": [ + "icmp", + "http", + "https", + "ssh" + ], "name": "concrete_net", "protocol": "tcp/ip" } @@ -76,7 +82,7 @@ "infra_element_name": "vm1", "name": "concrete_vm", "os": "Ubuntu-Focal-20.04-Daily-2022-04-19", - "vm_flavor": "small", + "vm_flavor": "ubuntu", "vm_name": "nginx-host" } ] @@ -85,8 +91,7 @@ }, { "data": { - "nginx": { - "name": "nginx", + "piacere_monitoring": { "node": { "credentials": "ssh_key", "group": "sg", @@ -99,18 +104,18 @@ "infra_element_name": "vm1", "name": "concrete_vm", "os": "Ubuntu-Focal-20.04-Daily-2022-04-19", - "vm_flavor": "small", + "vm_flavor": "ubuntu", "vm_name": "nginx-host" - }, - "source_code": "/usr/share/nginx/html/index.html" + } } }, "programming_language": "ansible", - "step_name": "nginx" + "step_name": "piacere_monitoring" }, { "data": { - "piacere_monitoring": { + "nginx": { + "name": "nginx", "node": { "credentials": "ssh_key", "group": "sg", @@ -123,13 +128,14 @@ "infra_element_name": "vm1", "name": "concrete_vm", "os": "Ubuntu-Focal-20.04-Daily-2022-04-19", - "vm_flavor": "small", + "vm_flavor": "ubuntu", "vm_name": "nginx-host" - } + }, + "source_code": "/usr/share/nginx/html/index.html" } }, "programming_language": "ansible", - "step_name": "piacere_monitoring" + "step_name": "nginx" } ] } \ No newline at end of file diff --git a/output_file_example/nginx_openstack/terraform/main.tf b/output_file_example/nginx_openstack/terraform/main.tf index 7245762..05a347f 100644 --- a/output_file_example/nginx_openstack/terraform/main.tf +++ b/output_file_example/nginx_openstack/terraform/main.tf @@ -68,7 +68,7 @@ resource "openstack_networking_port_v2" "nginx" { admin_state_up = true security_group_ids = [ openstack_compute_secgroup_v2.ssh.id, - openstack_compute_secgroup_v2.http.id + openstack_compute_secgroup_v2.http.id, ] fixed_ip { subnet_id = openstack_networking_subnet_v2.nginx.id diff --git a/output_files_generated/nginx_openstack/config.yaml b/output_files_generated/nginx_openstack/config.yaml index 9aa58bc..082e5e5 100644 --- a/output_files_generated/nginx_openstack/config.yaml +++ b/output_files_generated/nginx_openstack/config.yaml @@ -1,6 +1,6 @@ --- iac: - terraform -- nginx - piacere_monitoring +- nginx ... \ No newline at end of file diff --git a/output_files_generated/nginx_openstack/nginx/inventory.j2 b/output_files_generated/nginx_openstack/nginx/inventory.j2 index 7bd4920..b95b2c1 100644 --- a/output_files_generated/nginx_openstack/nginx/inventory.j2 +++ b/output_files_generated/nginx_openstack/nginx/inventory.j2 @@ -5,5 +5,5 @@ [servers_for_nginx:vars] ansible_connection=ssh -ansible_user= -ansible_ssh_private_key_file={{ instance_server_private_key_ }} +ansible_user=ubuntu +ansible_ssh_private_key_file=ssh_key diff --git a/output_files_generated/nginx_openstack/nginx/ssh_key.j2 b/output_files_generated/nginx_openstack/nginx/ssh_key.j2 index 1b9ad07..4d512f8 100644 --- a/output_files_generated/nginx_openstack/nginx/ssh_key.j2 +++ b/output_files_generated/nginx_openstack/nginx/ssh_key.j2 @@ -1 +1 @@ -{{ instance_server_private_key }} \ No newline at end of file +{{ instance_server_private_key_ssh_key }} diff --git a/output_files_generated/nginx_openstack/piacere_monitoring/inventory.j2 b/output_files_generated/nginx_openstack/piacere_monitoring/inventory.j2 index 9bd52bb..13bf63f 100644 --- a/output_files_generated/nginx_openstack/piacere_monitoring/inventory.j2 +++ b/output_files_generated/nginx_openstack/piacere_monitoring/inventory.j2 @@ -5,5 +5,5 @@ [servers_for_:vars] ansible_connection=ssh -ansible_user= -ansible_ssh_private_key_file={{ instance_server_private_key_ }} +ansible_user=ubuntu +ansible_ssh_private_key_file=ssh_key diff --git a/output_files_generated/nginx_openstack/piacere_monitoring/ssh_key.j2 b/output_files_generated/nginx_openstack/piacere_monitoring/ssh_key.j2 index 1b9ad07..4d512f8 100644 --- a/output_files_generated/nginx_openstack/piacere_monitoring/ssh_key.j2 +++ b/output_files_generated/nginx_openstack/piacere_monitoring/ssh_key.j2 @@ -1 +1 @@ -{{ instance_server_private_key }} \ No newline at end of file +{{ instance_server_private_key_ssh_key }} diff --git a/output_files_generated/nginx_openstack/terraform/config.yaml b/output_files_generated/nginx_openstack/terraform/config.yaml index 405c556..3262fae 100644 --- a/output_files_generated/nginx_openstack/terraform/config.yaml +++ b/output_files_generated/nginx_openstack/terraform/config.yaml @@ -9,8 +9,8 @@ input: - OS_PROJECT_NAME output: - - instance_server_public_key_ - - instance_server_private_key_ + - instance_server_public_key_ssh_key + - instance_server_private_key_ssh_key - instance_ip_nginx-host ... diff --git a/output_files_generated/nginx_openstack/terraform/main.tf b/output_files_generated/nginx_openstack/terraform/main.tf index e9ef70a..30cac93 100644 --- a/output_files_generated/nginx_openstack/terraform/main.tf +++ b/output_files_generated/nginx_openstack/terraform/main.tf @@ -12,10 +12,6 @@ required_version = ">= 0.14.0" # Configure the OpenStack Provider provider "openstack" { - #user_name = var.openstack_username - #tenant_name = "admin" - #password = var.openstack_password - #auth_url = var.openstack_auth_url insecure = true } @@ -24,42 +20,16 @@ data "openstack_networking_network_v2" "external" { name = "external" } -data "openstack_identity_project_v3" "test_tenant" { - name = "admin" -} - -data "openstack_networking_secgroup_v2" "default" { - name = "default" - tenant_id = data.openstack_identity_project_v3.test_tenant.id -} - # Create virtual machine resource "openstack_compute_instance_v2" "vm1" { name = "nginx-host" image_name = "Ubuntu-Focal-20.04-Daily-2022-04-19" - flavor_name = "small" + flavor_name = "ubuntu" key_pair = openstack_compute_keypair_v2.ssh_key.name network { port = openstack_networking_port_v2.net1.id } - - ## AGENTS TO ADD - # this is subject to be moved to IEM as part of its baseline - provisioner "local-exec" { - command = "ansible-galaxy collection install community.general" - } - - # this is subject to be moved to IEM as part of its baseline - provisioner "local-exec" { - command = "ansible-playbook ansible/playbooks/pma/site_requirements.yaml" - } - - # secrets can be taken from environment variables at IEM but these security issues I will leave them to y2, the user can also be problematic ubuntu/root/centos/... - provisioner "local-exec" { - command = "ansible-playbook -u root -i '${openstack_networking_floatingip_v2.vm1_floating_ip.address},' ansible/playbooks/pma/site.yaml --extra-vars '{\"pma_deployment_id\": \"123e4567-e89b-12d3-a456-426614174002\", \"pma_influxdb_bucket\": \"bucket\", \"pma_influxdb_token\": \"piacerePassword\", \"pma_influxdb_org\": \"piacere\", \"pma_influxdb_addr\": \"https://influxdb.pm.ci.piacere.digital.tecnalia.dev\" }'" - } - } # Create floating ip @@ -97,7 +67,11 @@ resource "openstack_networking_port_v2" "net1" { network_id = openstack_networking_network_v2.net1.id admin_state_up = true security_group_ids = [ - data.openstack_networking_secgroup_v2.default.id #default flavour id + openstack_compute_secgroup_v2.icmp.id, + openstack_compute_secgroup_v2.http.id, + openstack_compute_secgroup_v2.https.id, + openstack_compute_secgroup_v2.ssh.id, + ] fixed_ip { subnet_id = openstack_networking_subnet_v2.net1_subnet.id @@ -168,7 +142,7 @@ resource "openstack_compute_secgroup_v2" "ssh" { # Create ssh keys resource "openstack_compute_keypair_v2" "ssh_key" { - name = "ubuntu" - # public_key = "ubuntu" + name = "user1" + # public_key = "user1" } diff --git a/plugin/AnsiblePlugin.py b/plugin/AnsiblePlugin.py index 493eced..bac5a91 100644 --- a/plugin/AnsiblePlugin.py +++ b/plugin/AnsiblePlugin.py @@ -38,7 +38,7 @@ def find_operating_system(parameters): raise PluginResourceNotFoundError(plugin_name="AnsiblePlugin", resource_name="operating system") -def create_inventory_file(parameters, language, operating_system, template_name): +def create_template_file(parameters, language, operating_system, template_name): inventory_template_path = TemplateUtils.find_template_path(language, operating_system, template_name) template = TemplateUtils.read_template(inventory_template_path) template_filled = TemplateUtils.edit_template(template, parameters) @@ -65,12 +65,13 @@ def create_files(step, output_path): template = TemplateUtils.read_template(ansible_template_path) template_filled = TemplateUtils.edit_template(template, resource_params) - inventory_template_filled = create_inventory_file(resource_params, language, operating_system, "inventory") - config_template_filled = create_inventory_file(resource_params, language, operating_system, "config") + inventory_template_filled = create_template_file(resource_params, language, operating_system, "inventory") + config_template_filled = create_template_file(resource_params, language, operating_system, "config") + ssh_key_template_filled = create_template_file(resource_params, language, operating_system, "ssh_key") TemplateUtils.write_template(inventory_template_filled, inventory_output_file_path) TemplateUtils.write_template(template_filled, ansible_output_file_path) TemplateUtils.write_template(config_template_filled, config_output_file_path) - TemplateUtils.write_template("{{ instance_server_private_key }}", ssh_key_output_file_path) + TemplateUtils.write_template(ssh_key_template_filled, ssh_key_output_file_path) logging.info("File available at: {}".format(output_path)) diff --git a/template-location.properties b/template-location.properties index b9fa63f..d017685 100644 --- a/template-location.properties +++ b/template-location.properties @@ -36,6 +36,7 @@ computingGroup = templates/terraform/aws/port_rule.tpl [ansible.ubuntu] inventory = templates/ansible/ubuntu/inventory.tpl +ssh_key = templates/ansible/ubuntu/ssh_key.tpl config = templates/ansible/ubuntu/config.tpl nginx = templates/ansible/ubuntu/nginx.tpl mysql = templates/ansible/ubuntu/mysql.tpl diff --git a/templates/ansible/ubuntu/inventory.tpl b/templates/ansible/ubuntu/inventory.tpl index 4c7e43e..9f59b2a 100644 --- a/templates/ansible/ubuntu/inventory.tpl +++ b/templates/ansible/ubuntu/inventory.tpl @@ -19,5 +19,5 @@ [{{ "servers_for_" ~ name }}:vars] ansible_connection=ssh -ansible_user={{node.vm_key_name}} -ansible_ssh_private_key_file={% raw %}{{ instance_server_private_key_{% endraw %}{{ node.vm_key_name }} {% raw %}}}{% endraw %} +ansible_user=ubuntu +ansible_ssh_private_key_file=ssh_key diff --git a/templates/ansible/ubuntu/ssh_key.tpl b/templates/ansible/ubuntu/ssh_key.tpl new file mode 100644 index 0000000..798ef20 --- /dev/null +++ b/templates/ansible/ubuntu/ssh_key.tpl @@ -0,0 +1 @@ +{% raw %}{{ instance_server_private_key_{% endraw %}{{ node.credentials }} {% raw %}}}{% endraw %} \ No newline at end of file diff --git a/templates/terraform/open_stack/config.tpl b/templates/terraform/open_stack/config.tpl index c0f56cf..a50ebb7 100644 --- a/templates/terraform/open_stack/config.tpl +++ b/templates/terraform/open_stack/config.tpl @@ -23,8 +23,8 @@ input: - OS_PROJECT_NAME output: {% for vm in vms %} - - instance_server_public_key_{{ vm.vm_key_name }} - - instance_server_private_key_{{ vm.vm_key_name }} + - instance_server_public_key_{{ vm.credentials }} + - instance_server_private_key_{{ vm.credentials }} - instance_ip_{{ vm.vm_name }} {% endfor %} ... diff --git a/templates/terraform/open_stack/init.tpl b/templates/terraform/open_stack/init.tpl index 515d44e..ac2ff24 100644 --- a/templates/terraform/open_stack/init.tpl +++ b/templates/terraform/open_stack/init.tpl @@ -26,10 +26,6 @@ required_version = ">= 0.14.0" # Configure the OpenStack Provider provider "openstack" { - #user_name = var.openstack_username - #tenant_name = "admin" - #password = var.openstack_password - #auth_url = var.openstack_auth_url insecure = true } @@ -37,12 +33,3 @@ provider "openstack" { data "openstack_networking_network_v2" "external" { name = "external" } - -data "openstack_identity_project_v3" "test_tenant" { - name = "admin" -} - -data "openstack_networking_secgroup_v2" "default" { - name = "default" - tenant_id = data.openstack_identity_project_v3.test_tenant.id -} \ No newline at end of file diff --git a/templates/terraform/open_stack/network.tpl b/templates/terraform/open_stack/network.tpl index 09fa764..6ab4d6a 100644 --- a/templates/terraform/open_stack/network.tpl +++ b/templates/terraform/open_stack/network.tpl @@ -35,7 +35,8 @@ resource "openstack_networking_port_v2" "{{ infra_element_name }}" { network_id = openstack_networking_network_v2.{{ infra_element_name }}.id admin_state_up = true security_group_ids = [ - data.openstack_networking_secgroup_v2.default.id #default flavour id + {% for sg in infra_sgs %}openstack_compute_secgroup_v2.{{sg}}.id, + {% endfor %} ] fixed_ip { subnet_id = openstack_networking_subnet_v2.{{ infra_element_name ~ "_subnet" }}.id diff --git a/templates/terraform/open_stack/virtual_machine.tpl b/templates/terraform/open_stack/virtual_machine.tpl index 68e6f9b..9e37772 100644 --- a/templates/terraform/open_stack/virtual_machine.tpl +++ b/templates/terraform/open_stack/virtual_machine.tpl @@ -23,23 +23,6 @@ resource "openstack_compute_instance_v2" "{{ infra_element_name }}" { network { port = openstack_networking_port_v2.{{ i1.belongsTo }}.id } - - ## AGENTS TO ADD - # this is subject to be moved to IEM as part of its baseline - provisioner "local-exec" { - command = "ansible-galaxy collection install community.general" - } - - # this is subject to be moved to IEM as part of its baseline - provisioner "local-exec" { - command = "ansible-playbook ansible/playbooks/pma/site_requirements.yaml" - } - - # secrets can be taken from environment variables at IEM but these security issues I will leave them to y2, the user can also be problematic ubuntu/root/centos/... - provisioner "local-exec" { - command = "ansible-playbook -u root -i '${openstack_networking_floatingip_v2.{{ infra_element_name ~ "_floating_ip"}}.address},' ansible/playbooks/pma/site.yaml --extra-vars '{\"pma_deployment_id\": \"123e4567-e89b-12d3-a456-426614174002\", \"pma_influxdb_bucket\": \"bucket\", \"pma_influxdb_token\": \"piacerePassword\", \"pma_influxdb_org\": \"piacere\", \"pma_influxdb_addr\": \"https://influxdb.pm.ci.piacere.digital.tecnalia.dev\" }'" - } - } # Create floating ip -- GitLab