diff --git a/controller/Orchestrator.py b/controller/Orchestrator.py index d70f3f742f4e281d195f7c5e7ac802dd1d0ef556..95cba247636c4462b4d616e73e20e30d90e43a0a 100644 --- a/controller/Orchestrator.py +++ b/controller/Orchestrator.py @@ -21,7 +21,8 @@ import time import uuid import yaml -from icgparser import ModelParser +from icgparser import ModelParser, PiacereInternalToolsIntegrator +from icgparser.IntermediateRepresentationUtility import IntermediateRepresentationResources from plugin import AnsiblePlugin, TerraformPlugin from utility.FileParsingUtility import replace_none_with_empty_str @@ -46,9 +47,10 @@ def choose_plugin(parameters, template_generated_folder): for step in parameters["steps"]: if step["programming_language"] == "ansible": logging.info("Ansible Plugin chosen") - metadata_root_folder["iac"].append("ansible") - input_data = step["data"] - AnsiblePlugin.create_files(input_data, template_generated_folder) + step_name = step[IntermediateRepresentationResources.STEP_NAME.value] + metadata_root_folder["iac"].append(step_name) + # input_data = step["data"] + AnsiblePlugin.create_files(step, template_generated_folder) elif step["programming_language"] == "terraform": logging.info("Terraform Plugin chosen") metadata_root_folder["iac"].append("terraform") @@ -119,6 +121,8 @@ def create_intermediate_representation(model_path, is_multiecore_metamodel, meta metamodel_directory=metamodel_directory) # intermediate_representation = reorganize_info(intermediate_representation) logging.info(f"Successfully created intermediate representation {intermediate_representation}") + logging.info("Calling ICG PiacereInternalToolsIntegrator to add info for PIACERE internal tools") + intermediate_representation = PiacereInternalToolsIntegrator.add_internal_tool_information(intermediate_representation) intermediate_representation_path = "input_file_generated/ir.json" save_file(intermediate_representation, intermediate_representation_path) logging.info(f"Saved intermediate representation at {intermediate_representation_path}") @@ -159,6 +163,7 @@ def create_iac_from_doml(model, is_multiecore_metamodel, metamodel_directory): intermediate_representation = create_intermediate_representation(model_path, is_multiecore_metamodel, metamodel_directory) template_generated_folder = create_iac_from_intermediate_representation(intermediate_representation) + PiacereInternalToolsIntegrator.add_files_for_piacere_internal_tools(template_generated_folder) compress_folder_info = compress_iac_folder(template_generated_folder) return compress_folder_info diff --git a/icgparser/IntermediateRepresentationUtility.py b/icgparser/IntermediateRepresentationUtility.py new file mode 100644 index 0000000000000000000000000000000000000000..404f973d3d34f1028ad5c314fd3d89e1409c5047 --- /dev/null +++ b/icgparser/IntermediateRepresentationUtility.py @@ -0,0 +1,32 @@ +import logging +from enum import Enum + + +class NoValue(Enum): + def __repr__(self): + return '<%s.%s>' % (self.__class__.__name__, self.name) + + +class IntermediateRepresentationResources(NoValue): + STEP_NAME = 'step_name' + STEPS = 'steps' + DATA = 'data' + LANGUAGE = "programming_language" + VIRTUAL_MACHINES = 'vms' + + +def find_objects(object_name, intermediate_representation): + logging.info(f"Searching for {object_name.value} in intermediate representation") + steps = intermediate_representation[IntermediateRepresentationResources.STEPS.value] + for step in steps: + data = step[IntermediateRepresentationResources.DATA.value] + if object_name.value in data.keys(): + return data[IntermediateRepresentationResources.VIRTUAL_MACHINES.value] + return [] + + +def add_step(step, intermediate_representation): + logging.info("Adding step into intermediate representation") + steps = intermediate_representation[IntermediateRepresentationResources.STEPS.value] + steps.append(step) + return intermediate_representation diff --git a/icgparser/ModelParser.py b/icgparser/ModelParser.py index ac945f533543b2d08866dc9c59542c2e36b0dc0d..88c5aaf6c01a6a0a4c5915c5707acc79e5cb7589 100644 --- a/icgparser/ModelParser.py +++ b/icgparser/ModelParser.py @@ -100,7 +100,7 @@ def parse_application_layer(doml_model, infra_object_step): object_representation = {} application_resource = deployment.eGet("component") - ## TODO refactoring + ## TODO refactoring -> far diventare lista nodi? nel monitoring sono più nodi vm = deployment.eGet("node") try: for infra_vm in infra_object_step.get("data").get("vms"): @@ -113,6 +113,7 @@ def parse_application_layer(doml_model, infra_object_step): object_representation = DomlParserUtilities.save_attributes(application_resource, object_representation) application_object_step["data"][deployment_component_name] = object_representation + application_object_step["step_name"] = deployment_component_name return application_object_step diff --git a/icgparser/PiacereInternalToolsIntegrator.py b/icgparser/PiacereInternalToolsIntegrator.py new file mode 100644 index 0000000000000000000000000000000000000000..4b314fdeebf7cf2bca6500e6d307ba5c22b98ec8 --- /dev/null +++ b/icgparser/PiacereInternalToolsIntegrator.py @@ -0,0 +1,38 @@ +import logging +from distutils.dir_util import copy_tree + +from icgparser import IntermediateRepresentationUtility +from icgparser.IntermediateRepresentationUtility import IntermediateRepresentationResources + + +def extract_info_for_monitoring_agents(intermediate_representation): + logging.info("Adding info for monitoring step") + monitoring_object_step = {"programming_language": "ansible", "step_name": "piacere_monitoring", + "data": {"piacere_monitoring": {}}} + vms = IntermediateRepresentationUtility.find_objects(IntermediateRepresentationResources.VIRTUAL_MACHINES, + intermediate_representation) + # TODO restore these 2 commented lines: monitoring could be installed on multiple nodes! + # monitoring_object_step["data"]["monitoring"]["nodes"] = [] + # monitoring_object_step["data"]["monitoring"]["nodes"] += vms + # TODO remove this line: monitoring could be installed on multiple nodes! + if vms: + monitoring_object_step["data"]["piacere_monitoring"]["node"] = vms[0] + logging.info(f"Monitoring step: {monitoring_object_step}") + return monitoring_object_step + + +def add_internal_tool_information(intermediate_representation): + monitoring_step = extract_info_for_monitoring_agents(intermediate_representation) + intermediate_representation = IntermediateRepresentationUtility.add_step(monitoring_step, + intermediate_representation) + return intermediate_representation + + +def add_files_for_monitoring_agents(template_generated_folder_path): + monitoring_folder_path = template_generated_folder_path + "/piacere_monitoring" + logging.info(f"Adding monitoring agents folder in {monitoring_folder_path}") + copy_tree("templates/ansible/ubuntu/monitoring", monitoring_folder_path) + + +def add_files_for_piacere_internal_tools(template_generated_folder_path): + add_files_for_monitoring_agents(template_generated_folder_path) \ No newline at end of file diff --git a/input_file_generated/ir.json b/input_file_generated/ir.json index da327c87fed30ec8ed80fc98faa55b25629b2a2d..4d6918935c269c58eb10635fe40560f957bde7bf 100644 --- a/input_file_generated/ir.json +++ b/input_file_generated/ir.json @@ -105,7 +105,31 @@ "source_code": "/usr/share/nginx/html/index.html" } }, - "programming_language": "ansible" + "programming_language": "ansible", + "step_name": "nginx" + }, + { + "data": { + "piacere_monitoring": { + "node": { + "credentials": "ssh_key", + "group": "sg", + "i1": { + "associated": "sg", + "belongsTo": "net1", + "endPoint": "16.0.0.1", + "name": "i1" + }, + "infra_element_name": "vm1", + "name": "concrete_vm", + "os": "Ubuntu-Focal-20.04-Daily-2022-04-19", + "vm_flavor": "small", + "vm_name": "nginx-host" + } + } + }, + "programming_language": "ansible", + "step_name": "piacere_monitoring" } ] } \ No newline at end of file diff --git a/main.py b/main.py index 4babd6f6be70f3587a5ef907bc5c70fa43fd51ba..2766ca70bdc56f4546ea86e064a4cbe3e4a172c9 100644 --- a/main.py +++ b/main.py @@ -45,8 +45,8 @@ logging.getLogger().setLevel(logging.INFO) # Parse parameters # ------------------------------------------------------------------------- skip_next = False -doml_directory = "./icgparser/doml" -model_filename = "icgparser/doml/v1/nginx-aws_v1.domlx" +doml_directory = "./icgparser/doml/v1" +model_filename = "icgparser/doml/v1/nginx-openstack_v1.domlx" load_split_model = False output_file_name = "iac_files.tar.gz" diff --git a/output_files_generated/nginx_openstack/config.yaml b/output_files_generated/nginx_openstack/config.yaml index 6464675fc28c7eb4c9e955f4184bf662c8f3f0a5..9aa58bc55e5cf8c9366dd3087f48e7c6fa92c6a4 100644 --- a/output_files_generated/nginx_openstack/config.yaml +++ b/output_files_generated/nginx_openstack/config.yaml @@ -1,5 +1,6 @@ --- iac: - terraform -- ansible +- nginx +- piacere_monitoring ... \ No newline at end of file diff --git a/output_files_generated/nginx_openstack/ansible/config.yaml b/output_files_generated/nginx_openstack/nginx/config.yaml similarity index 63% rename from output_files_generated/nginx_openstack/ansible/config.yaml rename to output_files_generated/nginx_openstack/nginx/config.yaml index e84af9b9c528953c797be67512f73d896b953260..7778c71d6b0453c44ab84da50655e5740087e2f4 100644 --- a/output_files_generated/nginx_openstack/ansible/config.yaml +++ b/output_files_generated/nginx_openstack/nginx/config.yaml @@ -2,7 +2,7 @@ --- input: - instance_ip_nginx-host - - instance_server_private_key_ + - instance_server_private_key_ssh_key output: [] engine: ansible ... diff --git a/output_files_generated/nginx_openstack/ansible/inventory.j2 b/output_files_generated/nginx_openstack/nginx/inventory.j2 similarity index 100% rename from output_files_generated/nginx_openstack/ansible/inventory.j2 rename to output_files_generated/nginx_openstack/nginx/inventory.j2 diff --git a/output_files_generated/nginx_openstack/ansible/nginx.yaml b/output_files_generated/nginx_openstack/nginx/nginx.yaml similarity index 100% rename from output_files_generated/nginx_openstack/ansible/nginx.yaml rename to output_files_generated/nginx_openstack/nginx/nginx.yaml diff --git a/output_files_generated/nginx_openstack/ansible/ssh_key.j2 b/output_files_generated/nginx_openstack/nginx/ssh_key.j2 similarity index 100% rename from output_files_generated/nginx_openstack/ansible/ssh_key.j2 rename to output_files_generated/nginx_openstack/nginx/ssh_key.j2 diff --git a/output_files_generated/nginx_openstack/terraform/ansible/playbooks/pma/ansible.cfg b/output_files_generated/nginx_openstack/piacere_monitoring/ansible.cfg similarity index 100% rename from output_files_generated/nginx_openstack/terraform/ansible/playbooks/pma/ansible.cfg rename to output_files_generated/nginx_openstack/piacere_monitoring/ansible.cfg diff --git a/output_files_generated/nginx_openstack/terraform/ansible/playbooks/pma/ansible_requirements.yml b/output_files_generated/nginx_openstack/piacere_monitoring/ansible_requirements.yml similarity index 100% rename from output_files_generated/nginx_openstack/terraform/ansible/playbooks/pma/ansible_requirements.yml rename to output_files_generated/nginx_openstack/piacere_monitoring/ansible_requirements.yml diff --git a/output_files_generated/nginx_openstack/piacere_monitoring/config.yaml b/output_files_generated/nginx_openstack/piacere_monitoring/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7778c71d6b0453c44ab84da50655e5740087e2f4 --- /dev/null +++ b/output_files_generated/nginx_openstack/piacere_monitoring/config.yaml @@ -0,0 +1,8 @@ + +--- +input: + - instance_ip_nginx-host + - instance_server_private_key_ssh_key +output: [] +engine: ansible +... diff --git a/output_files_generated/nginx_openstack/terraform/ansible/playbooks/pma/hosts.yaml b/output_files_generated/nginx_openstack/piacere_monitoring/hosts.yaml similarity index 100% rename from output_files_generated/nginx_openstack/terraform/ansible/playbooks/pma/hosts.yaml rename to output_files_generated/nginx_openstack/piacere_monitoring/hosts.yaml diff --git a/output_files_generated/nginx_openstack/terraform/ansible/playbooks/pma/install_playbook_requirements.sh b/output_files_generated/nginx_openstack/piacere_monitoring/install_playbook_requirements.sh similarity index 100% rename from output_files_generated/nginx_openstack/terraform/ansible/playbooks/pma/install_playbook_requirements.sh rename to output_files_generated/nginx_openstack/piacere_monitoring/install_playbook_requirements.sh diff --git a/output_files_generated/nginx_openstack/piacere_monitoring/inventory.j2 b/output_files_generated/nginx_openstack/piacere_monitoring/inventory.j2 new file mode 100644 index 0000000000000000000000000000000000000000..9bd52bbf200909d93fa271990cfeb103f26d35a2 --- /dev/null +++ b/output_files_generated/nginx_openstack/piacere_monitoring/inventory.j2 @@ -0,0 +1,9 @@ + + +[servers_for_] +{{ instance_ip_nginx-host }} + +[servers_for_:vars] +ansible_connection=ssh +ansible_user= +ansible_ssh_private_key_file={{ instance_server_private_key_ }} diff --git a/output_files_generated/nginx_openstack/piacere_monitoring/main.yml b/output_files_generated/nginx_openstack/piacere_monitoring/main.yml new file mode 100644 index 0000000000000000000000000000000000000000..657a7ea8c4c1376b0437e53ec82d7c4ebe6d0ec2 --- /dev/null +++ b/output_files_generated/nginx_openstack/piacere_monitoring/main.yml @@ -0,0 +1,22 @@ +--- +- hosts: localhost + tasks: + - name: print disclamer + debug: + msg: this can also be done with "ansible-galaxy install -r requirements" + - name: install telegraf from galaxy + community.general.ansible_galaxy_install: + type: role + requirements_file: ansible_requirements.yml + +- hosts: all + pre_tasks: + - name: Ensure gnupg package + package: + name: gnupg + state: present + become: true + vars_files: + - vars/main.yaml + roles: + - dj-wasabi.telegraf \ No newline at end of file diff --git a/output_files_generated/nginx_openstack/piacere_monitoring/piacere_monitoring.yaml b/output_files_generated/nginx_openstack/piacere_monitoring/piacere_monitoring.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8b137891791fe96927ad78e64b0aad7bded08bdc --- /dev/null +++ b/output_files_generated/nginx_openstack/piacere_monitoring/piacere_monitoring.yaml @@ -0,0 +1 @@ + diff --git a/output_files_generated/nginx_openstack/terraform/ansible/playbooks/pma/run-playbook.sh b/output_files_generated/nginx_openstack/piacere_monitoring/run-playbook.sh similarity index 100% rename from output_files_generated/nginx_openstack/terraform/ansible/playbooks/pma/run-playbook.sh rename to output_files_generated/nginx_openstack/piacere_monitoring/run-playbook.sh diff --git a/output_files_generated/nginx_openstack/terraform/ansible/playbooks/pma/site.yaml b/output_files_generated/nginx_openstack/piacere_monitoring/site.yaml similarity index 100% rename from output_files_generated/nginx_openstack/terraform/ansible/playbooks/pma/site.yaml rename to output_files_generated/nginx_openstack/piacere_monitoring/site.yaml diff --git a/output_files_generated/nginx_openstack/terraform/ansible/playbooks/pma/site_requirements.yaml b/output_files_generated/nginx_openstack/piacere_monitoring/site_requirements.yaml similarity index 100% rename from output_files_generated/nginx_openstack/terraform/ansible/playbooks/pma/site_requirements.yaml rename to output_files_generated/nginx_openstack/piacere_monitoring/site_requirements.yaml diff --git a/output_files_generated/nginx_openstack/piacere_monitoring/ssh_key.j2 b/output_files_generated/nginx_openstack/piacere_monitoring/ssh_key.j2 new file mode 100644 index 0000000000000000000000000000000000000000..1b9ad076e0fea507ee1cbf04472da28a893fc929 --- /dev/null +++ b/output_files_generated/nginx_openstack/piacere_monitoring/ssh_key.j2 @@ -0,0 +1 @@ +{{ instance_server_private_key }} \ No newline at end of file diff --git a/output_files_generated/nginx_openstack/terraform/ansible/playbooks/pma/vars/main.yaml b/output_files_generated/nginx_openstack/piacere_monitoring/vars/main.yaml similarity index 67% rename from output_files_generated/nginx_openstack/terraform/ansible/playbooks/pma/vars/main.yaml rename to output_files_generated/nginx_openstack/piacere_monitoring/vars/main.yaml index ada23165aebadbfe12bffb77368f0181fb4f1737..861faf3e8413d8a708702083d34c7ecd6f9a1fb1 100644 --- a/output_files_generated/nginx_openstack/terraform/ansible/playbooks/pma/vars/main.yaml +++ b/output_files_generated/nginx_openstack/piacere_monitoring/vars/main.yaml @@ -1,3 +1,9 @@ +pma_deployment_id: "123e4567-e89b-12d3-a456-426614174002" +pma_influxdb_bucket: "bucket" +pma_influxdb_token: "piacerePassword" +pma_influxdb_org: "piacere" +pma_influxdb_addr: "https://influxdb.pm.ci.piacere.digital.tecnalia.dev" + telegraf_agent_package_state: latest telegraf_agent_output: @@ -18,6 +24,4 @@ telegraf_plugins_default: - plugin: mem - plugin: processes - plugin: disk - - plugin: net - - + - plugin: net \ No newline at end of file diff --git a/output_files_generated/nginx_openstack/terraform/output.tf b/output_files_generated/nginx_openstack/terraform/output.tf index 2f4e6e32723a7b105ad6d6757d6890f75fb9b8f7..6fcb19b25700b08936928d8fef0c8db55047eb0e 100644 --- a/output_files_generated/nginx_openstack/terraform/output.tf +++ b/output_files_generated/nginx_openstack/terraform/output.tf @@ -1,10 +1,10 @@ -output "instance_server_public_key_" { +output "instance_server_public_key_ssh_key" { value = openstack_compute_keypair_v2.ssh_key.public_key } -output "instance_server_private_key_" { +output "instance_server_private_key_ssh_key" { value = openstack_compute_keypair_v2.ssh_key.private_key } diff --git a/plugin/AnsiblePlugin.py b/plugin/AnsiblePlugin.py index e0eb144f31a81b23eacee592d97f862b412369bc..493eced74e546829a2fbb1da40ea11f6a93a8816 100644 --- a/plugin/AnsiblePlugin.py +++ b/plugin/AnsiblePlugin.py @@ -14,6 +14,8 @@ #------------------------------------------------------------------------- import logging + +from icgparser.IntermediateRepresentationUtility import IntermediateRepresentationResources from plugin import TemplateUtils from plugin.PluginException import PluginResourceNotFoundError @@ -43,8 +45,10 @@ def create_inventory_file(parameters, language, operating_system, template_name) return template_filled -def create_files(parameters, output_path): - language = "ansible" +def create_files(step, output_path): + language = step[IntermediateRepresentationResources.LANGUAGE.value] + step_name = step[IntermediateRepresentationResources.STEP_NAME.value] + parameters = step["data"] for resource_name, resource in parameters.items(): logging.info("Creating template for resource '%s'", resource_name) operating_system = find_operating_system(resource) @@ -53,10 +57,10 @@ def create_files(parameters, output_path): # for resource_params in parameters[resource_name]: resource_params = parameters[resource_name] - ansible_output_file_path = output_path + "/".join([language, resource_name]) + ".yaml" - inventory_output_file_path = output_path + "/".join([language, "inventory"]) + ".j2" - config_output_file_path = output_path + "/".join([language, "config"]) + ".yaml" - ssh_key_output_file_path = output_path + "/".join([language, "ssh_key.j2"]) + ansible_output_file_path = output_path + "/".join([step_name, resource_name]) + ".yaml" + inventory_output_file_path = output_path + "/".join([step_name, "inventory"]) + ".j2" + config_output_file_path = output_path + "/".join([step_name, "config"]) + ".yaml" + ssh_key_output_file_path = output_path + "/".join([step_name, "ssh_key.j2"]) template = TemplateUtils.read_template(ansible_template_path) template_filled = TemplateUtils.edit_template(template, resource_params) diff --git a/plugin/TerraformPlugin.py b/plugin/TerraformPlugin.py index 69f5e737096cb4e45bbe92f68d6f65fbe78dbd0e..01fa43bddafa661598fd8dc65a4777dcca7f035f 100644 --- a/plugin/TerraformPlugin.py +++ b/plugin/TerraformPlugin.py @@ -19,11 +19,6 @@ from distutils.dir_util import copy_tree from plugin import TemplateUtils, PluginUtility -def store_monitoring_agents_folder(output_path): - logging.info(f"Adding monitoring agents folder in {output_path}") - copy_tree("templates/terraform/open_stack/agents_playbook", output_path) - - def create_files(parameters, output_path): language = "terraform" provider = parameters["provider"] @@ -58,7 +53,6 @@ def create_files(parameters, output_path): TemplateUtils.write_template(terraform_out_file, output_file_stored_path) config_file_stored_path = output_path + "/config.yaml" TemplateUtils.write_template(config_file, config_file_stored_path) - store_monitoring_agents_folder(output_path) logging.info("Terraform main file available at: {}".format(main_file_stored_path)) logging.info(f"Terraform output file available at {output_file_stored_path}") diff --git a/template-location.properties b/template-location.properties index dc464b3e224c99423ce6eea6c231984f180d6280..b9fa63fa5af690e19ef6782a6f19b1d06ef9a4ad 100644 --- a/template-location.properties +++ b/template-location.properties @@ -41,6 +41,7 @@ nginx = templates/ansible/ubuntu/nginx.tpl mysql = templates/ansible/ubuntu/mysql.tpl wordpress = templates/ansible/ubuntu/wordpress.tpl postgres = templates/ansible/ubuntu/postgres.tpl +piacere_monitoring = templates/ansible/ubuntu/piacere_init.tpl [ansible.centos] mysql = templates/ansible/centos/mysql.tpl diff --git a/templates/ansible/ubuntu/config.tpl b/templates/ansible/ubuntu/config.tpl index d58935294cef534027a7e6c7d0f3969fb5cff460..015123ee6f50408cd6b038e8088963171031dc5e 100644 --- a/templates/ansible/ubuntu/config.tpl +++ b/templates/ansible/ubuntu/config.tpl @@ -16,7 +16,7 @@ --- input: - instance_ip_{{ node.vm_name }} - - instance_server_private_key_{{ node.vm_key_name }} + - instance_server_private_key_{{ node.credentials }} output: [] engine: ansible ... diff --git a/templates/ansible/ubuntu/piacere_init.tpl b/templates/ansible/ubuntu/piacere_init.tpl new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/templates/terraform/open_stack/virtual_machine_out.tpl b/templates/terraform/open_stack/virtual_machine_out.tpl index 6df40799d46043989ebea90d54ad1e5fadf3f8db..8f5f19ea0a7ad4644e1612a7e3aa5a8c80e22004 100644 --- a/templates/terraform/open_stack/virtual_machine_out.tpl +++ b/templates/terraform/open_stack/virtual_machine_out.tpl @@ -14,11 +14,11 @@ #------------------------------------------------------------------------- #} -output "instance_server_public_key_{{ vm_key_name }}" { +output "instance_server_public_key_{{ credentials }}" { value = openstack_compute_keypair_v2.{{ credentials }}.public_key } -output "instance_server_private_key_{{ vm_key_name }}" { +output "instance_server_private_key_{{ credentials }}" { value = openstack_compute_keypair_v2.{{ credentials }}.private_key }