Add icg parser, new API definition

.gitignore

 .idea
+*.tar.gz
+icgparser/doml/*domlx
+output_files_generated/nginx_openstack/terraform/*
+output_files_generated/nginx_openstack/ansible/*
 
 # Byte-compiled / optimized / DLL files
 __pycache__/

api/InfrastructureTemplateController.py

+import json
 import logging
 import tarfile
+import uuid
 
 from fastapi import APIRouter, Body
 from fastapi.responses import FileResponse
 
-from plugin import TerraformPlugin
-from plugin import AnsiblePlugin
-
+from controller.PluginOrchestrator import create_infrastructure_files
+from icgparser import ModelParser
 
 api_router = APIRouter()
 
+base_compress_file_name = "iac_files_"
+
 @api_router.post("/infrastructure/files")
-def create_infrastructure_files(intermediate_representation: dict = Body(...)):
-    logging.info("Received intermediate representation create_infrastructure_files request")
-    choose_plugin(intermediate_representation)
-    logging.info("Creating compress folder with iac files")
-    output_template_folder = intermediate_representation["output_path"]
-    compress_file_name = "outputIaC.tar.gz"
-    compress_file_path = compress_file(output_template_folder, compress_file_name)
+def create_iac_from_intermediate_representation(intermediate_representation: dict = Body(...)):
+    logging.info("Received intermediate representation create_iac_from_intermediate_representation request")
+    template_generated_folder = create_infrastructure_files(intermediate_representation)
+    compress_file_name = random_file_name_generation(base_compress_file_name)
+    compress_file_path = compress_file(template_generated_folder, compress_file_name)
     return FileResponse(compress_file_path, media_type='application/octet-stream', filename=compress_file_name)
 
-def choose_plugin(parameters):
-    # os.system('rm -f /opt/output_files_generated/*')
-    for step in parameters["steps"]:
-        if step["programming_language"] == "ansible":
-            input_data = step["data"]
-            AnsiblePlugin.create_files(input_data, parameters["output_path"])
-        elif step["programming_language"] == "terraform":
-            input_data = step["data"]
-            TerraformPlugin.create_files(input_data, parameters["output_path"])
+
+@api_router.post("/iac/files")
+def create_iac_from_doml(data: str = Body(..., media_type="application/xml")):
+    logging.info("Received create_iac_from_doml request")
+    temp_model_file_path = "icgparser/doml/nginx-openstack.domlx"
+    logging.info("Writing model file in temp folder '%s' for parsing", temp_model_file_path)
+    f = open(temp_model_file_path, "w")
+    f.write(data)
+    f.close()
+    ModelParser.parse_model(temp_model_file_path)
+    with open("input_file_generated/ir.json") as json_file:
+        data = json.load(json_file)
+        template_generated_folder = create_infrastructure_files(data)
+        compress_file_name = random_file_name_generation(base_compress_file_name)
+        compress_file_folder = compress_file(template_generated_folder, compress_file_name)
+        return FileResponse(compress_file_folder,
+                            media_type='application/octet-stream',
+                            filename=compress_file_name)
+
+def random_file_name_generation(base_name):
+    return base_name + str(uuid.uuid4().hex) + ".tar.gz"
 
 def compress_file(source_folder, dest_file_name):
     # prefix_path = "/opt/"

controller/PluginOrchestrator.py

+import logging
+
+from plugin import AnsiblePlugin, TerraformPlugin
+
+
+def create_infrastructure_files(intermediate_representation: dict):
+    template_generated_folder = intermediate_representation["output_path"]
+    choose_plugin(intermediate_representation, template_generated_folder)
+    logging.info("iac files available at %s", template_generated_folder)
+    return template_generated_folder
+
+
+def choose_plugin(parameters, template_generated_folder):
+    # os.system('rm -f /opt/output_files_generated/*')
+    logging.info("Choosing plugin")
+    for step in parameters["steps"]:
+        if step["programming_language"] == "ansible":
+            logging.info("Ansible Plugin chosen")
+            input_data = step["data"]
+            AnsiblePlugin.create_files(input_data, template_generated_folder)
+        elif step["programming_language"] == "terraform":
+            logging.info("Terraform Plugin chosen")
+            input_data = step["data"]
+            TerraformPlugin.create_files(input_data, template_generated_folder)

icgparser/ModelParser.py

+# -------------------------------------------------------------------------
+#		PIACERE ICG Parser
+#
+#       This module has been tested with Python v3.7.4
+#       To use it you must first install PyEcore
+#           $ pip install pyecore
+#
+#       Usage: python icgparser.py [-h] [-d dir] [-v] [--single] model
+#           -h          prints usage
+#           -d dir      loads metamodel from <dir>
+#           --single / --single_mmodel   use the single (non-split) metamodel
+#           model       the input model to be translated into the ICG intermediate representation
+#
+#		Author: Lorenzo Blasi
+#		23/2/2022 - created
+#		© Copyright 2022 Hewlett Packard Enterprise Development LP
+# -------------------------------------------------------------------------
+import logging
+import sys
+from pyecore.resources import ResourceSet, URI, global_registry
+import pyecore.ecore as Ecore  # This gets a reference to the Ecore metamodel implementation
+
+# -------------------------------------------------------------------------
+# Utility functions to printout the loaded model
+# -------------------------------------------------------------------------
+newline = "\n"
+spaces = "    "
+comment = "#"
+
+
+def write_to(outchannel, line):
+    # for now we just print on the console
+    if outchannel == "console":
+        print(line)
+    # if the channel is different we don't print at all
+
+
+def print_obj(obj, level=0):
+    #    for x in range(level):
+    #        print("    ", end='')
+    class_name = obj.eClass.name
+    if class_name == 'Property':
+        #        print('Class: {0}\t\t{1} = {2}'.format(class_name, obj.key, obj.value))
+        print(f'{comment}{level * spaces}Class: {class_name}\t\t{obj.key} = {obj.value}')
+        return False
+    if class_name == 'Deployment':
+        print(
+            f'{comment}{level * spaces}Class: {class_name}\t\tcomponent = {obj.component.eClass.name}/{obj.component.name} node = {obj.node.eClass.name}/{obj.node.name}')
+        return False
+    try:
+        obj_name = obj.name
+        print(f'{comment}{level * spaces}Class: {class_name}\t\tObject: {obj_name}')
+        return True
+    except Exception:
+        print(f'{comment}{level * spaces}Class: {class_name}\t\tObject: no name')
+        return False
+
+
+def print_contents_recursive(obj, level=0):
+    if print_obj(obj, level):
+        for x in obj.eContents:
+            print_contents_recursive(x, level + 1)
+
+
+# -------------------------------------------------------------------------
+# Utility functions to produce the output Intermediate Language
+# -------------------------------------------------------------------------
+# --- Helpers
+def extract_image_name(concretevm_obj):
+    # To find the VM image name you could search into the inverse relations of the abstract image generating its related abstract VM, looking for a concrete image object (whose class is VMImage) and extract the value from its contents
+    #   concretevm_obj is a VirtualMachine (nginx-openstack_v2.doml:81, it should have been a OpenStackVM),
+    #   concretevm_obj.maps is a VirtualMachine (the abstract one)
+    #   concretevm_obj.maps.generatedFrom is a VMImage (the abstract one)
+    for x in concretevm_obj.maps.generatedFrom._inverse_rels:
+        if x[0].eClass.name == 'VMImage':
+            return x[0].eContents[0].value
+
+
+def extract_concrete_network_name(abstractnet_obj):
+    for x in abstractnet_obj._inverse_rels:
+        if x[0].eClass.name == 'Network':
+            return x[0].eContents[0].value
+
+
+# --- Handlers
+def model_handler(obj, model_root, level, intermediate_repr):
+    # output prefix
+    append_in_file(intermediate_repr,
+                   f'{level * spaces}{{{newline}{level * spaces}{spaces}"output_path": "output_files_generated/{obj.name}/",')
+    append_in_file(intermediate_repr, f'{level * spaces}{spaces}"steps": [')
+    # handle contents
+    for x in obj.eContents:
+        handle_obj(x, model_root, level + 2, intermediate_repr)
+    # output suffix
+    append_in_file(intermediate_repr, f'{level * spaces}{spaces}]')
+    append_in_file(intermediate_repr, f'{level * spaces}}}')
+
+
+def concrete_infra_handler(obj, model_root, level, intermediate_repr):
+    # output prefix
+    append_in_file(intermediate_repr, f'{level * spaces}{{{newline}{level * spaces}{spaces}"programming_language": "terraform",')
+    # handle contents
+    for x in obj.eContents:
+        handle_obj(x, model_root, level + 1, intermediate_repr)
+    # output suffix
+    append_in_file(intermediate_repr, f'{level * spaces}}}')
+
+
+def network_handler(obj, model_root, level, intermediate_repr):
+    # ignore the concrete network, since its name has been extracted separately and included in the concrete VM
+    logging.warning('Ignoring Network')
+
+
+def property_handler(obj, model_root, level, intermediate_repr):
+    key = obj.key
+    append_in_file(intermediate_repr, f'{level * spaces}"{key}" :     "{obj.value}",')
+
+
+def provider_handler(obj, model_root, level, intermediate_repr):
+    # output prefix
+    append_in_file(intermediate_repr, f'{level * spaces}"data": {{{newline}{level * spaces}{spaces}"provider": "{obj.name}",')
+    # handle contents
+    for x in obj.eContents:
+        handle_obj(x, model_root, level + 1, intermediate_repr)
+    # output suffix
+    append_in_file(intermediate_repr, f'{level * spaces}}}')
+
+
+def concrete_vm_handler(obj, model_root, level, intermediate_repr):
+    # output prefix
+    append_in_file(intermediate_repr, f'{level * spaces}"vm": [{{')  # VMs can be more than one: I need an example...
+    level = level + 1
+    # print(f'{level * spaces}# maps {obj.maps.name}')
+    logging.warning(f"Ignoring map {obj.maps.name}")
+    # handle contents
+    for x in obj.eContents:
+        handle_obj(x, model_root, level, intermediate_repr)
+    # add other attributes defined elsewhere: image name, address, ...
+    append_in_file(intermediate_repr, f'{level * spaces}"image" :     "{extract_image_name(obj)}",')
+    for iface in obj.maps.ifaces:
+        append_in_file(intermediate_repr, f'{level * spaces}"address" :     "{iface.endPoint}",')
+        append_in_file(intermediate_repr, f'{level * spaces}"network_name" :     "{extract_concrete_network_name(iface.belongsTo)}"')
+    # output suffix
+    level = level - 1
+    append_in_file(intermediate_repr, f'{level * spaces}}}]')
+
+
+def vm_image_handler(obj, model_root, level, intermediate_repr):
+    # ignore the concrete image, since its image name has been extracted separately and included in the concrete VM
+    logging.warning(f'Ignoring VMImage')
+
+
+class_handler = {
+
+    "DOMLModel": model_handler,
+    "ConcreteInfrastructure": concrete_infra_handler,
+    "Network": network_handler,
+    "Property": property_handler,
+    "RuntimeProvider": provider_handler,
+    "VirtualMachine": concrete_vm_handler,  # Warning: the class here might change to some concrete VM class
+    "VMImage": vm_image_handler
+}
+
+
+def handle_obj(obj, model_root, level, intermediate_repr):
+    if obj.eClass.name in class_handler:
+        class_handler[obj.eClass.name](obj, model_root, level, intermediate_repr)
+    else:
+        logging.warning(f'Class {obj.eClass.name} has no handler')
+
+
+# -------------------------------------------------------------------------
+# Parse parameters
+# -------------------------------------------------------------------------
+skip_next = False
+doml_directory = "icgparser/doml"
+
+
+# -------------------------------------------------------------------------
+# Load each part of the DOML metamodel and register them
+# -------------------------------------------------------------------------
+def load_metamodel(load_split_model):
+    global_registry[Ecore.nsURI] = Ecore  # Load the Ecore metamodel first
+    rset = ResourceSet()
+    if load_split_model:
+        mm_parts = ["doml", "commons", "application", "infrastructure", "concrete", "optimization"]
+        for mm_filename in mm_parts:
+            resource = rset.get_resource(URI(f"{doml_directory}/{mm_filename}.ecore"))
+            mm_root = resource.contents[0]  # Get the root of the MetaModel (EPackage)
+            rset.metamodel_registry[mm_root.nsURI] = mm_root
+    else:
+        resource = rset.get_resource(URI(f"{doml_directory}/doml.ecore"))
+        mm_root = resource.contents[0]  # Get the root of the MetaModel (EPackage)
+        rset.metamodel_registry[mm_root.nsURI] = mm_root
+        for subp in mm_root.eSubpackages:
+            rset.metamodel_registry[subp.nsURI] = subp
+    return rset
+
+
+# -------------------------------------------------------------------------
+# Finally load the model and print it out
+# -------------------------------------------------------------------------
+
+def parse_model(model):
+    load_split_model = None
+    rset = load_metamodel(load_split_model)
+    doml_model_resource = rset.get_resource(URI(model))
+    doml_model = doml_model_resource.contents[0]
+    single = "single-file (doml.ecore)"
+    split = "split"
+    dash = "-"
+    logging.info(f'{comment}{80 * dash}')
+    logging.info(f'{comment} Using {split if load_split_model else single} metamodel from directory {doml_directory}')
+    print(f'{comment} Model loaded from file {model}:')
+    print(f'{comment}{80 * dash}')
+    print_contents_recursive(doml_model)
+    print(f'{comment}{80 * dash}{newline}{comment} Generated Intermediate Representation follows:{newline}{comment}')
+    intermediate_repr_file_path = "input_file_generated/ir.json"
+    create_file("input_file_generated/ir.json")
+    handle_obj(doml_model, doml_model, 0, intermediate_repr_file_path)
+
+def create_file(file_name):
+    f = open(file_name, "w")
+    f.write("")
+    f.close()
+
+def append_in_file(file_name, data):
+    f = open(file_name, "a")
+    f.write(data)
+    f.write("\n")
+    f.close()

input_file_generated/ir.json

+{
+    "output_path": "output_files_generated/nginx_openstack/",
+    "steps": [
+        {
+            "programming_language": "terraform",
+            "data": {
+                "provider": "openstack",
+                "vm": [{
+                    "vm_name" :     "nginx-host",
+                    "vm_flavor" :     "small",
+                    "vm_key_name" :     "user1",
+                    "ssh_user" :     "ubuntu",
+                    "ssh_key_file" :     "/home/user1/.ssh/openstack.key",
+                    "image" :     "ubuntu-20.04.3",
+                    "address" :     "16.0.0.1",
+                    "network_name" :     "ostack2"
+                }]
+            }
+        }
+    ]
+}

main.py

 import logging
 from fastapi import FastAPI
 import api.InfrastructureTemplateController
+from icgparser import ModelParser
 
 fast_api = FastAPI()
 
@@ -9,3 +10,4 @@ logging.getLogger().setLevel(logging.INFO)
 
 if __name__ == '__main__':
     logging.info("Starting ICG application")
+    ModelParser.parse_model("icgparser/doml/nginx-openstack_v2.domlx")

output_file_example/wordpress_azure/terraform/main.tf

@@ -13,37 +13,121 @@ provider "azurerm" {
   features {}
 }
 
+resource "azurerm_resource_group" "rg" {
+  name     = "TerraformTesting"
+  location = "eastus" ## REQUIRED
+}
+
 ## VIRTUAL NETWORK
-resource "azurerm_virtual_network" "wordpress_net_vnetwork" {
-  name                = "wordpress_net"
+resource "azurerm_virtual_network" "vnet" {
+  name                = "vNet"
   address_space       = ["10.0.0.0/16"]
-  location            = azurerm_resource_group.wordpress-example.location
-  resource_group_name = azurerm_resource_group.wordpress-example.name
+  location            = azurerm_resource_group.rg.location
+  resource_group_name = azurerm_resource_group.rg.name
 }
 
-## SUBNET
-resource "azurerm_subnet" "wordpress_net_subnet" {
+resource "azurerm_subnet" "subnet" {
   name                 = "internal"
-  resource_group_name  = azurerm_resource_group.wordpress-example.name
-  virtual_network_name = azurerm_virtual_network.wordpress_net_vnetwork.name
+  resource_group_name  = azurerm_resource_group.rg.name
+  virtual_network_name = azurerm_virtual_network.vnet.name
   address_prefixes       = ["10.0.2.0/24"]
 }
 
+## WORDPRESS PUBLIC IP
+resource "azurerm_public_ip" "wordpress_public_ip" {
+  name = "wordpress_public_ip"
+  location = azurerm_resource_group.rg.location
+  resource_group_name = azurerm_resource_group.rg.name
+  allocation_method = "Dynamic" ##REQUIRED??
+  sku = "Basic"
+}
+
+## WORDPRESS NETWORK INTERFACE
+resource "azurerm_network_interface" "wordpress_nic" {
+  name                = "wordpress_nic"
+  location            = azurerm_resource_group.rg.location
+  resource_group_name = azurerm_resource_group.rg.name
 
-## VIRTUAL NETWORK
-resource "azurerm_virtual_network" "mysql_net_vnetwork" {
-  name                = "mysql_net"
-  address_space       = ["10.0.0.0/16"]
-  location            = azurerm_resource_group.mysql-example.location
-  resource_group_name = azurerm_resource_group.mysql-example.name
+  ip_configuration {
+    name                          = "ipconfig1"
+    subnet_id                     = azurerm_subnet.subnet.id
+    private_ip_address_allocation = "Dynamic"
+    public_ip_address_id = azurerm_public_ip.wordpress_public_ip.id
+  }
+}
+
+## WORDPRESS VM
+resource "azurerm_linux_virtual_machine" "wordpress" { ## REQUIRED
+  resource_group_name = azurerm_resource_group.rg.name
+  ## instance details
+  name                = "wordpress-machine"
+  location            = azurerm_resource_group.rg.location
+  size                = "Standard_B1s" ## REQUIRED
+  ## administrator account
+  admin_username      = "adminuser"
+  admin_password      = "P@$$w0rd1234!" ##For Bastion Connection
+  disable_password_authentication = false
+  #availability_set_id = azurerm_availability_set.DemoAset.id
+  network_interface_ids = [
+    azurerm_network_interface.wordpress_nic.id
+  ]
+
+  os_disk {
+    caching              = "None"
+    storage_account_type = "Standard_LRS" ## REQUIRED
+  }
+
+  admin_ssh_key {
+    username = "adminuser"
+    public_key = file("${path.module}/ssh_keys/wordpress_rsa.pub")
+  }
+
+  source_image_reference {
+    publisher = "Canonical"
+    offer     = "UbuntuServer"
+    sku       = "18.04-LTS"
+    version   = "latest"
   }
+}
+
+## MYSQL NETWORK INTERFACE
+resource "azurerm_network_interface" "mysql_nic" {
+  name                = "mysql_nic"
+  location            = azurerm_resource_group.rg.location
+  resource_group_name = azurerm_resource_group.rg.name
 
-## SUBNET
-resource "azurerm_subnet" "mysql_net_subnet" {
+  ip_configuration {
     name                          = "internal"
-  resource_group_name  = azurerm_resource_group.mysql-example.name
-  virtual_network_name = azurerm_virtual_network.mysql_net_vnetwork.name
-  address_prefixes       = ["10.0.2.0/24"]
+    subnet_id                     = azurerm_subnet.subnet.id
+    private_ip_address_allocation = "Dynamic"
   }
+}
+
+## MYSQL VM
+resource "azurerm_linux_virtual_machine" "mysql" { ## REQUIRED
+  resource_group_name = azurerm_resource_group.rg.name
+  ## instance details
+  name                = "mysql-machine"
+  location            = azurerm_resource_group.rg.location
+  size                = "Standard_B1s" ## REQUIRED
+  ## administrator account
+  admin_username      = "adminuser"
+  admin_password      = "P@$$w0rd1234!"
+  disable_password_authentication = false
+  #availability_set_id = azurerm_availability_set.DemoAset.id
+  network_interface_ids = [
+    azurerm_network_interface.mysql_nic.id
+  ]
 
+  os_disk {
+    caching              = "None"
+    storage_account_type = "Standard_LRS"
+  }
 
+  source_image_reference {
+    publisher = "Canonical"
+    offer     = "UbuntuServer"
+    sku       = "18.04-LTS"
+    version   = "latest"
+  }
+}

output_files_generated/nginx_openstack/terraform/main.tf

@@ -45,57 +45,3 @@ resource "openstack_compute_floatingip_associate_v2" "nginx-host_floating_ip_ass
   instance_id = openstack_compute_instance_v2.nginx-host.id
 }
 
-## Network
-
-# Create Network
-resource "openstack_networking_network_v2" "ostack2" {
-  name = "ostack2"
-}
-
-# Create Subnet
-resource "openstack_networking_subnet_v2" "ostack2_subnet" {
-  name            = "ostack2_subnet"
-  network_id      = openstack_networking_network_v2.ostack2.id
-  cidr            = "16.0.0.0/24"
-  dns_nameservers = ["8.8.8.8", "8.8.8.4"]
-}
-
-# Attach networking port
-resource "openstack_networking_port_v2" "ostack2" {
-  name           = "ostack2"
-  network_id     = openstack_networking_network_v2.ostack2.id
-  admin_state_up = true
-  security_group_ids = [
-    
-    openstack_compute_secgroup_v2.rule_1_secgroup.id,
-    
-    openstack_compute_secgroup_v2.rule_2_secgroup.id,
-    
-  ]
-  fixed_ip {
-    subnet_id = openstack_networking_subnet_v2.ostack2_subnet.id
-  }
-}
-
-resource "openstack_compute_secgroup_v2" "rule_1_secgroup" {
-  name        = "rule_1"
-  description  = "Security group rule for port 80-80"
-  rule {
-    from_port   = 80
-    to_port     = 80
-    ip_protocol = "tcp"
-    cidr        = "0.0.0.0/0"
-  }
-}
-
-resource "openstack_compute_secgroup_v2" "rule_2_secgroup" {
-  name        = "rule_2"
-  description  = "Security group rule for port 22-22"
-  rule {
-    from_port   = 22
-    to_port     = 22
-    ip_protocol = "tcp"
-    cidr        = "0.0.0.0/0"
-  }
-}
-

requirements.txt

@@ -2,3 +2,4 @@ Jinja2==3.0.3
 PyYAML==6.0
 fastapi~=0.74.1
 uvicorn==0.17.5
+pyecore~=0.12.2
\ No newline at end of file

templates/terraform/open_stack/virtual_machine.tpl

 # Create virtual machine
-resource "openstack_compute_instance_v2" "{{ name }}" {
-  name        = "{{ name }}"
+resource "openstack_compute_instance_v2" "{{ vm_name }}" {
+  name        = "{{ vm_name }}"
   image_name  = "{{ image }}"
-  flavor_name = "{{ flavor }}"
-  key_pair    = openstack_compute_keypair_v2.{{ name ~ "_ssh_key" }}.name
+  flavor_name = "{{ vm_flavor }}"
+  key_pair    = openstack_compute_keypair_v2.{{ vm_name ~ "_ssh_key" }}.name
   network {
     port = openstack_networking_port_v2.{{ network_name }}.id
   }
 }
 
 # Create ssh keys
-resource "openstack_compute_keypair_v2" "{{ name ~ "_ssh_key" }}" {
+resource "openstack_compute_keypair_v2" "{{ vm_name ~ "_ssh_key" }}" {
   name       = "{{ ssh_user }}"
   public_key = "{{ ssh_key_file }}"
 }
 
 # Create floating ip
-resource "openstack_networking_floatingip_v2" "{{name ~ "_floating_ip"}}" {
+resource "openstack_networking_floatingip_v2" "{{vm_name ~ "_floating_ip"}}" {
   pool = "external"
   # fixed_ip = "{{ address }}"
 }
 
 # Attach floating ip to instance
-resource "openstack_compute_floatingip_associate_v2" "{{ name ~ "_floating_ip_association" }}" {
-  floating_ip = openstack_networking_floatingip_v2.{{ name ~ "_floating_ip" }}.address
-  instance_id = openstack_compute_instance_v2.{{ name }}.id
+resource "openstack_compute_floatingip_associate_v2" "{{ vm_name ~ "_floating_ip_association" }}" {
+  floating_ip = openstack_networking_floatingip_v2.{{ vm_name ~ "_floating_ip" }}.address
+  instance_id = openstack_compute_instance_v2.{{ vm_name }}.id
 }