diff --git a/Dockerfile b/Dockerfile index a87b323777267d2f79caaeab81579c0b655eec78..91794c3ef230e7f4bddcadefce6991e43e6fe4aa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,13 +4,19 @@ RUN apt-get update -eany RUN apt-get upgrade -y --no-install-recommends +RUN apt-get install -y --no-install-recommends patch + RUN /usr/local/bin/python3.8 -m venv /opt/kolla-ansible-venv # Xena COPY openstack_requirements/upper-constraints.txt /tmp/ RUN /opt/kolla-ansible-venv/bin/python -m pip install -c /tmp/upper-constraints.txt \ - kolla-ansible==13.4.0 \ + kolla-ansible==13.8.0 \ ansible==4.10.0 +COPY patches/ovs-no-external-interface.patch /tmp/ +RUN sh -c "cd /opt/kolla-ansible-venv/share/kolla-ansible; patch -p1 < /tmp/ovs-no-external-interface.patch" +COPY patches/destroy-only-baremetal.patch /tmp/ +RUN sh -c "cd /opt/kolla-ansible-venv/share/kolla-ansible; patch -p1 < /tmp/destroy-only-baremetal.patch" FROM python:3.10.7-slim-bullseye as csep-build-stage diff --git a/csep_api/main.py b/csep_api/main.py index a918272e3f69b25f1c4280b577ac0ceaa03f8cdb..2d1d1d2f548a847180f30ca8bbdae9755d6e28ca 100644 --- a/csep_api/main.py +++ b/csep_api/main.py @@ -150,6 +150,24 @@ def redeploy_deployment( raise HTTPException(status_code=400, detail="Incompatible spec patch") +@app.post( + "/deployments/{deployment_name}/undeploy", + response_model=DeploymentOut, + response_model_exclude_unset=True, +) +def undeploy_deployment( + deployment_name: str, + data_store: DataStore = Depends(get_data_store), +) -> DeploymentInDB: + try: + deployment = data_store.get_deployment(deployment_name) + deployment.status.progress = OperationProgress.ToUndeploy + data_store.update_deployment(deployment) + return deployment + except DeploymentNotFoundError: + raise HTTPException(status_code=404, detail="Deployment not found") + + @app.delete("/deployments/{deployment_name}") def delete_deployment( deployment_name: str, data_store: DataStore = Depends(get_data_store) diff --git a/csep_common/datamodels.py b/csep_common/datamodels.py index 46ff6e5d591e731b9d55f19fef6f1ae633609e30..8e1dc9c9d731e294d4bc8c955e9d149886685aa7 100644 --- a/csep_common/datamodels.py +++ b/csep_common/datamodels.py @@ -20,6 +20,10 @@ class OperationProgress(str, Enum): Running = "Running" Failed = "Failed" Completed = "Completed" + ToUndeploy = "ToUndeploy" + Undeploying = "Undeploying" + Undeployed = "Undeployed" + UndeployFailed = "UndeployFailed" class EventSeverity(str, Enum): @@ -77,8 +81,12 @@ class VaultAuth(BaseModel): key: str +PlainAuth = UsernamePasswordAuth | UsernameSSHKeyAuth +AnyAuth = PlainAuth | VaultAuth + + class DeploymentSpecBase(BaseModel): - auth: Union[UsernamePasswordAuth, UsernameSSHKeyAuth, VaultAuth] + auth: AnyAuth bmc_auth: Optional[UsernamePasswordAuth] hosts: List[Host] = Field(min_items=1) diff --git a/csep_common/datastore.py b/csep_common/datastore.py index b2d9b4659a23899983a4b940e6659e001805c155..38fd94da3e55824ae0f75be2038464bc9b0460fa 100644 --- a/csep_common/datastore.py +++ b/csep_common/datastore.py @@ -75,7 +75,7 @@ class Etcd3DataStore(DataStore): def get_all_deployments(self) -> List[DeploymentInDB]: result = [] - for (value, _) in self.etcd3_client.get_prefix(self.DEPLOYMENTS_KEY_PREFIX): + for value, _ in self.etcd3_client.get_prefix(self.DEPLOYMENTS_KEY_PREFIX): result.append(DeploymentInDB(**json.loads(value))) return result diff --git a/csep_worker/backend.py b/csep_worker/backend.py index 2a9354e915e1c91d87e7779b495c3907b3bca989..657ab48fb3037bfffd8482f0f0970e4c2fb9dfaf 100644 --- a/csep_worker/backend.py +++ b/csep_worker/backend.py @@ -13,3 +13,7 @@ class Backend(ABC): @abstractmethod def run_deployment(self, deployment: DeploymentInDB) -> None: ... + + @abstractmethod + def undeploy(self, deployment: DeploymentInDB) -> None: + ... diff --git a/csep_worker/backends/dummy.py b/csep_worker/backends/dummy.py index 934d070bbebab7a3cc0b7063bb3ecf8e85504b9d..7506712b3230a0f05ec4ca013cf6df9253d23df8 100644 --- a/csep_worker/backends/dummy.py +++ b/csep_worker/backends/dummy.py @@ -24,3 +24,6 @@ class DummyBackend(Backend): if not spec.should_succeed: raise Exception() + + def undeploy(self, deployment: DeploymentInDB) -> None: + pass diff --git a/csep_worker/backends/openstack.py b/csep_worker/backends/openstack.py index 1fe5e3322e011d1760a4375c4c2f03baf358ba55..1496df4fcc5c56532af3ae7c404d59a300f13505 100644 --- a/csep_worker/backends/openstack.py +++ b/csep_worker/backends/openstack.py @@ -10,14 +10,17 @@ from typing import List, cast import yaml from csep_common.datamodels import ( + AnyAuth, DeploymentInDB, Event, EventSeverity, Host, OpenStackDeploymentSpec, OperationProgress, + PlainAuth, UsernamePasswordAuth, UsernameSSHKeyAuth, + VaultAuth, ) from csep_worker.backend import Backend from csep_worker.vault_client import vault_client @@ -32,6 +35,9 @@ KOLLA_ANSIBLE_SITE_PLAYBOOK = path.join( KOLLA_ANSIBLE_KOLLA_HOST_PLAYBOOK = path.join( KOLLA_BASE, "share/kolla-ansible/ansible/kolla-host.yml" ) +KOLLA_ANSIBLE_DESTROY_PLAYBOOK = path.join( + KOLLA_BASE, "share/kolla-ansible/ansible/destroy.yml" +) KOLLA_PASSWORDS_YAML = path.join( KOLLA_BASE, "share/kolla-ansible/etc_examples/kolla/passwords.yml" ) @@ -52,6 +58,7 @@ KOLLA_GLOBALS_DEFAULTS = { "enable_haproxy": False, "kolla_internal_vip_address": "{{ 'api' | kolla_address(groups.control | first) }}", "neutron_external_interface": "", + "neutron_bridge_name": "", } @@ -69,6 +76,105 @@ def private_file_opener(path: str, flags: int) -> int: return os.open(path, flags, mode=0o0600) +def gen_host_entry(host: Host, ansible_auth_str: str) -> str: + # TODO: it's probably nicer to do it with array and join later + + if host.name is None: + name = "host_" + host.ip_address.replace(".", "_") + else: + name = host.name + ip_address = host.ip_address + + entry = f"{name} ansible_host={ip_address}" + + if host.port is not None: + entry += f" ansible_port={host.port}" + + entry += ansible_auth_str + + if host.network_interface_name is None: + network_interface_name = "eth0" + else: + network_interface_name = host.network_interface_name + + entry += f" network_interface={network_interface_name}" + + return entry + + +def unpack_vault_auth(auth: VaultAuth) -> PlainAuth: + # need to establish the type dynamically + s = vault_client.read_secret(auth.key) + if "username" in s and "password" in s: + return UsernamePasswordAuth( + type="username_password", + username=s["username"], + password=s["password"], + ) + elif "username" in s and "private_ssh_key" in s: + return UsernameSSHKeyAuth( + type="username_sshkey", + username=s["username"], + private_ssh_key=s["private_ssh_key"], + ) + else: + # TODO: use a custom exception type + raise Exception("Unknown auth type") + + +def ensure_plain_auth(auth: AnyAuth) -> PlainAuth: + if auth.type == "vault": + return unpack_vault_auth(auth) + else: + return auth + + +def plain_auth_to_ansible_auth_str( + auth: PlainAuth, ssh_private_key_file_path: str +) -> str: + # NOTE: we are not using match/case because of mypy failing to + # recognise that pattern of type hint + if auth.type == "username_sshkey": + with open(ssh_private_key_file_path, "w", opener=private_file_opener) as f: + # TODO: validate the key format + f.write(auth.private_ssh_key) + # write an empty line in case it was missing (very likely) + # in the input string + f.write("\n") + + return ( + f" ansible_user={auth.username}" + f" ansible_ssh_private_key_file={ssh_private_key_file_path}" + ) + else: + return f" ansible_user={auth.username}" f" ansible_password={auth.password}" + + +def gen_inventory( + hosts: list[Host], inventory_file_path: str, ansible_auth_str: str +) -> None: + # TODO: validate if roles are coherent and avoid the "I use a single one + # below" (also in kolla_external_fqdn) + + inventory_lines: List[str] = [] + inventory_lines.append("[control]") + inventory_lines.append(gen_host_entry(hosts[0], ansible_auth_str)) + inventory_lines.append("[network]") + inventory_lines.append(gen_host_entry(hosts[0], ansible_auth_str)) + inventory_lines.append("[compute]") + for host in hosts: + if host.role is None or host.role == "compute": + inventory_lines.append(gen_host_entry(host, ansible_auth_str)) + inventory_lines.append(gen_host_entry(hosts[0], ansible_auth_str)) + inventory_lines.append("[monitoring]") + inventory_lines.append(gen_host_entry(hosts[0], ansible_auth_str)) + inventory_lines.append("[storage]") + inventory_lines.append(gen_host_entry(hosts[0], ansible_auth_str)) + with open(inventory_file_path, "w") as f: + f.writelines((f"{line}\n" for line in inventory_lines)) + f.write(TRAILING_INVENTORY) + + class OpenStackBackend(Backend): def run_deployment(self, deployment: DeploymentInDB) -> None: spec = cast(OpenStackDeploymentSpec, deployment.spec) @@ -77,103 +183,19 @@ class OpenStackBackend(Backend): self.data_store.update_deployment(deployment) logging.info(f"Deployment {deployment.name!r} marked running") - auth: UsernamePasswordAuth | UsernameSSHKeyAuth - if spec.auth.type == "vault": - # need to establish the type dynamically - s = vault_client.read_secret(spec.auth.key) - if "username" in s and "password" in s: - auth = UsernamePasswordAuth( - type="username_password", - username=s["username"], - password=s["password"], - ) - elif "username" in s and "private_ssh_key" in s: - auth = UsernameSSHKeyAuth( - type="username_sshkey", - username=s["username"], - private_ssh_key=s["private_ssh_key"], - ) - else: - # TODO: use a custom exception type - raise Exception("Unknown auth type") - else: - auth = spec.auth - - def gen_host_entry(host: Host) -> str: - # TODO: it's probably nicer to do it with array and join later - - if host.name is None: - name = "host_" + host.ip_address.replace(".", "_") - else: - name = host.name - ip_address = host.ip_address - - entry = f"{name} ansible_host={ip_address}" - - if host.port is not None: - entry += f" ansible_port={host.port}" - - entry += ansible_auth_str - - if host.network_interface_name is None: - network_interface_name = "eth0" - else: - network_interface_name = host.network_interface_name - - entry += f" network_interface={network_interface_name}" - - return entry + auth = ensure_plain_auth(spec.auth) with tempfile.TemporaryDirectory() as tmpdirpath: def gen_file_path(name: str) -> str: return path.join(tmpdirpath, name) - # NOTE: we are not using match/case because of mypy failing to - # recognise that pattern of type hint - if auth.type == "username_sshkey": - ssh_private_key_file_path = gen_file_path("ssh_key") - - with open( - ssh_private_key_file_path, "w", opener=private_file_opener - ) as f: - # TODO: validate the key format - f.write(auth.private_ssh_key) - # write an empty line in case it was missing (very likely) - # in the input string - f.write("\n") - - ansible_auth_str = ( - f" ansible_user={auth.username}" - f" ansible_ssh_private_key_file={ssh_private_key_file_path}" - ) - elif auth.type == "username_password": - ansible_auth_str = ( - f" ansible_user={auth.username}" - f" ansible_password={auth.password}" - ) - - # TODO: validate if roles are coherent and avoid the "I use a single one - # below" (also in kolla_external_fqdn) + ansible_auth_str = plain_auth_to_ansible_auth_str( + auth, gen_file_path("ssh_key") + ) inventory_file_path = gen_file_path("inventory.ini") - inventory_lines: List[str] = [] - inventory_lines.append("[control]") - inventory_lines.append(gen_host_entry(spec.hosts[0])) - inventory_lines.append("[network]") - inventory_lines.append(gen_host_entry(spec.hosts[0])) - inventory_lines.append("[compute]") - for host in spec.hosts: - if host.role is None or host.role == "compute": - inventory_lines.append(gen_host_entry(host)) - inventory_lines.append(gen_host_entry(spec.hosts[0])) - inventory_lines.append("[monitoring]") - inventory_lines.append(gen_host_entry(spec.hosts[0])) - inventory_lines.append("[storage]") - inventory_lines.append(gen_host_entry(spec.hosts[0])) - with open(inventory_file_path, "w") as f: - f.writelines((f"{line}\n" for line in inventory_lines)) - f.write(TRAILING_INVENTORY) + gen_inventory(spec.hosts, inventory_file_path, ansible_auth_str) # prepare globals @@ -254,6 +276,50 @@ class OpenStackBackend(Backend): # TODO: could we capture the progress as it moves forward? + def undeploy(self, deployment: DeploymentInDB) -> None: + spec = cast(OpenStackDeploymentSpec, deployment.spec) + + deployment.status.progress = OperationProgress.Undeploying + self.data_store.update_deployment(deployment) + logging.info(f"Deployment {deployment.name!r} marked undeploying") + + auth = ensure_plain_auth(spec.auth) + + with tempfile.TemporaryDirectory() as tmpdirpath: + + def gen_file_path(name: str) -> str: + return path.join(tmpdirpath, name) + + ansible_auth_str = plain_auth_to_ansible_auth_str( + auth, gen_file_path("ssh_key") + ) + + inventory_file_path = gen_file_path("inventory.ini") + gen_inventory(spec.hosts, inventory_file_path, ansible_auth_str) + + # prepare globals + + globals_file_path = gen_file_path("globals.yaml") + + with open(globals_file_path, "w") as f: + yaml.safe_dump(spec.globals, f) + + # prepare passwords + + passwords_file_path = gen_file_path("passwords.yaml") + + with open(passwords_file_path, "w") as f: + yaml.safe_dump(spec.passwords, f) + + self._run_kolla_ansible( + deployment, + inventory_file_path, + globals_file_path, + passwords_file_path, + tmpdirpath, + "destroy", + ) + def _run_kolla_ansible( self, deployment: DeploymentInDB, @@ -263,8 +329,16 @@ class OpenStackBackend(Backend): config_dir_path: str, action: str, # TODO: turn into an Enum ) -> None: + extra_args: list[str] = [] + if action == "bootstrap-servers": playbook = KOLLA_ANSIBLE_KOLLA_HOST_PLAYBOOK + elif action == "destroy": + playbook = KOLLA_ANSIBLE_DESTROY_PLAYBOOK + extra_args = [ + "-e", + "destroy_include_images=yes", + ] else: playbook = KOLLA_ANSIBLE_SITE_PLAYBOOK @@ -296,6 +370,7 @@ class OpenStackBackend(Backend): f"CONFIG_DIR={config_dir_path}", "-e", f"kolla_action={action}", + *extra_args, playbook, ], check=True, diff --git a/csep_worker/main.py b/csep_worker/main.py index d588c1738c8d7ec6f31136398752f0a8c0b4745c..907103b51179e24ffb07c1fae33dc4b3c7e30be0 100644 --- a/csep_worker/main.py +++ b/csep_worker/main.py @@ -31,23 +31,37 @@ def main_loop() -> None: def process_put_event(event: PutEvent) -> None: deployment = DeploymentInDB(**json.loads(event.value)) - # TODO: better check value version - if deployment.status.progress != OperationProgress.New: + if deployment.status.progress not in [ + OperationProgress.New, + OperationProgress.ToUndeploy, + ]: logging.debug("Skipping deployment update event") return backend = backends.get(deployment.spec.type) if backend is not None: - try: - backend.run_deployment(deployment) - except Exception: - logging.exception("Failed running the deployment") - deployment.status.progress = OperationProgress.Failed - data_store.update_deployment(deployment) - else: - deployment.status.progress = OperationProgress.Completed - data_store.update_deployment(deployment) - logging.info(f"Deployment {deployment.name!r} marked as completed") + if deployment.status.progress == OperationProgress.New: + try: + backend.run_deployment(deployment) + except Exception: + logging.exception("Failed running the deployment") + deployment.status.progress = OperationProgress.Failed + data_store.update_deployment(deployment) + else: + deployment.status.progress = OperationProgress.Completed + data_store.update_deployment(deployment) + logging.info(f"Deployment {deployment.name!r} marked as completed") + elif deployment.status.progress == OperationProgress.ToUndeploy: + try: + backend.undeploy(deployment) + except Exception: + logging.exception("Failed running the undeployment") + deployment.status.progress = OperationProgress.UndeployFailed + data_store.update_deployment(deployment) + else: + deployment.status.progress = OperationProgress.Undeployed + data_store.update_deployment(deployment) + logging.info(f"Undeployment of {deployment.name!r} complete") events_iterator, cancel = data_store.watch_deployments() logging.info("Running the worker's main loop") diff --git a/patches/destroy-only-baremetal.patch b/patches/destroy-only-baremetal.patch new file mode 100644 index 0000000000000000000000000000000000000000..8b99a25f32f1bd14d769295bd2b4ca9278c3c7f4 --- /dev/null +++ b/patches/destroy-only-baremetal.patch @@ -0,0 +1,11 @@ +diff --git a/ansible/destroy.yml b/ansible/destroy.yml +index 9d302bb34..bb569872c 100644 +--- a/ansible/destroy.yml ++++ b/ansible/destroy.yml +@@ -1,5 +1,5 @@ + --- + - name: Apply role destroy +- hosts: all ++ hosts: baremetal + roles: + - destroy diff --git a/patches/ovs-no-external-interface.patch b/patches/ovs-no-external-interface.patch new file mode 100644 index 0000000000000000000000000000000000000000..c95c67cd8938f1f772a21fc688e9fb8a2034d289 --- /dev/null +++ b/patches/ovs-no-external-interface.patch @@ -0,0 +1,39 @@ +diff --git a/ansible/roles/neutron/templates/ml2_conf.ini.j2 b/ansible/roles/neutron/templates/ml2_conf.ini.j2 +index e55423e33..eb079cc2c 100644 +--- a/ansible/roles/neutron/templates/ml2_conf.ini.j2 ++++ b/ansible/roles/neutron/templates/ml2_conf.ini.j2 +@@ -24,7 +24,7 @@ network_vlan_ranges = + {% if enable_ironic | bool %} + flat_networks = * + {% else %} +-flat_networks = {% for interface in neutron_external_interface.split(',') %}physnet{{ loop.index0 + 1 }}{% if not loop.last %},{% endif %}{% endfor %} ++flat_networks = {% if neutron_external_interface | length > 0 %}{% for interface in neutron_external_interface.split(',') %}physnet{{ loop.index0 + 1 }}{% if not loop.last %},{% endif %}{% endfor %}{% endif %} + {% endif %} + + [ml2_type_vxlan] +diff --git a/ansible/roles/neutron/templates/openvswitch_agent.ini.j2 b/ansible/roles/neutron/templates/openvswitch_agent.ini.j2 +index 88834e2de..d0e5dcb8f 100644 +--- a/ansible/roles/neutron/templates/openvswitch_agent.ini.j2 ++++ b/ansible/roles/neutron/templates/openvswitch_agent.ini.j2 +@@ -14,7 +14,7 @@ extensions = {{ neutron_agent_extensions|map(attribute='name')|join(',') }} + firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver + + [ovs] +-{% if inventory_hostname in groups["network"] or (inventory_hostname in groups["compute"] and computes_need_external_bridge | bool ) %} ++{% if (inventory_hostname in groups["network"] or (inventory_hostname in groups["compute"] and computes_need_external_bridge | bool )) and neutron_bridge_name | length > 0 %} + bridge_mappings = {% for bridge in neutron_bridge_name.split(',') %}physnet{{ loop.index0 + 1 }}:{{ bridge }}{% if not loop.last %},{% endif %}{% endfor %} + {% endif %} + datapath_type = {{ ovs_datapath }} +diff --git a/ansible/roles/openvswitch/tasks/post-config.yml b/ansible/roles/openvswitch/tasks/post-config.yml +index ce412b884..e18b46cbb 100644 +--- a/ansible/roles/openvswitch/tasks/post-config.yml ++++ b/ansible/roles/openvswitch/tasks/post-config.yml +@@ -12,6 +12,8 @@ + when: + - inventory_hostname in groups["network"] + or (inventory_hostname in groups["compute"] and computes_need_external_bridge | bool ) ++ - neutron_bridge_name | length > 0 ++ - neutron_external_interface | length > 0 + with_together: + - "{{ neutron_bridge_name.split(',') }}" + - "{{ neutron_external_interface.split(',') }}"