diff --git a/Dockerfile b/Dockerfile
index a87b323777267d2f79caaeab81579c0b655eec78..91794c3ef230e7f4bddcadefce6991e43e6fe4aa 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -4,13 +4,19 @@ RUN apt-get update -eany
RUN apt-get upgrade -y --no-install-recommends
+RUN apt-get install -y --no-install-recommends patch
+
RUN /usr/local/bin/python3.8 -m venv /opt/kolla-ansible-venv
# Xena
COPY openstack_requirements/upper-constraints.txt /tmp/
RUN /opt/kolla-ansible-venv/bin/python -m pip install -c /tmp/upper-constraints.txt \
- kolla-ansible==13.4.0 \
+ kolla-ansible==13.8.0 \
ansible==4.10.0
+COPY patches/ovs-no-external-interface.patch /tmp/
+RUN sh -c "cd /opt/kolla-ansible-venv/share/kolla-ansible; patch -p1 < /tmp/ovs-no-external-interface.patch"
+COPY patches/destroy-only-baremetal.patch /tmp/
+RUN sh -c "cd /opt/kolla-ansible-venv/share/kolla-ansible; patch -p1 < /tmp/destroy-only-baremetal.patch"
FROM python:3.10.7-slim-bullseye as csep-build-stage
diff --git a/csep_api/main.py b/csep_api/main.py
index a918272e3f69b25f1c4280b577ac0ceaa03f8cdb..2d1d1d2f548a847180f30ca8bbdae9755d6e28ca 100644
--- a/csep_api/main.py
+++ b/csep_api/main.py
@@ -150,6 +150,24 @@ def redeploy_deployment(
raise HTTPException(status_code=400, detail="Incompatible spec patch")
+@app.post(
+ "/deployments/{deployment_name}/undeploy",
+ response_model=DeploymentOut,
+ response_model_exclude_unset=True,
+)
+def undeploy_deployment(
+ deployment_name: str,
+ data_store: DataStore = Depends(get_data_store),
+) -> DeploymentInDB:
+ try:
+ deployment = data_store.get_deployment(deployment_name)
+ deployment.status.progress = OperationProgress.ToUndeploy
+ data_store.update_deployment(deployment)
+ return deployment
+ except DeploymentNotFoundError:
+ raise HTTPException(status_code=404, detail="Deployment not found")
+
+
@app.delete("/deployments/{deployment_name}")
def delete_deployment(
deployment_name: str, data_store: DataStore = Depends(get_data_store)
diff --git a/csep_common/datamodels.py b/csep_common/datamodels.py
index 46ff6e5d591e731b9d55f19fef6f1ae633609e30..8e1dc9c9d731e294d4bc8c955e9d149886685aa7 100644
--- a/csep_common/datamodels.py
+++ b/csep_common/datamodels.py
@@ -20,6 +20,10 @@ class OperationProgress(str, Enum):
Running = "Running"
Failed = "Failed"
Completed = "Completed"
+ ToUndeploy = "ToUndeploy"
+ Undeploying = "Undeploying"
+ Undeployed = "Undeployed"
+ UndeployFailed = "UndeployFailed"
class EventSeverity(str, Enum):
@@ -77,8 +81,12 @@ class VaultAuth(BaseModel):
key: str
+PlainAuth = UsernamePasswordAuth | UsernameSSHKeyAuth
+AnyAuth = PlainAuth | VaultAuth
+
+
class DeploymentSpecBase(BaseModel):
- auth: Union[UsernamePasswordAuth, UsernameSSHKeyAuth, VaultAuth]
+ auth: AnyAuth
bmc_auth: Optional[UsernamePasswordAuth]
hosts: List[Host] = Field(min_items=1)
diff --git a/csep_common/datastore.py b/csep_common/datastore.py
index b2d9b4659a23899983a4b940e6659e001805c155..38fd94da3e55824ae0f75be2038464bc9b0460fa 100644
--- a/csep_common/datastore.py
+++ b/csep_common/datastore.py
@@ -75,7 +75,7 @@ class Etcd3DataStore(DataStore):
def get_all_deployments(self) -> List[DeploymentInDB]:
result = []
- for (value, _) in self.etcd3_client.get_prefix(self.DEPLOYMENTS_KEY_PREFIX):
+ for value, _ in self.etcd3_client.get_prefix(self.DEPLOYMENTS_KEY_PREFIX):
result.append(DeploymentInDB(**json.loads(value)))
return result
diff --git a/csep_worker/backend.py b/csep_worker/backend.py
index 2a9354e915e1c91d87e7779b495c3907b3bca989..657ab48fb3037bfffd8482f0f0970e4c2fb9dfaf 100644
--- a/csep_worker/backend.py
+++ b/csep_worker/backend.py
@@ -13,3 +13,7 @@ class Backend(ABC):
@abstractmethod
def run_deployment(self, deployment: DeploymentInDB) -> None:
...
+
+ @abstractmethod
+ def undeploy(self, deployment: DeploymentInDB) -> None:
+ ...
diff --git a/csep_worker/backends/dummy.py b/csep_worker/backends/dummy.py
index 934d070bbebab7a3cc0b7063bb3ecf8e85504b9d..7506712b3230a0f05ec4ca013cf6df9253d23df8 100644
--- a/csep_worker/backends/dummy.py
+++ b/csep_worker/backends/dummy.py
@@ -24,3 +24,6 @@ class DummyBackend(Backend):
if not spec.should_succeed:
raise Exception()
+
+ def undeploy(self, deployment: DeploymentInDB) -> None:
+ pass
diff --git a/csep_worker/backends/openstack.py b/csep_worker/backends/openstack.py
index 1fe5e3322e011d1760a4375c4c2f03baf358ba55..1496df4fcc5c56532af3ae7c404d59a300f13505 100644
--- a/csep_worker/backends/openstack.py
+++ b/csep_worker/backends/openstack.py
@@ -10,14 +10,17 @@ from typing import List, cast
import yaml
from csep_common.datamodels import (
+ AnyAuth,
DeploymentInDB,
Event,
EventSeverity,
Host,
OpenStackDeploymentSpec,
OperationProgress,
+ PlainAuth,
UsernamePasswordAuth,
UsernameSSHKeyAuth,
+ VaultAuth,
)
from csep_worker.backend import Backend
from csep_worker.vault_client import vault_client
@@ -32,6 +35,9 @@ KOLLA_ANSIBLE_SITE_PLAYBOOK = path.join(
KOLLA_ANSIBLE_KOLLA_HOST_PLAYBOOK = path.join(
KOLLA_BASE, "share/kolla-ansible/ansible/kolla-host.yml"
)
+KOLLA_ANSIBLE_DESTROY_PLAYBOOK = path.join(
+ KOLLA_BASE, "share/kolla-ansible/ansible/destroy.yml"
+)
KOLLA_PASSWORDS_YAML = path.join(
KOLLA_BASE, "share/kolla-ansible/etc_examples/kolla/passwords.yml"
)
@@ -52,6 +58,7 @@ KOLLA_GLOBALS_DEFAULTS = {
"enable_haproxy": False,
"kolla_internal_vip_address": "{{ 'api' | kolla_address(groups.control | first) }}",
"neutron_external_interface": "",
+ "neutron_bridge_name": "",
}
@@ -69,6 +76,105 @@ def private_file_opener(path: str, flags: int) -> int:
return os.open(path, flags, mode=0o0600)
+def gen_host_entry(host: Host, ansible_auth_str: str) -> str:
+ # TODO: it's probably nicer to do it with array and join later
+
+ if host.name is None:
+ name = "host_" + host.ip_address.replace(".", "_")
+ else:
+ name = host.name
+ ip_address = host.ip_address
+
+ entry = f"{name} ansible_host={ip_address}"
+
+ if host.port is not None:
+ entry += f" ansible_port={host.port}"
+
+ entry += ansible_auth_str
+
+ if host.network_interface_name is None:
+ network_interface_name = "eth0"
+ else:
+ network_interface_name = host.network_interface_name
+
+ entry += f" network_interface={network_interface_name}"
+
+ return entry
+
+
+def unpack_vault_auth(auth: VaultAuth) -> PlainAuth:
+ # need to establish the type dynamically
+ s = vault_client.read_secret(auth.key)
+ if "username" in s and "password" in s:
+ return UsernamePasswordAuth(
+ type="username_password",
+ username=s["username"],
+ password=s["password"],
+ )
+ elif "username" in s and "private_ssh_key" in s:
+ return UsernameSSHKeyAuth(
+ type="username_sshkey",
+ username=s["username"],
+ private_ssh_key=s["private_ssh_key"],
+ )
+ else:
+ # TODO: use a custom exception type
+ raise Exception("Unknown auth type")
+
+
+def ensure_plain_auth(auth: AnyAuth) -> PlainAuth:
+ if auth.type == "vault":
+ return unpack_vault_auth(auth)
+ else:
+ return auth
+
+
+def plain_auth_to_ansible_auth_str(
+ auth: PlainAuth, ssh_private_key_file_path: str
+) -> str:
+ # NOTE: we are not using match/case because of mypy failing to
+ # recognise that pattern of type hint
+ if auth.type == "username_sshkey":
+ with open(ssh_private_key_file_path, "w", opener=private_file_opener) as f:
+ # TODO: validate the key format
+ f.write(auth.private_ssh_key)
+ # write an empty line in case it was missing (very likely)
+ # in the input string
+ f.write("\n")
+
+ return (
+ f" ansible_user={auth.username}"
+ f" ansible_ssh_private_key_file={ssh_private_key_file_path}"
+ )
+ else:
+ return f" ansible_user={auth.username}" f" ansible_password={auth.password}"
+
+
+def gen_inventory(
+ hosts: list[Host], inventory_file_path: str, ansible_auth_str: str
+) -> None:
+ # TODO: validate if roles are coherent and avoid the "I use a single one
+ # below" (also in kolla_external_fqdn)
+
+ inventory_lines: List[str] = []
+ inventory_lines.append("[control]")
+ inventory_lines.append(gen_host_entry(hosts[0], ansible_auth_str))
+ inventory_lines.append("[network]")
+ inventory_lines.append(gen_host_entry(hosts[0], ansible_auth_str))
+ inventory_lines.append("[compute]")
+ for host in hosts:
+ if host.role is None or host.role == "compute":
+ inventory_lines.append(gen_host_entry(host, ansible_auth_str))
+ inventory_lines.append(gen_host_entry(hosts[0], ansible_auth_str))
+ inventory_lines.append("[monitoring]")
+ inventory_lines.append(gen_host_entry(hosts[0], ansible_auth_str))
+ inventory_lines.append("[storage]")
+ inventory_lines.append(gen_host_entry(hosts[0], ansible_auth_str))
+ with open(inventory_file_path, "w") as f:
+ f.writelines((f"{line}\n" for line in inventory_lines))
+ f.write(TRAILING_INVENTORY)
+
+
class OpenStackBackend(Backend):
def run_deployment(self, deployment: DeploymentInDB) -> None:
spec = cast(OpenStackDeploymentSpec, deployment.spec)
@@ -77,103 +183,19 @@ class OpenStackBackend(Backend):
self.data_store.update_deployment(deployment)
logging.info(f"Deployment {deployment.name!r} marked running")
- auth: UsernamePasswordAuth | UsernameSSHKeyAuth
- if spec.auth.type == "vault":
- # need to establish the type dynamically
- s = vault_client.read_secret(spec.auth.key)
- if "username" in s and "password" in s:
- auth = UsernamePasswordAuth(
- type="username_password",
- username=s["username"],
- password=s["password"],
- )
- elif "username" in s and "private_ssh_key" in s:
- auth = UsernameSSHKeyAuth(
- type="username_sshkey",
- username=s["username"],
- private_ssh_key=s["private_ssh_key"],
- )
- else:
- # TODO: use a custom exception type
- raise Exception("Unknown auth type")
- else:
- auth = spec.auth
-
- def gen_host_entry(host: Host) -> str:
- # TODO: it's probably nicer to do it with array and join later
-
- if host.name is None:
- name = "host_" + host.ip_address.replace(".", "_")
- else:
- name = host.name
- ip_address = host.ip_address
-
- entry = f"{name} ansible_host={ip_address}"
-
- if host.port is not None:
- entry += f" ansible_port={host.port}"
-
- entry += ansible_auth_str
-
- if host.network_interface_name is None:
- network_interface_name = "eth0"
- else:
- network_interface_name = host.network_interface_name
-
- entry += f" network_interface={network_interface_name}"
-
- return entry
+ auth = ensure_plain_auth(spec.auth)
with tempfile.TemporaryDirectory() as tmpdirpath:
def gen_file_path(name: str) -> str:
return path.join(tmpdirpath, name)
- # NOTE: we are not using match/case because of mypy failing to
- # recognise that pattern of type hint
- if auth.type == "username_sshkey":
- ssh_private_key_file_path = gen_file_path("ssh_key")
-
- with open(
- ssh_private_key_file_path, "w", opener=private_file_opener
- ) as f:
- # TODO: validate the key format
- f.write(auth.private_ssh_key)
- # write an empty line in case it was missing (very likely)
- # in the input string
- f.write("\n")
-
- ansible_auth_str = (
- f" ansible_user={auth.username}"
- f" ansible_ssh_private_key_file={ssh_private_key_file_path}"
- )
- elif auth.type == "username_password":
- ansible_auth_str = (
- f" ansible_user={auth.username}"
- f" ansible_password={auth.password}"
- )
-
- # TODO: validate if roles are coherent and avoid the "I use a single one
- # below" (also in kolla_external_fqdn)
+ ansible_auth_str = plain_auth_to_ansible_auth_str(
+ auth, gen_file_path("ssh_key")
+ )
inventory_file_path = gen_file_path("inventory.ini")
- inventory_lines: List[str] = []
- inventory_lines.append("[control]")
- inventory_lines.append(gen_host_entry(spec.hosts[0]))
- inventory_lines.append("[network]")
- inventory_lines.append(gen_host_entry(spec.hosts[0]))
- inventory_lines.append("[compute]")
- for host in spec.hosts:
- if host.role is None or host.role == "compute":
- inventory_lines.append(gen_host_entry(host))
- inventory_lines.append(gen_host_entry(spec.hosts[0]))
- inventory_lines.append("[monitoring]")
- inventory_lines.append(gen_host_entry(spec.hosts[0]))
- inventory_lines.append("[storage]")
- inventory_lines.append(gen_host_entry(spec.hosts[0]))
- with open(inventory_file_path, "w") as f:
- f.writelines((f"{line}\n" for line in inventory_lines))
- f.write(TRAILING_INVENTORY)
+ gen_inventory(spec.hosts, inventory_file_path, ansible_auth_str)
# prepare globals
@@ -254,6 +276,50 @@ class OpenStackBackend(Backend):
# TODO: could we capture the progress as it moves forward?
+ def undeploy(self, deployment: DeploymentInDB) -> None:
+ spec = cast(OpenStackDeploymentSpec, deployment.spec)
+
+ deployment.status.progress = OperationProgress.Undeploying
+ self.data_store.update_deployment(deployment)
+ logging.info(f"Deployment {deployment.name!r} marked undeploying")
+
+ auth = ensure_plain_auth(spec.auth)
+
+ with tempfile.TemporaryDirectory() as tmpdirpath:
+
+ def gen_file_path(name: str) -> str:
+ return path.join(tmpdirpath, name)
+
+ ansible_auth_str = plain_auth_to_ansible_auth_str(
+ auth, gen_file_path("ssh_key")
+ )
+
+ inventory_file_path = gen_file_path("inventory.ini")
+ gen_inventory(spec.hosts, inventory_file_path, ansible_auth_str)
+
+ # prepare globals
+
+ globals_file_path = gen_file_path("globals.yaml")
+
+ with open(globals_file_path, "w") as f:
+ yaml.safe_dump(spec.globals, f)
+
+ # prepare passwords
+
+ passwords_file_path = gen_file_path("passwords.yaml")
+
+ with open(passwords_file_path, "w") as f:
+ yaml.safe_dump(spec.passwords, f)
+
+ self._run_kolla_ansible(
+ deployment,
+ inventory_file_path,
+ globals_file_path,
+ passwords_file_path,
+ tmpdirpath,
+ "destroy",
+ )
+
def _run_kolla_ansible(
self,
deployment: DeploymentInDB,
@@ -263,8 +329,16 @@ class OpenStackBackend(Backend):
config_dir_path: str,
action: str, # TODO: turn into an Enum
) -> None:
+ extra_args: list[str] = []
+
if action == "bootstrap-servers":
playbook = KOLLA_ANSIBLE_KOLLA_HOST_PLAYBOOK
+ elif action == "destroy":
+ playbook = KOLLA_ANSIBLE_DESTROY_PLAYBOOK
+ extra_args = [
+ "-e",
+ "destroy_include_images=yes",
+ ]
else:
playbook = KOLLA_ANSIBLE_SITE_PLAYBOOK
@@ -296,6 +370,7 @@ class OpenStackBackend(Backend):
f"CONFIG_DIR={config_dir_path}",
"-e",
f"kolla_action={action}",
+ *extra_args,
playbook,
],
check=True,
diff --git a/csep_worker/main.py b/csep_worker/main.py
index d588c1738c8d7ec6f31136398752f0a8c0b4745c..907103b51179e24ffb07c1fae33dc4b3c7e30be0 100644
--- a/csep_worker/main.py
+++ b/csep_worker/main.py
@@ -31,23 +31,37 @@ def main_loop() -> None:
def process_put_event(event: PutEvent) -> None:
deployment = DeploymentInDB(**json.loads(event.value))
- # TODO: better check value version
- if deployment.status.progress != OperationProgress.New:
+ if deployment.status.progress not in [
+ OperationProgress.New,
+ OperationProgress.ToUndeploy,
+ ]:
logging.debug("Skipping deployment update event")
return
backend = backends.get(deployment.spec.type)
if backend is not None:
- try:
- backend.run_deployment(deployment)
- except Exception:
- logging.exception("Failed running the deployment")
- deployment.status.progress = OperationProgress.Failed
- data_store.update_deployment(deployment)
- else:
- deployment.status.progress = OperationProgress.Completed
- data_store.update_deployment(deployment)
- logging.info(f"Deployment {deployment.name!r} marked as completed")
+ if deployment.status.progress == OperationProgress.New:
+ try:
+ backend.run_deployment(deployment)
+ except Exception:
+ logging.exception("Failed running the deployment")
+ deployment.status.progress = OperationProgress.Failed
+ data_store.update_deployment(deployment)
+ else:
+ deployment.status.progress = OperationProgress.Completed
+ data_store.update_deployment(deployment)
+ logging.info(f"Deployment {deployment.name!r} marked as completed")
+ elif deployment.status.progress == OperationProgress.ToUndeploy:
+ try:
+ backend.undeploy(deployment)
+ except Exception:
+ logging.exception("Failed running the undeployment")
+ deployment.status.progress = OperationProgress.UndeployFailed
+ data_store.update_deployment(deployment)
+ else:
+ deployment.status.progress = OperationProgress.Undeployed
+ data_store.update_deployment(deployment)
+ logging.info(f"Undeployment of {deployment.name!r} complete")
events_iterator, cancel = data_store.watch_deployments()
logging.info("Running the worker's main loop")
diff --git a/patches/destroy-only-baremetal.patch b/patches/destroy-only-baremetal.patch
new file mode 100644
index 0000000000000000000000000000000000000000..8b99a25f32f1bd14d769295bd2b4ca9278c3c7f4
--- /dev/null
+++ b/patches/destroy-only-baremetal.patch
@@ -0,0 +1,11 @@
+diff --git a/ansible/destroy.yml b/ansible/destroy.yml
+index 9d302bb34..bb569872c 100644
+--- a/ansible/destroy.yml
++++ b/ansible/destroy.yml
+@@ -1,5 +1,5 @@
+ ---
+ - name: Apply role destroy
+- hosts: all
++ hosts: baremetal
+ roles:
+ - destroy
diff --git a/patches/ovs-no-external-interface.patch b/patches/ovs-no-external-interface.patch
new file mode 100644
index 0000000000000000000000000000000000000000..c95c67cd8938f1f772a21fc688e9fb8a2034d289
--- /dev/null
+++ b/patches/ovs-no-external-interface.patch
@@ -0,0 +1,39 @@
+diff --git a/ansible/roles/neutron/templates/ml2_conf.ini.j2 b/ansible/roles/neutron/templates/ml2_conf.ini.j2
+index e55423e33..eb079cc2c 100644
+--- a/ansible/roles/neutron/templates/ml2_conf.ini.j2
++++ b/ansible/roles/neutron/templates/ml2_conf.ini.j2
+@@ -24,7 +24,7 @@ network_vlan_ranges =
+ {% if enable_ironic | bool %}
+ flat_networks = *
+ {% else %}
+-flat_networks = {% for interface in neutron_external_interface.split(',') %}physnet{{ loop.index0 + 1 }}{% if not loop.last %},{% endif %}{% endfor %}
++flat_networks = {% if neutron_external_interface | length > 0 %}{% for interface in neutron_external_interface.split(',') %}physnet{{ loop.index0 + 1 }}{% if not loop.last %},{% endif %}{% endfor %}{% endif %}
+ {% endif %}
+
+ [ml2_type_vxlan]
+diff --git a/ansible/roles/neutron/templates/openvswitch_agent.ini.j2 b/ansible/roles/neutron/templates/openvswitch_agent.ini.j2
+index 88834e2de..d0e5dcb8f 100644
+--- a/ansible/roles/neutron/templates/openvswitch_agent.ini.j2
++++ b/ansible/roles/neutron/templates/openvswitch_agent.ini.j2
+@@ -14,7 +14,7 @@ extensions = {{ neutron_agent_extensions|map(attribute='name')|join(',') }}
+ firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver
+
+ [ovs]
+-{% if inventory_hostname in groups["network"] or (inventory_hostname in groups["compute"] and computes_need_external_bridge | bool ) %}
++{% if (inventory_hostname in groups["network"] or (inventory_hostname in groups["compute"] and computes_need_external_bridge | bool )) and neutron_bridge_name | length > 0 %}
+ bridge_mappings = {% for bridge in neutron_bridge_name.split(',') %}physnet{{ loop.index0 + 1 }}:{{ bridge }}{% if not loop.last %},{% endif %}{% endfor %}
+ {% endif %}
+ datapath_type = {{ ovs_datapath }}
+diff --git a/ansible/roles/openvswitch/tasks/post-config.yml b/ansible/roles/openvswitch/tasks/post-config.yml
+index ce412b884..e18b46cbb 100644
+--- a/ansible/roles/openvswitch/tasks/post-config.yml
++++ b/ansible/roles/openvswitch/tasks/post-config.yml
+@@ -12,6 +12,8 @@
+ when:
+ - inventory_hostname in groups["network"]
+ or (inventory_hostname in groups["compute"] and computes_need_external_bridge | bool )
++ - neutron_bridge_name | length > 0
++ - neutron_external_interface | length > 0
+ with_together:
+ - "{{ neutron_bridge_name.split(',') }}"
+ - "{{ neutron_external_interface.split(',') }}"