diff --git a/mc_openapi/doml_mc/main.py b/mc_openapi/doml_mc/main.py index dce6d80ec31812870fd6c2fa72ac05cbeed1238c..1a9caa2105f74084ccef858f17e9a2b7627c2041 100644 --- a/mc_openapi/doml_mc/main.py +++ b/mc_openapi/doml_mc/main.py @@ -92,6 +92,8 @@ def verify_model( res = results.summarize() + res['doml_version'] = dmc.doml_version.name + logging.info(res) return res diff --git a/requirements.txt b/requirements.txt index f742a1c9723037135b891bb45f415ad1c3d68480..d03d7da597fea86daaf08eb48ae0f62403f8f17e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -53,4 +53,5 @@ uvicorn==0.20.0 Werkzeug==2.2.2 z3-solver==4.11.2.0 tabulate==0.9.0 -pytest-xdist==3.3.1 \ No newline at end of file +pytest-xdist==3.3.1 +subtests==0.11.0 \ No newline at end of file diff --git a/tests/test_modelchecker.py b/tests/test_modelchecker.py index 1881918a7844d898635fa070cd2bf35e5cff107e..df51edc5375beec929fb578c927e65b357bc408d 100644 --- a/tests/test_modelchecker.py +++ b/tests/test_modelchecker.py @@ -1,6 +1,69 @@ +from importlib.resources import files from mc_openapi.doml_mc import init_model, verify_model +from pathlib import Path -def setup(src, ver): +from mc_openapi.doml_mc.intermediate_model.metamodel import DOMLVersion + +doml_test_dir = files(package='tests') / 'doml' + +def run(src, ver): dmc = init_model(src, ver) - verify_model(dmc) + return verify_model(dmc) + +def test_domlx_models_by_version(subtests): + i = 0 + for doml_ver_dir in [ver for ver in doml_test_dir.iterdir() if ver.name != 'CaseStudies']: + doml_ver = doml_ver_dir.name + domlx_files = [f for f in doml_ver_dir.iterdir() if f.is_file() and f.name.endswith('.domlx')] + for domlx in domlx_files: + with open(domlx, "rb") as f: + domlx_file = f.read() + assert_ver = DOMLVersion.get(doml_ver) + res = run(domlx_file, assert_ver) + try: + assert_result = OUTPUT[doml_ver][domlx.name] + i += 1 + except: + pass + if assert_result: + with subtests.test(msg=f"{doml_ver}/{domlx.name}", i=i): + assert assert_result == res['result'].name + assert assert_ver.name == res['doml_version'] + +OUTPUT = { + 'v2.0': { + 'faas.domlx': 'unsat', + 'nginx-openstack_v2.0.domlx': 'sat', + 'nginx-openstack_v2.0_wrong_all_concrete_map_something.domlx': 'unsat', + 'nginx-openstack_v2.0_wrong_all_infrastructure_elements_deployed.domlx': 'unsat', + 'nginx-openstack_v2.0_wrong_iface_uniq.domlx': 'unsat', + 'nginx-openstack_v2.0_wrong_software_package_iface_net.domlx': 'unsat', + 'nginx-openstack_v2.0_valid_mem_req.domlx': 'sat', + 'nginx-openstack_v2.0_wrong_nginx_source_code.domlx': 'sat', + 'openstack_template.domlx': 'unsat', + 'saas.domlx': 'sat', + 'saas_https_no_attrs.domlx': 'sat', + 'saas_no_https_rule.domlx': 'sat', + 'saas_wrong_no_iface_sg.domlx': 'sat', + 'nginx-openstack_v2.0_wrong_vm_has_iface.domlx': 'unsat', + 'nginx-openstack_v2.0_wrong_all_software_components_deployed.domlx': 'unsat' + }, + 'v2.1': { + 'faas.domlx': 'unsat', + 'nginx-aws-ec2.domlx': 'sat' + }, + 'v2.2': { + 'faas.domlx': 'unsat', + 'iot_simple_app.domlx': 'unsat', + 'nginx-aws-ec2.domlx': 'sat', + 'nginx_func_req2_unsat.domlx': 'unsat', + 'nginx_func_req2_unsat_neg.domlx': 'unsat', + 'nginx_func_req_neg.domlx': 'sat', + 'nginx-csp-compatibility-test.domlx': 'sat', + 'nginx_func_req.domlx': 'sat', + 'nginx_flags.domlx': 'sat' + }, + 'v2.3': { + }, +} \ No newline at end of file