zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

unit-test: fix reading Cray manifest files

Browse Source
This commit is contained in:
Massimiliano Culpo 2024-12-18 16:05:58 +01:00
parent 28d42eed5e
commit eb85f2e862
No known key found for this signature in database
GPG Key ID: 3E52BB992233066C
1 changed files with 104 additions and 157 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

261
lib/spack/spack/test/cray_manifest.py
View File

@ -27,6 +27,13 @@
import spack.store import spack.store
from spack.cray_manifest import compiler_from_entry, entries_to_specs from spack.cray_manifest import compiler_from_entry, entries_to_specs
pytestmark = [
pytest.mark.skipif(
str(spack.platforms.host()) != "linux", reason="Cray manifest files are only for linux"
),
pytest.mark.usefixtures("mutable_config", "mock_packages"),
]
class JsonSpecEntry: class JsonSpecEntry:
def __init__(self, name, hash, prefix, version, arch, compiler, dependencies, parameters): def __init__(self, name, hash, prefix, version, arch, compiler, dependencies, parameters):
@ -69,27 +76,24 @@ def compiler_json(self):
class JsonCompilerEntry: class JsonCompilerEntry:
def __init__(self, name, version, arch=None, executables=None): def __init__(self, *, name, version, arch=None, executables=None, prefix=None):
self.name = name self.name = name
self.version = version self.version = version
if not arch: self.arch = arch or JsonArchEntry("anyplatform", "anyos", "anytarget")
arch = JsonArchEntry("anyplatform", "anyos", "anytarget") self.executables = executables or {"cc": "cc", "cxx": "cxx", "fc": "fc"}
if not executables: self.prefix = prefix
executables = {
"cc": "/path/to/compiler/cc",
"cxx": "/path/to/compiler/cxx",
"fc": "/path/to/compiler/fc",
}
self.arch = arch
self.executables = executables
def compiler_json(self): def compiler_json(self):
return { result = {
"name": self.name, "name": self.name,
"version": self.version, "version": self.version,
"arch": self.arch.compiler_json(), "arch": self.arch.compiler_json(),
"executables": self.executables, "executables": self.executables,
} }
# See https://github.com/spack/spack/pull/40061
if self.prefix is not None:
result["prefix"] = self.prefix
return result
def spec_json(self): def spec_json(self):
"""The compiler spec only lists the name/version, not """The compiler spec only lists the name/version, not
@ -178,30 +182,27 @@ def test_manifest_compatibility(_common_arch, _common_compiler, _raw_json_x):
assert x_from_entry == _raw_json_x assert x_from_entry == _raw_json_x
def test_compiler_from_entry(): def test_compiler_from_entry(mock_executable):
compiler_data = json.loads( """Tests that we can detect a compiler from a valid entry in the Cray manifest"""
"""\ cc = mock_executable("gcc", output="echo 7.5.0")
{ cxx = mock_executable("g++", output="echo 7.5.0")
"name": "gcc", fc = mock_executable("gfortran", output="echo 7.5.0")
"prefix": "/path/to/compiler/",
"version": "7.5.0", compiler = compiler_from_entry(
"arch": { JsonCompilerEntry(
"os": "centos8", name="gcc",
"target": "x86_64" version="7.5.0",
}, arch=JsonArchEntry(platform="linux", os="centos8", target="x86_64"),
"executables": { prefix=str(cc.parent),
"cc": "/path/to/compiler/cc", executables={"cc": "gcc", "cxx": "g++", "fc": "gfortran"},
"cxx": "/path/to/compiler/cxx", ).compiler_json(),
"fc": "/path/to/compiler/fc" manifest_path="/example/file",
}
}
"""
) )
compiler = compiler_from_entry(compiler_data, "/example/file")
assert compiler.cc == "/path/to/compiler/cc" assert compiler.satisfies("gcc@7.5.0 target=x86_64 os=centos8")
assert compiler.cxx == "/path/to/compiler/cxx" assert compiler.extra_attributes["compilers"]["c"] == str(cc)
assert compiler.fc == "/path/to/compiler/fc" assert compiler.extra_attributes["compilers"]["cxx"] == str(cxx)
assert compiler.operating_system == "centos8" assert compiler.extra_attributes["compilers"]["fortran"] == str(fc)
@pytest.fixture @pytest.fixture
@ -262,7 +263,7 @@ def the_host_is_linux():
cray_arch = JsonArchEntry(platform="cray", os="rhel8", target="x86_64") cray_arch = JsonArchEntry(platform="cray", os="rhel8", target="x86_64")
spec_json = JsonSpecEntry( spec_json = JsonSpecEntry(
name="cray-mpich", name="mpich",
hash="craympichfakehashaaa", hash="craympichfakehashaaa",
prefix="/path/to/cray-mpich/", prefix="/path/to/cray-mpich/",
version="1.0.0", version="1.0.0",
@ -276,30 +277,12 @@ def the_host_is_linux():
assert spec.architecture.platform == "linux" assert spec.architecture.platform == "linux"
def test_translate_compiler_name(_common_arch): @pytest.mark.parametrize(
nvidia_compiler = JsonCompilerEntry( "name_in_manifest,expected_name",
name="nvidia", [("nvidia", "nvhpc"), ("rocm", "llvm-amdgpu"), ("clang", "llvm")],
version="19.1", )
arch=_common_arch, def test_translated_compiler_name(name_in_manifest, expected_name):
executables={"cc": "/path/to/compiler/nvc", "cxx": "/path/to/compiler/nvc++"}, assert cray_manifest.translated_compiler_name(name_in_manifest) == expected_name
)
compiler = compiler_from_entry(nvidia_compiler.compiler_json(), "/example/file")
assert compiler.name == "nvhpc"
spec_json = JsonSpecEntry(
name="hwloc",
hash="hwlocfakehashaaa",
prefix="/path/to/hwloc-install/",
version="2.0.3",
arch=_common_arch.spec_json(),
compiler=nvidia_compiler.spec_json(),
dependencies={},
parameters={},
).to_dict()
(spec,) = entries_to_specs([spec_json]).values()
assert spec.compiler.name == "nvhpc"
def test_failed_translate_compiler_name(_common_arch): def test_failed_translate_compiler_name(_common_arch):
@ -326,11 +309,9 @@ def test_failed_translate_compiler_name(_common_arch):
@pytest.fixture @pytest.fixture
def manifest_content(generate_openmpi_entries, _common_compiler, _other_compiler): def manifest_content(generate_openmpi_entries, _common_compiler, _other_compiler):
return { return {
# Note: the cray_manifest module doesn't use the _meta section right
# now, but it is anticipated to be useful
"_meta": { "_meta": {
"file-type": "cray-pe-json", "file-type": "cray-pe-json",
"system-type": "test", "system-type": "EX",
"schema-version": "1.3", "schema-version": "1.3",
"cpe-version": "22.06", "cpe-version": "22.06",
}, },
@ -339,86 +320,60 @@ def manifest_content(generate_openmpi_entries, _common_compiler, _other_compiler
} }
def test_read_cray_manifest( def test_read_cray_manifest(temporary_store, manifest_file):
tmpdir, mutable_config, mock_packages, mutable_database, manifest_content
):
"""Check that (a) we can read the cray manifest and add it to the Spack """Check that (a) we can read the cray manifest and add it to the Spack
Database and (b) we can concretize specs based on that. Database and (b) we can concretize specs based on that.
""" """
with tmpdir.as_cwd(): cray_manifest.read(str(manifest_file), True)
test_db_fname = "external-db.json"
with open(test_db_fname, "w", encoding="utf-8") as db_file:
json.dump(manifest_content, db_file)
cray_manifest.read(test_db_fname, True)
query_specs = spack.store.STORE.db.query("openmpi")
assert any(x.dag_hash() == "openmpifakehasha" for x in query_specs)
concretized_specs = spack.cmd.parse_specs( query_specs = temporary_store.db.query("openmpi")
"depends-on-openmpi ^/openmpifakehasha".split(), concretize=True assert any(x.dag_hash() == "openmpifakehasha" for x in query_specs)
)
assert concretized_specs[0]["hwloc"].dag_hash() == "hwlocfakehashaaa" concretized_spec = spack.spec.Spec("depends-on-openmpi ^/openmpifakehasha").concretized()
assert concretized_spec["hwloc"].dag_hash() == "hwlocfakehashaaa"
def test_read_cray_manifest_add_compiler_failure( def test_read_cray_manifest_add_compiler_failure(temporary_store, manifest_file, monkeypatch):
tmpdir, mutable_config, mock_packages, mutable_database, manifest_content, monkeypatch """Tests the Cray manifest can be read even if some compilers cannot be added."""
def _mock(entry, *, manifest_path):
if entry["name"] == "clang":
raise RuntimeError("cannot determine the compiler")
return spack.spec.Spec(f"{entry['name']}@{entry['version']}")
monkeypatch.setattr(cray_manifest, "compiler_from_entry", _mock)
cray_manifest.read(str(manifest_file), True)
query_specs = spack.store.STORE.db.query("openmpi")
assert any(x.dag_hash() == "openmpifakehasha" for x in query_specs)
def test_read_cray_manifest_twice_no_duplicates(
mutable_config, temporary_store, manifest_file, monkeypatch, tmp_path
): ):
"""Check that cray manifest can be read even if some compilers cannot def _mock(entry, *, manifest_path):
be added. return spack.spec.Spec(f"{entry['name']}@{entry['version']}", external_path=str(tmp_path))
"""
orig_add_compiler_to_config = spack.compilers.config.add_compiler_to_config
class fail_for_clang: monkeypatch.setattr(cray_manifest, "compiler_from_entry", _mock)
def __init__(self):
self.called_with_clang = False
def __call__(self, compiler, **kwargs): # Read the manifest twice
if compiler.name == "clang": cray_manifest.read(str(manifest_file), True)
self.called_with_clang = True cray_manifest.read(str(manifest_file), True)
raise Exception()
return orig_add_compiler_to_config(compiler, **kwargs)
checker = fail_for_clang() config_data = mutable_config.get("packages")["gcc"]
monkeypatch.setattr(spack.compilers.config, "add_compiler_to_config", checker) assert "externals" in config_data
with tmpdir.as_cwd(): specs = [spack.spec.Spec(x["spec"]) for x in config_data["externals"]]
test_db_fname = "external-db.json" assert len(specs) == len(set(specs))
with open(test_db_fname, "w", encoding="utf-8") as db_file: assert len([c for c in specs if c.satisfies("gcc@10.2.0.2112")]) == 1
json.dump(manifest_content, db_file)
cray_manifest.read(test_db_fname, True)
query_specs = spack.store.STORE.db.query("openmpi")
assert any(x.dag_hash() == "openmpifakehasha" for x in query_specs)
assert checker.called_with_clang
def test_read_cray_manifest_twice_no_compiler_duplicates( def test_read_old_manifest_v1_2(tmp_path, temporary_store):
tmpdir, mutable_config, mock_packages, mutable_database, manifest_content """Test reading a file using the older format ('version' instead of 'schema-version')."""
): manifest = tmp_path / "manifest_dir" / "test.json"
with tmpdir.as_cwd(): manifest.parent.mkdir(parents=True)
test_db_fname = "external-db.json" manifest.write_text(
with open(test_db_fname, "w", encoding="utf-8") as db_file: """\
json.dump(manifest_content, db_file)
# Read the manifest twice
cray_manifest.read(test_db_fname, True)
cray_manifest.read(test_db_fname, True)
compilers = spack.compilers.config.all_compilers()
filtered = list(
c for c in compilers if c.spec == spack.spec.CompilerSpec("gcc@=10.2.0.2112")
)
assert len(filtered) == 1
def test_read_old_manifest_v1_2(tmpdir, mutable_config, mock_packages, mutable_database):
"""Test reading a file using the older format
('version' instead of 'schema-version').
"""
manifest_dir = str(tmpdir.mkdir("manifest_dir"))
manifest_file_path = os.path.join(manifest_dir, "test.json")
with open(manifest_file_path, "w", encoding="utf-8") as manifest_file:
manifest_file.write(
"""\
{ {
"_meta": { "_meta": {
"file-type": "cray-pe-json", "file-type": "cray-pe-json",
@ -428,11 +383,11 @@ def test_read_old_manifest_v1_2(tmpdir, mutable_config, mock_packages, mutable_d
"specs": [] "specs": []
} }
""" """
) )
cray_manifest.read(manifest_file_path, True) cray_manifest.read(str(manifest), True)
def test_convert_validation_error(tmpdir, mutable_config, mock_packages, mutable_database): def test_convert_validation_error(tmpdir, mutable_config, mock_packages, temporary_store):
manifest_dir = str(tmpdir.mkdir("manifest_dir")) manifest_dir = str(tmpdir.mkdir("manifest_dir"))
# Does not parse as valid JSON # Does not parse as valid JSON
invalid_json_path = os.path.join(manifest_dir, "invalid-json.json") invalid_json_path = os.path.join(manifest_dir, "invalid-json.json")
@ -464,48 +419,40 @@ def test_convert_validation_error(tmpdir, mutable_config, mock_packages, mutable
) )
with pytest.raises(cray_manifest.ManifestValidationError) as e: with pytest.raises(cray_manifest.ManifestValidationError) as e:
cray_manifest.read(invalid_schema_path, True) cray_manifest.read(invalid_schema_path, True)
str(e)
@pytest.fixture @pytest.fixture
def directory_with_manifest(tmpdir, manifest_content): def manifest_file(tmp_path, manifest_content):
"""Create a manifest file in a directory. Used by 'spack external'.""" """Create a manifest file in a directory. Used by 'spack external'."""
with tmpdir.as_cwd(): filename = tmp_path / "external-db.json"
test_db_fname = "external-db.json" with open(filename, "w", encoding="utf-8") as db_file:
with open(test_db_fname, "w", encoding="utf-8") as db_file: json.dump(manifest_content, db_file)
json.dump(manifest_content, db_file) return filename
yield str(tmpdir)
def test_find_external_nonempty_default_manifest_dir( def test_find_external_nonempty_default_manifest_dir(
mutable_database, mutable_mock_repo, tmpdir, monkeypatch, directory_with_manifest temporary_store, mutable_mock_repo, tmpdir, monkeypatch, manifest_file
): ):
"""The user runs 'spack external find'; the default manifest directory """The user runs 'spack external find'; the default manifest directory
contains a manifest file. Ensure that the specs are read. contains a manifest file. Ensure that the specs are read.
""" """
monkeypatch.setenv("PATH", "") monkeypatch.setenv("PATH", "")
monkeypatch.setattr(spack.cray_manifest, "default_path", str(directory_with_manifest)) monkeypatch.setattr(spack.cray_manifest, "default_path", str(manifest_file.parent))
spack.cmd.external._collect_and_consume_cray_manifest_files(ignore_default_dir=False) spack.cmd.external._collect_and_consume_cray_manifest_files(ignore_default_dir=False)
specs = spack.store.STORE.db.query("hwloc") specs = temporary_store.db.query("hwloc")
assert any(x.dag_hash() == "hwlocfakehashaaa" for x in specs) assert any(x.dag_hash() == "hwlocfakehashaaa" for x in specs)
def test_reusable_externals_cray_manifest( def test_reusable_externals_cray_manifest(temporary_store, manifest_file):
tmpdir, mutable_config, mock_packages, temporary_store, manifest_content
):
"""The concretizer should be able to reuse specs imported from a manifest without a """The concretizer should be able to reuse specs imported from a manifest without a
externals config entry in packages.yaml""" externals config entry in packages.yaml"""
with tmpdir.as_cwd(): cray_manifest.read(path=str(manifest_file), apply_updates=True)
with open("external-db.json", "w", encoding="utf-8") as f:
json.dump(manifest_content, f)
cray_manifest.read(path="external-db.json", apply_updates=True)
# Get any imported spec # Get any imported spec
spec = temporary_store.db.query_local()[0] spec = temporary_store.db.query_local()[0]
# Reusable if imported locally # Reusable if imported locally
assert spack.solver.asp._is_reusable(spec, packages={}, local=True) assert spack.solver.asp._is_reusable(spec, packages={}, local=True)
# If cray manifest entries end up in a build cache somehow, they are not reusable # If cray manifest entries end up in a build cache somehow, they are not reusable
assert not spack.solver.asp._is_reusable(spec, packages={}, local=False) assert not spack.solver.asp._is_reusable(spec, packages={}, local=False)