zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
8a8dcb9479
spack/lib/spack/spack/test/modules/lmod.py
Massimiliano Culpo 8a8dcb9479
modules: unit-tests without polluted user scope (#41041)
2023-11-14 10:29:28 +00:00

521 lines
22 KiB
Python
Raw Blame History

# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
import os
import pytest
import spack.environment as ev
import spack.main
import spack.modules.lmod
import spack.spec
mpich_spec_string = "mpich@3.0.4"
mpileaks_spec_string = "mpileaks"
libdwarf_spec_string = "libdwarf arch=x64-linux"
install = spack.main.SpackCommand("install")
#: Class of the writer tested in this module
writer_cls = spack.modules.lmod.LmodModulefileWriter
pytestmark = [
pytest.mark.not_on_windows("does not run on windows"),
pytest.mark.usefixtures("mock_modules_root"),
]
@pytest.fixture(params=["clang@=12.0.0", "gcc@=10.2.1"])
def compiler(request):
return request.param
@pytest.fixture(
params=[
("mpich@3.0.4", ("mpi",)),
("mpich@3.0.1", []),
("openblas@0.2.15", ("blas",)),
("openblas-with-lapack@0.2.15", ("blas", "lapack")),
("mpileaks@2.3", ("mpi",)),
("mpileaks@2.1", []),
]
)
def provider(request):
return request.param
@pytest.mark.usefixtures("config", "mock_packages")
class TestLmod:
@pytest.mark.regression("37788")
@pytest.mark.parametrize("modules_config", ["core_compilers", "core_compilers_at_equal"])
def test_layout_for_specs_compiled_with_core_compilers(
self, modules_config, module_configuration, factory
):
"""Tests that specs compiled with core compilers are in the 'Core' folder. Also tests that
we can use both ``compiler@version`` and ``compiler@=version`` to specify a core compiler.
"""
module_configuration(modules_config)
module, spec = factory("libelf%clang@12.0.0")
assert "Core" in module.layout.available_path_parts
def test_file_layout(self, compiler, provider, factory, module_configuration):
"""Tests the layout of files in the hierarchy is the one expected."""
module_configuration("complex_hierarchy")
spec_string, services = provider
module, spec = factory(spec_string + "%" + compiler)
layout = module.layout
# Check that the services provided are in the hierarchy
for s in services:
assert s in layout.conf.hierarchy_tokens
# Check that the compiler part of the path has no hash and that it
# is transformed to r"Core" if the compiler is listed among core
# compilers
# Check that specs listed as core_specs are transformed to "Core"
if compiler == "clang@=12.0.0" or spec_string == "mpich@3.0.1":
assert "Core" in layout.available_path_parts
else:
assert compiler.replace("@=", "/") in layout.available_path_parts
# Check that the provider part instead has always an hash even if
# hash has been disallowed in the configuration file
path_parts = layout.available_path_parts
service_part = spec_string.replace("@", "/")
service_part = "-".join([service_part, layout.spec.dag_hash(length=7)])
if "mpileaks" in spec_string:
# It's a user, not a provider, so create the provider string
service_part = layout.spec["mpi"].format("{name}/{version}-{hash:7}")
else:
# Only relevant for providers, not users, of virtuals
assert service_part in path_parts
# Check that multi-providers have repetitions in path parts
repetitions = len([x for x in path_parts if service_part == x])
if spec_string == "openblas-with-lapack@0.2.15":
assert repetitions == 2
elif spec_string == "mpileaks@2.1":
assert repetitions == 0
else:
assert repetitions == 1
def test_compilers_provided_different_name(self, factory, module_configuration):
module_configuration("complex_hierarchy")
module, spec = factory("intel-oneapi-compilers%clang@3.3")
provides = module.conf.provides
assert "compiler" in provides
assert provides["compiler"] == spack.spec.CompilerSpec("oneapi@=3.0")
def test_simple_case(self, modulefile_content, module_configuration):
"""Tests the generation of a simple Lua module file."""
module_configuration("autoload_direct")
content = modulefile_content(mpich_spec_string)
assert "-- -*- lua -*-" in content
assert "whatis([[Name : mpich]])" in content
assert "whatis([[Version : 3.0.4]])" in content
assert 'family("mpi")' in content
def test_autoload_direct(self, modulefile_content, module_configuration):
"""Tests the automatic loading of direct dependencies."""
module_configuration("autoload_direct")
content = modulefile_content(mpileaks_spec_string)
assert len([x for x in content if "depends_on(" in x]) == 2
def test_autoload_all(self, modulefile_content, module_configuration):
"""Tests the automatic loading of all dependencies."""
module_configuration("autoload_all")
content = modulefile_content(mpileaks_spec_string)
assert len([x for x in content if "depends_on(" in x]) == 5
def test_alter_environment(self, modulefile_content, module_configuration):
"""Tests modifications to run-time environment."""
module_configuration("alter_environment")
content = modulefile_content("mpileaks platform=test target=x86_64")
assert len([x for x in content if x.startswith('prepend_path("CMAKE_PREFIX_PATH"')]) == 0
assert len([x for x in content if 'setenv("FOO", "foo")' in x]) == 1
assert len([x for x in content if 'unsetenv("BAR")' in x]) == 1
content = modulefile_content("libdwarf platform=test target=core2")
assert len([x for x in content if x.startswith('prepend_path("CMAKE_PREFIX_PATH"')]) == 0
assert len([x for x in content if 'setenv("FOO", "foo")' in x]) == 0
assert len([x for x in content if 'unsetenv("BAR")' in x]) == 0
def test_prepend_path_separator(self, modulefile_content, module_configuration):
"""Tests that we can use custom delimiters to manipulate path lists."""
module_configuration("module_path_separator")
content = modulefile_content("module-path-separator")
assert len([x for x in content if 'append_path("COLON", "foo", ":")' in x]) == 1
assert len([x for x in content if 'prepend_path("COLON", "foo", ":")' in x]) == 1
assert len([x for x in content if 'remove_path("COLON", "foo", ":")' in x]) == 1
assert len([x for x in content if 'append_path("SEMICOLON", "bar", ";")' in x]) == 1
assert len([x for x in content if 'prepend_path("SEMICOLON", "bar", ";")' in x]) == 1
assert len([x for x in content if 'remove_path("SEMICOLON", "bar", ";")' in x]) == 1
assert len([x for x in content if 'append_path("SPACE", "qux", " ")' in x]) == 1
assert len([x for x in content if 'remove_path("SPACE", "qux", " ")' in x]) == 1
@pytest.mark.regression("11355")
def test_manpath_setup(self, modulefile_content, module_configuration):
"""Tests specific setup of MANPATH environment variable."""
module_configuration("autoload_direct")
# no manpath set by module
content = modulefile_content("mpileaks")
assert len([x for x in content if 'append_path("MANPATH", "", ":")' in x]) == 0
# manpath set by module with prepend_path
content = modulefile_content("module-manpath-prepend")
assert (
len([x for x in content if 'prepend_path("MANPATH", "/path/to/man", ":")' in x]) == 1
)
assert (
len([x for x in content if 'prepend_path("MANPATH", "/path/to/share/man", ":")' in x])
== 1
)
assert len([x for x in content if 'append_path("MANPATH", "", ":")' in x]) == 1
# manpath set by module with append_path
content = modulefile_content("module-manpath-append")
assert len([x for x in content if 'append_path("MANPATH", "/path/to/man", ":")' in x]) == 1
assert len([x for x in content if 'append_path("MANPATH", "", ":")' in x]) == 1
# manpath set by module with setenv
content = modulefile_content("module-manpath-setenv")
assert len([x for x in content if 'setenv("MANPATH", "/path/to/man")' in x]) == 1
assert len([x for x in content if 'append_path("MANPATH", "", ":")' in x]) == 0
@pytest.mark.regression("29578")
def test_setenv_raw_value(self, modulefile_content, module_configuration):
"""Tests that we can set environment variable value without formatting it."""
module_configuration("autoload_direct")
content = modulefile_content("module-setenv-raw")
assert len([x for x in content if 'setenv("FOO", "{{name}}, {name}, {{}}, {}")' in x]) == 1
def test_help_message(self, modulefile_content, module_configuration):
"""Tests the generation of module help message."""
module_configuration("autoload_direct")
content = modulefile_content("mpileaks target=core2")
help_msg = (
"help([[Name : mpileaks]])"
"help([[Version: 2.3]])"
"help([[Target : core2]])"
"help()"
"help([[Mpileaks is a mock package that passes audits]])"
)
assert help_msg in "".join(content)
content = modulefile_content("libdwarf target=core2")
help_msg = (
"help([[Name : libdwarf]])"
"help([[Version: 20130729]])"
"help([[Target : core2]])"
"depends_on("
)
assert help_msg in "".join(content)
content = modulefile_content("module-long-help target=core2")
help_msg = (
"help([[Name : module-long-help]])"
"help([[Version: 1.0]])"
"help([[Target : core2]])"
"help()"
"help([[Package to test long description message generated in modulefile."
"Message too long is wrapped over multiple lines.]])"
)
assert help_msg in "".join(content)
def test_exclude(self, modulefile_content, module_configuration):
"""Tests excluding the generation of selected modules."""
module_configuration("exclude")
content = modulefile_content(mpileaks_spec_string)
assert len([x for x in content if "depends_on(" in x]) == 1
def test_no_hash(self, factory, module_configuration):
"""Makes sure that virtual providers (in the hierarchy) always
include a hash. Make sure that the module file for the spec
does not include a hash if hash_length is 0.
"""
module_configuration("no_hash")
module, spec = factory(mpileaks_spec_string)
path = module.layout.filename
mpi_spec = spec["mpi"]
mpi_element = "{0}/{1}-{2}/".format(
mpi_spec.name, mpi_spec.version, mpi_spec.dag_hash(length=7)
)
assert mpi_element in path
mpileaks_spec = spec
mpileaks_element = "{0}/{1}.lua".format(mpileaks_spec.name, mpileaks_spec.version)
assert path.endswith(mpileaks_element)
def test_no_core_compilers(self, factory, module_configuration):
"""Ensures that missing 'core_compilers' in the configuration file
raises the right exception.
"""
# In this case we miss the entry completely
module_configuration("missing_core_compilers")
module, spec = factory(mpileaks_spec_string)
with pytest.raises(spack.modules.lmod.CoreCompilersNotFoundError):
module.write()
# Here we have an empty list
module_configuration("core_compilers_empty")
module, spec = factory(mpileaks_spec_string)
with pytest.raises(spack.modules.lmod.CoreCompilersNotFoundError):
module.write()
def test_non_virtual_in_hierarchy(self, factory, module_configuration):
"""Ensures that if a non-virtual is in hierarchy, an exception will
be raised.
"""
module_configuration("non_virtual_in_hierarchy")
module, spec = factory(mpileaks_spec_string)
with pytest.raises(spack.modules.lmod.NonVirtualInHierarchyError):
module.write()
def test_conflicts(self, modulefile_content, module_configuration):
"""Tests adding conflicts to the module."""
# This configuration has no error, so check the conflicts directives
# are there
module_configuration("conflicts")
content = modulefile_content("mpileaks")
assert len([x for x in content if x.startswith("conflict")]) == 2
assert len([x for x in content if x == 'conflict("mpileaks")']) == 1
assert len([x for x in content if x == 'conflict("intel/14.0.1")']) == 1
def test_inconsistent_conflict_in_modules_yaml(self, modulefile_content, module_configuration):
"""Tests inconsistent conflict definition in `modules.yaml`."""
# This configuration is inconsistent, check an error is raised
module_configuration("wrong_conflicts")
with pytest.raises(spack.modules.common.ModulesError):
modulefile_content("mpileaks")
def test_override_template_in_package(self, modulefile_content, module_configuration):
"""Tests overriding a template from and attribute in the package."""
module_configuration("autoload_direct")
content = modulefile_content("override-module-templates")
assert "Override successful!" in content
def test_override_template_in_modules_yaml(self, modulefile_content, module_configuration):
"""Tests overriding a template from `modules.yaml`"""
module_configuration("override_template")
content = modulefile_content("override-module-templates")
assert "Override even better!" in content
content = modulefile_content("mpileaks target=x86_64")
assert "Override even better!" in content
@pytest.mark.usefixtures("config")
def test_external_configure_args(self, factory):
# If this package is detected as an external, its configure option line
# in the module file starts with 'unknown'
writer, spec = factory("externaltool")
assert "unknown" in writer.context.configure_options
def test_guess_core_compilers(self, factory, module_configuration, monkeypatch):
"""Check that we can guess core compilers."""
# In this case we miss the entry completely
module_configuration("missing_core_compilers")
# Our mock paths must be detected as system paths
monkeypatch.setattr(spack.util.environment, "SYSTEM_DIRS", ["/path/to"])
# We don't want to really write into user configuration
# when running tests
def no_op_set(*args, **kwargs):
pass
monkeypatch.setattr(spack.config, "set", no_op_set)
# Assert we have core compilers now
writer, _ = factory(mpileaks_spec_string)
assert writer.conf.core_compilers
@pytest.mark.parametrize(
"spec_str", ["mpileaks target=nocona", "mpileaks target=core2", "mpileaks target=x86_64"]
)
@pytest.mark.regression("13005")
def test_only_generic_microarchitectures_in_root(
self, spec_str, factory, module_configuration
):
module_configuration("complex_hierarchy")
writer, spec = factory(spec_str)
assert str(spec.target.family) in writer.layout.arch_dirname
if spec.target.family != spec.target:
assert str(spec.target) not in writer.layout.arch_dirname
def test_projections_specific(self, factory, module_configuration):
"""Tests reading the correct naming scheme."""
# This configuration has no error, so check the conflicts directives
# are there
module_configuration("projections")
# Test we read the expected configuration for the naming scheme
writer, _ = factory("mpileaks")
expected = {"all": "{name}/v{version}", "mpileaks": "{name}-mpiprojection"}
assert writer.conf.projections == expected
projection = writer.spec.format(writer.conf.projections["mpileaks"])
assert projection in writer.layout.use_name
def test_projections_all(self, factory, module_configuration):
"""Tests reading the correct naming scheme."""
# This configuration has no error, so check the conflicts directives
# are there
module_configuration("projections")
# Test we read the expected configuration for the naming scheme
writer, _ = factory("libelf")
expected = {"all": "{name}/v{version}", "mpileaks": "{name}-mpiprojection"}
assert writer.conf.projections == expected
projection = writer.spec.format(writer.conf.projections["all"])
assert projection in writer.layout.use_name
def test_modules_relative_to_view(
self, tmpdir, modulefile_content, module_configuration, install_mockery, mock_fetch
):
with ev.create_in_dir(str(tmpdir), with_view=True) as e:
module_configuration("with_view")
install("--add", "cmake")
spec = spack.spec.Spec("cmake").concretized()
content = modulefile_content("cmake")
expected = e.default_view.get_projection_for_spec(spec)
# Rather than parse all lines, ensure all prefixes in the content
# point to the right one
assert any(expected in line for line in content)
assert not any(spec.prefix in line for line in content)
def test_modules_no_arch(self, factory, module_configuration):
module_configuration("no_arch")
module, spec = factory(mpileaks_spec_string)
path = module.layout.filename
assert str(spec.os) not in path
def test_hide_implicits(self, module_configuration, temporary_store):
"""Tests the addition and removal of hide command in modulerc."""
module_configuration("hide_implicits")
spec = spack.spec.Spec("mpileaks@2.3").concretized()
# mpileaks is defined as implicit, thus hide command should appear in modulerc
writer = writer_cls(spec, "default", False)
writer.write()
assert os.path.exists(writer.layout.modulerc)
with open(writer.layout.modulerc) as f:
content = [line.strip() for line in f.readlines()]
hide_implicit_mpileaks = f'hide_version("{writer.layout.use_name}")'
assert len([x for x in content if hide_implicit_mpileaks == x]) == 1
# The direct dependencies are all implicitly installed, and they should all be hidden,
# except for mpich, which is provider for mpi, which is in the hierarchy, and therefore
# can't be hidden. All other hidden modules should have a 7 character hash (the config
# hash_length = 0 only applies to exposed modules).
with open(writer.layout.filename) as f:
depends_statements = [line.strip() for line in f.readlines() if "depends_on" in line]
for dep in spec.dependencies(deptype=("link", "run")):
if dep.satisfies("mpi"):
assert not any(dep.dag_hash(7) in line for line in depends_statements)
else:
assert any(dep.dag_hash(7) in line for line in depends_statements)
# when mpileaks becomes explicit, its file name changes (hash_length = 0), meaning an
# extra module file is created; the old one still exists and remains hidden.
writer = writer_cls(spec, "default", True)
writer.write()
assert os.path.exists(writer.layout.modulerc)
with open(writer.layout.modulerc) as f:
content = [line.strip() for line in f.readlines()]
assert hide_implicit_mpileaks in content # old, implicit mpileaks is still hidden
assert f'hide_version("{writer.layout.use_name}")' not in content
# after removing both the implicit and explicit module, the modulerc file would be empty
# and should be removed.
writer_cls(spec, "default", False).remove()
writer_cls(spec, "default", True).remove()
assert not os.path.exists(writer.layout.modulerc)
assert not os.path.exists(writer.layout.filename)
# implicit module is removed
writer = writer_cls(spec, "default", False)
writer.write()
assert os.path.exists(writer.layout.filename)
assert os.path.exists(writer.layout.modulerc)
writer.remove()
assert not os.path.exists(writer.layout.modulerc)
assert not os.path.exists(writer.layout.filename)
# three versions of mpileaks are implicit
writer = writer_cls(spec, "default", False)
writer.write(overwrite=True)
spec_alt1 = spack.spec.Spec("mpileaks@2.2").concretized()
spec_alt2 = spack.spec.Spec("mpileaks@2.1").concretized()
writer_alt1 = writer_cls(spec_alt1, "default", False)
writer_alt1.write(overwrite=True)
writer_alt2 = writer_cls(spec_alt2, "default", False)
writer_alt2.write(overwrite=True)
assert os.path.exists(writer.layout.modulerc)
with open(writer.layout.modulerc) as f:
content = [line.strip() for line in f.readlines()]
hide_cmd = f'hide_version("{writer.layout.use_name}")'
hide_cmd_alt1 = f'hide_version("{writer_alt1.layout.use_name}")'
hide_cmd_alt2 = f'hide_version("{writer_alt2.layout.use_name}")'
assert len([x for x in content if hide_cmd == x]) == 1
assert len([x for x in content if hide_cmd_alt1 == x]) == 1
assert len([x for x in content if hide_cmd_alt2 == x]) == 1
# one version is removed
writer_alt1.remove()
assert os.path.exists(writer.layout.modulerc)
with open(writer.layout.modulerc) as f:
content = [line.strip() for line in f.readlines()]
assert len([x for x in content if hide_cmd == x]) == 1
assert len([x for x in content if hide_cmd_alt1 == x]) == 0
assert len([x for x in content if hide_cmd_alt2 == x]) == 1
Reference in New Issue View Git Blame Copy Permalink