zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

spack create: add depends_on(<lang>) statements (#45296)

Browse Source
This commit is contained in:
Harmen Stoppels 2024-08-23 10:33:05 +02:00 committed by GitHub
parent d40f847497
commit b8cbbb8e2e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 236 additions and 106 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

284
lib/spack/spack/cmd/create.py
View File

@ -6,6 +6,7 @@
import re
import sys
import urllib.parse
from typing import List
import llnl.util.tty as tty
from llnl.util.filesystem import mkdirp
@ -14,9 +15,15 @@
import spack.stage
import spack.util.web
from spack.spec import Spec
from spack.url import UndetectableNameError, UndetectableVersionError, parse_name, parse_version
from spack.url import (
UndetectableNameError,
UndetectableVersionError,
find_versions_of_archive,
parse_name,
parse_version,
)
from spack.util.editor import editor
from spack.util.executable import ProcessError, which
from spack.util.executable import which
from spack.util.format import get_version_lines
from spack.util.naming import mod_to_class, simplify_name, valid_fully_qualified_module_name
@ -89,14 +96,20 @@ class BundlePackageTemplate:
url_def = " # There is no URL since there is no code to download."
body_def = " # There is no need for install() since there is no code."
def __init__(self, name, versions):
def __init__(self, name: str, versions, languages: List[str]):
self.name = name
self.class_name = mod_to_class(name)
self.versions = versions
self.languages = languages
def write(self, pkg_path):
"""Writes the new package file."""
all_deps = [f' depends_on("{lang}", type="build")' for lang in self.languages]
if all_deps and self.dependencies:
all_deps.append("")
all_deps.append(self.dependencies)
# Write out a template for the file
with open(pkg_path, "w") as pkg_file:
pkg_file.write(
@ -106,7 +119,7 @@ def write(self, pkg_path):
base_class_name=self.base_class_name,
url_def=self.url_def,
versions=self.versions,
dependencies=self.dependencies,
dependencies="\n".join(all_deps),
body_def=self.body_def,
)
)
@ -125,8 +138,8 @@ def install(self, spec, prefix):
url_line = ' url = "{url}"'
def __init__(self, name, url, versions):
super().__init__(name, versions)
def __init__(self, name, url, versions, languages: List[str]):
super().__init__(name, versions, languages)
self.url_def = self.url_line.format(url=url)
@ -214,13 +227,13 @@ def luarocks_args(self):
args = []
return args"""
def __init__(self, name, url, *args, **kwargs):
def __init__(self, name, url, versions, languages: List[str]):
# If the user provided `--name lua-lpeg`, don't rename it lua-lua-lpeg
if not name.startswith("lua-"):
# Make it more obvious that we are renaming the package
tty.msg("Changing package name from {0} to lua-{0}".format(name))
name = "lua-{0}".format(name)
super().__init__(name, url, *args, **kwargs)
super().__init__(name, url, versions, languages)
class MesonPackageTemplate(PackageTemplate):
@ -321,14 +334,14 @@ class RacketPackageTemplate(PackageTemplate):
# subdirectory = None
"""
def __init__(self, name, url, *args, **kwargs):
def __init__(self, name, url, versions, languages: List[str]):
# If the user provided `--name rkt-scribble`, don't rename it rkt-rkt-scribble
if not name.startswith("rkt-"):
# Make it more obvious that we are renaming the package
tty.msg("Changing package name from {0} to rkt-{0}".format(name))
name = "rkt-{0}".format(name)
self.body_def = self.body_def.format(name[4:])
super().__init__(name, url, *args, **kwargs)
super().__init__(name, url, versions, languages)
class PythonPackageTemplate(PackageTemplate):
@ -361,7 +374,7 @@ def config_settings(self, spec, prefix):
settings = {}
return settings"""
def __init__(self, name, url, *args, **kwargs):
def __init__(self, name, url, versions, languages: List[str]):
# If the user provided `--name py-numpy`, don't rename it py-py-numpy
if not name.startswith("py-"):
# Make it more obvious that we are renaming the package
@ -415,7 +428,7 @@ def __init__(self, name, url, *args, **kwargs):
+ self.url_line
)
super().__init__(name, url, *args, **kwargs)
super().__init__(name, url, versions, languages)
class RPackageTemplate(PackageTemplate):
@ -434,7 +447,7 @@ def configure_args(self):
args = []
return args"""
def __init__(self, name, url, *args, **kwargs):
def __init__(self, name, url, versions, languages: List[str]):
# If the user provided `--name r-rcpp`, don't rename it r-r-rcpp
if not name.startswith("r-"):
# Make it more obvious that we are renaming the package
@ -454,7 +467,7 @@ def __init__(self, name, url, *args, **kwargs):
if bioc:
self.url_line = ' url = "{0}"\n' ' bioc = "{1}"'.format(url, r_name)
super().__init__(name, url, *args, **kwargs)
super().__init__(name, url, versions, languages)
class PerlmakePackageTemplate(PackageTemplate):
@ -474,14 +487,14 @@ def configure_args(self):
args = []
return args"""
def __init__(self, name, *args, **kwargs):
def __init__(self, name, url, versions, languages: List[str]):
# If the user provided `--name perl-cpp`, don't rename it perl-perl-cpp
if not name.startswith("perl-"):
# Make it more obvious that we are renaming the package
tty.msg("Changing package name from {0} to perl-{0}".format(name))
name = "perl-{0}".format(name)
super().__init__(name, *args, **kwargs)
super().__init__(name, url, versions, languages)
class PerlbuildPackageTemplate(PerlmakePackageTemplate):
@ -506,7 +519,7 @@ class OctavePackageTemplate(PackageTemplate):
# FIXME: Add additional dependencies if required.
# depends_on("octave-foo", type=("build", "run"))"""
def __init__(self, name, *args, **kwargs):
def __init__(self, name, url, versions, languages: List[str]):
# If the user provided `--name octave-splines`, don't rename it
# octave-octave-splines
if not name.startswith("octave-"):
@ -514,7 +527,7 @@ def __init__(self, name, *args, **kwargs):
tty.msg("Changing package name from {0} to octave-{0}".format(name))
name = "octave-{0}".format(name)
super().__init__(name, *args, **kwargs)
super().__init__(name, url, versions, languages)
class RubyPackageTemplate(PackageTemplate):
@ -534,7 +547,7 @@ def build(self, spec, prefix):
# FIXME: If not needed delete this function
pass"""
def __init__(self, name, *args, **kwargs):
def __init__(self, name, url, versions, languages: List[str]):
# If the user provided `--name ruby-numpy`, don't rename it
# ruby-ruby-numpy
if not name.startswith("ruby-"):
@ -542,7 +555,7 @@ def __init__(self, name, *args, **kwargs):
tty.msg("Changing package name from {0} to ruby-{0}".format(name))
name = "ruby-{0}".format(name)
super().__init__(name, *args, **kwargs)
super().__init__(name, url, versions, languages)
class MakefilePackageTemplate(PackageTemplate):
@ -580,14 +593,14 @@ def configure_args(self, spec, prefix):
args = []
return args"""
def __init__(self, name, *args, **kwargs):
def __init__(self, name, url, versions, languages: List[str]):
# If the user provided `--name py-pyqt4`, don't rename it py-py-pyqt4
if not name.startswith("py-"):
# Make it more obvious that we are renaming the package
tty.msg("Changing package name from {0} to py-{0}".format(name))
name = "py-{0}".format(name)
super().__init__(name, *args, **kwargs)
super().__init__(name, url, versions, languages)
templates = {
@ -658,8 +671,48 @@ def setup_parser(subparser):
)
class BuildSystemGuesser:
"""An instance of BuildSystemGuesser provides a callable object to be used
#: C file extensions
C_EXT = {".c"}
#: C++ file extensions
CXX_EXT = {
".C",
".c++",
".cc",
".ccm",
".cpp",
".CPP",
".cxx",
".h++",
".hh",
".hpp",
".hxx",
".inl",
".ipp",
".ixx",
".tcc",
".tpp",
}
#: Fortran file extensions
FORTRAN_EXT = {
".f77",
".F77",
".f90",
".F90",
".f95",
".F95",
".f",
".F",
".for",
".FOR",
".ftn",
".FTN",
}
class BuildSystemAndLanguageGuesser:
"""An instance of BuildSystemAndLanguageGuesser provides a callable object to be used
during ``spack create``. By passing this object to ``spack checksum``, we
can take a peek at the fetched tarball and discern the build system it uses
"""
@ -667,81 +720,119 @@ class BuildSystemGuesser:
def __init__(self):
"""Sets the default build system."""
self.build_system = "generic"
self._c = False
self._cxx = False
self._fortran = False
def __call__(self, stage, url):
# List of files in the archive ordered by their depth in the directory tree.
self._file_entries: List[str] = []
def __call__(self, archive: str, url: str) -> None:
"""Try to guess the type of build system used by a project based on
the contents of its archive or the URL it was downloaded from."""
if url is not None:
# Most octave extensions are hosted on Octave-Forge:
# https://octave.sourceforge.net/index.html
# They all have the same base URL.
if "downloads.sourceforge.net/octave/" in url:
self.build_system = "octave"
return
if url.endswith(".gem"):
self.build_system = "ruby"
return
if url.endswith(".whl") or ".whl#" in url:
self.build_system = "python"
return
if url.endswith(".rock"):
self.build_system = "lua"
return
# A list of clues that give us an idea of the build system a package
# uses. If the regular expression matches a file contained in the
# archive, the corresponding build system is assumed.
# NOTE: Order is important here. If a package supports multiple
# build systems, we choose the first match in this list.
clues = [
(r"/CMakeLists\.txt$", "cmake"),
(r"/NAMESPACE$", "r"),
(r"/Cargo\.toml$", "cargo"),
(r"/go\.mod$", "go"),
(r"/configure$", "autotools"),
(r"/configure\.(in|ac)$", "autoreconf"),
(r"/Makefile\.am$", "autoreconf"),
(r"/pom\.xml$", "maven"),
(r"/SConstruct$", "scons"),
(r"/waf$", "waf"),
(r"/pyproject.toml", "python"),
(r"/setup\.(py|cfg)$", "python"),
(r"/WORKSPACE$", "bazel"),
(r"/Build\.PL$", "perlbuild"),
(r"/Makefile\.PL$", "perlmake"),
(r"/.*\.gemspec$", "ruby"),
(r"/Rakefile$", "ruby"),
(r"/setup\.rb$", "ruby"),
(r"/.*\.pro$", "qmake"),
(r"/.*\.rockspec$", "lua"),
(r"/(GNU)?[Mm]akefile$", "makefile"),
(r"/DESCRIPTION$", "octave"),
(r"/meson\.build$", "meson"),
(r"/configure\.py$", "sip"),
]
# Peek inside the compressed file.
if stage.archive_file.endswith(".zip") or ".zip#" in stage.archive_file:
if archive.endswith(".zip") or ".zip#" in archive:
try:
unzip = which("unzip")
output = unzip("-lq", stage.archive_file, output=str)
except ProcessError:
assert unzip is not None
output = unzip("-lq", archive, output=str)
except Exception:
output = ""
else:
try:
tar = which("tar")
output = tar("--exclude=*/*/*", "-tf", stage.archive_file, output=str)
except ProcessError:
assert tar is not None
output = tar("tf", archive, output=str)
except Exception:
output = ""
lines = output.splitlines()
self._file_entries[:] = output.splitlines()
# Determine the build system based on the files contained
# in the archive.
for pattern, bs in clues:
if any(re.search(pattern, line) for line in lines):
self.build_system = bs
break
# Files closest to the root should be considered first when determining build system.
self._file_entries.sort(key=lambda p: p.count("/"))
self._determine_build_system(url)
self._determine_language()
def _determine_build_system(self, url: str) -> None:
# Most octave extensions are hosted on Octave-Forge:
# https://octave.sourceforge.net/index.html
# They all have the same base URL.
if "downloads.sourceforge.net/octave/" in url:
self.build_system = "octave"
elif url.endswith(".gem"):
self.build_system = "ruby"
elif url.endswith(".whl") or ".whl#" in url:
self.build_system = "python"
elif url.endswith(".rock"):
self.build_system = "lua"
elif self._file_entries:
# A list of clues that give us an idea of the build system a package
# uses. If the regular expression matches a file contained in the
# archive, the corresponding build system is assumed.
# NOTE: Order is important here. If a package supports multiple
# build systems, we choose the first match in this list.
clues = [
(re.compile(pattern), build_system)
for pattern, build_system in (
(r"/CMakeLists\.txt$", "cmake"),
(r"/NAMESPACE$", "r"),
(r"/Cargo\.toml$", "cargo"),
(r"/go\.mod$", "go"),
(r"/configure$", "autotools"),
(r"/configure\.(in|ac)$", "autoreconf"),
(r"/Makefile\.am$", "autoreconf"),
(r"/pom\.xml$", "maven"),
(r"/SConstruct$", "scons"),
(r"/waf$", "waf"),
(r"/pyproject.toml", "python"),
(r"/setup\.(py|cfg)$", "python"),
(r"/WORKSPACE$", "bazel"),
(r"/Build\.PL$", "perlbuild"),
(r"/Makefile\.PL$", "perlmake"),
(r"/.*\.gemspec$", "ruby"),
(r"/Rakefile$", "ruby"),
(r"/setup\.rb$", "ruby"),
(r"/.*\.pro$", "qmake"),
(r"/.*\.rockspec$", "lua"),
(r"/(GNU)?[Mm]akefile$", "makefile"),
(r"/DESCRIPTION$", "octave"),
(r"/meson\.build$", "meson"),
(r"/configure\.py$", "sip"),
)
]
# Determine the build system based on the files contained in the archive.
for file in self._file_entries:
for pattern, build_system in clues:
if pattern.search(file):
self.build_system = build_system
return
def _determine_language(self):
for entry in self._file_entries:
_, ext = os.path.splitext(entry)
if not self._c and ext in C_EXT:
self._c = True
elif not self._cxx and ext in CXX_EXT:
self._cxx = True
elif not self._fortran and ext in FORTRAN_EXT:
self._fortran = True
if self._c and self._cxx and self._fortran:
return
@property
def languages(self) -> List[str]:
langs: List[str] = []
if self._c:
langs.append("c")
if self._cxx:
langs.append("cxx")
if self._fortran:
langs.append("fortran")
return langs
def get_name(name, url):
@ -811,7 +902,7 @@ def get_url(url):
def get_versions(args, name):
"""Returns a list of versions and hashes for a package.
Also returns a BuildSystemGuesser object.
Also returns a BuildSystemAndLanguageGuesser object.
Returns default values if no URL is provided.
@ -820,7 +911,7 @@ def get_versions(args, name):
name (str): The name of the package
Returns:
tuple: versions and hashes, and a BuildSystemGuesser object
tuple: versions and hashes, and a BuildSystemAndLanguageGuesser object
"""
# Default version with hash
@ -834,7 +925,7 @@ def get_versions(args, name):
# version("1.2.4")"""
# Default guesser
guesser = BuildSystemGuesser()
guesser = BuildSystemAndLanguageGuesser()
valid_url = True
try:
@ -847,7 +938,7 @@ def get_versions(args, name):
if args.url is not None and args.template != "bundle" and valid_url:
# Find available versions
try:
url_dict = spack.url.find_versions_of_archive(args.url)
url_dict = find_versions_of_archive(args.url)
if len(url_dict) > 1 and not args.batch and sys.stdin.isatty():
url_dict_filtered = spack.stage.interactive_version_filter(url_dict)
if url_dict_filtered is None:
@ -874,7 +965,7 @@ def get_versions(args, name):
return versions, guesser
def get_build_system(template, url, guesser):
def get_build_system(template: str, url: str, guesser: BuildSystemAndLanguageGuesser) -> str:
"""Determine the build system template.
If a template is specified, always use that. Otherwise, if a URL
@ -882,11 +973,10 @@ def get_build_system(template, url, guesser):
build system it uses. Otherwise, use a generic template by default.
Args:
template (str): ``--template`` argument given to ``spack create``
url (str): ``url`` argument given to ``spack create``
args (argparse.Namespace): The arguments given to ``spack create``
guesser (BuildSystemGuesser): The first_stage_function given to
``spack checksum`` which records the build system it detects
template: ``--template`` argument given to ``spack create``
url: ``url`` argument given to ``spack create``
guesser: The first_stage_function given to ``spack checksum`` which records the build
system it detects
Returns:
str: The name of the build system template to use
@ -960,7 +1050,7 @@ def create(parser, args):
build_system = get_build_system(args.template, url, guesser)
# Create the package template object
constr_args = {"name": name, "versions": versions}
constr_args = {"name": name, "versions": versions, "languages": guesser.languages}
package_class = templates[build_system]
if package_class != BundlePackageTemplate:
constr_args["url"] = url

8
lib/spack/spack/stage.py
View File

@ -1179,13 +1179,15 @@ def _fetch_and_checksum(url, options, keep_stage, action_fn=None):
with Stage(url_or_fs, keep=keep_stage) as stage:
# Fetch the archive
stage.fetch()
if action_fn is not None:
archive = stage.archive_file
assert archive is not None, f"Archive not found for {url}"
if action_fn is not None and archive:
# Only run first_stage_function the first time,
# no need to run it every time
action_fn(stage, url)
action_fn(archive, url)
# Checksum the archive and add it to the list
checksum = spack.util.crypto.checksum(hashlib.sha256, stage.archive_file)
checksum = spack.util.crypto.checksum(hashlib.sha256, archive)
return checksum, None
except fs.FailedDownloadError:
return None, f"[WORKER] Failed to fetch {url}"

4
lib/spack/spack/test/build_system_guess.py
View File

@ -56,6 +56,6 @@ def test_build_systems(url_and_build_system):
url, build_system = url_and_build_system
with spack.stage.Stage(url) as stage:
stage.fetch()
guesser = spack.cmd.create.BuildSystemGuesser()
guesser(stage, url)
guesser = spack.cmd.create.BuildSystemAndLanguageGuesser()
guesser(stage.archive_file, url)
assert build_system == guesser.build_system

46
lib/spack/spack/test/cmd/create.py
View File

@ -4,6 +4,7 @@
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
import os
import tarfile
import pytest
@ -154,24 +155,24 @@ def test_create_template_bad_name(mock_test_repo, name, expected):
def test_build_system_guesser_no_stage():
"""Test build system guesser when stage not provided."""
guesser = spack.cmd.create.BuildSystemGuesser()
guesser = spack.cmd.create.BuildSystemAndLanguageGuesser()
# Ensure get the expected build system
with pytest.raises(AttributeError, match="'NoneType' object has no attribute"):
guesser(None, "/the/url/does/not/matter")
def test_build_system_guesser_octave():
def test_build_system_guesser_octave(tmp_path):
"""
Test build system guesser for the special case, where the same base URL
identifies the build system rather than guessing the build system from
files contained in the archive.
"""
url, expected = "downloads.sourceforge.net/octave/", "octave"
guesser = spack.cmd.create.BuildSystemGuesser()
guesser = spack.cmd.create.BuildSystemAndLanguageGuesser()
# Ensure get the expected build system
guesser(None, url)
guesser(str(tmp_path / "archive.tar.gz"), url)
assert guesser.build_system == expected
# Also ensure get the correct template
@ -207,3 +208,40 @@ def _parse_name_offset(path, v):
def test_no_url():
"""Test creation of package without a URL."""
create("--skip-editor", "-n", "create-new-package")
@pytest.mark.parametrize(
"source_files,languages",
[
(["fst.c", "snd.C"], ["c", "cxx"]),
(["fst.c", "snd.cxx"], ["c", "cxx"]),
(["fst.F", "snd.cc"], ["cxx", "fortran"]),
(["fst.f", "snd.c"], ["c", "fortran"]),
(["fst.jl", "snd.py"], []),
],
)
def test_language_and_build_system_detection(tmp_path, source_files, languages):
"""Test that languages are detected from tarball, and the build system is guessed from the
most top-level build system file."""
def add(tar: tarfile.TarFile, name: str, type):
tarinfo = tarfile.TarInfo(name)
tarinfo.type = type
tar.addfile(tarinfo)
tarball = str(tmp_path / "example.tar.gz")
with tarfile.open(tarball, "w:gz") as tar:
add(tar, "./third-party/", tarfile.DIRTYPE)
add(tar, "./third-party/example/", tarfile.DIRTYPE)
add(tar, "./third-party/example/CMakeLists.txt", tarfile.REGTYPE) # false positive
add(tar, "./configure", tarfile.REGTYPE) # actual build system
add(tar, "./src/", tarfile.DIRTYPE)
for file in source_files:
add(tar, f"src/{file}", tarfile.REGTYPE)
guesser = spack.cmd.create.BuildSystemAndLanguageGuesser()
guesser(str(tarball), "https://example.com")
assert guesser.build_system == "autotools"
assert guesser.languages == languages