zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

allow group_arguments to take a prefix length

Browse Source 
Signed-off-by: Todd Gamblin <tgamblin@llnl.gov>
This commit is contained in:
Todd Gamblin 2025-05-19 17:05:46 -07:00
parent 52ce977b93
commit 0b0cf998b4
No known key found for this signature in database
GPG Key ID: C16729F1AACF66C6
3 changed files with 21 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
lib/spack/spack/cmd/__init__.py
View File

@ -705,7 +705,11 @@ def first_line(docstring):
def group_arguments( def group_arguments(
args: Sequence[str], max_group_size: int = 500, max_group_len: Optional[int] = None args: Sequence[str],
*,
max_group_size: int = 500,
prefix_length: int = 0,
max_group_length: Optional[int] = None,
) -> Generator[List[str], None, None]: ) -> Generator[List[str], None, None]:
"""Splits the supplied list of arguments into groups for passing to CLI tools. """Splits the supplied list of arguments into groups for passing to CLI tools.
@ -722,30 +726,32 @@ def group_arguments(
Arguments: Arguments:
args: list of arguments to split into groups args: list of arguments to split into groups
max_group_size: max number of elements in any group (default 500) max_group_size: max number of elements in any group (default 500)
max_group_len: max length of characters that if a group of args is joined by " " prefix_length: length of any additional arguments to be passed before the groups from args;
defaults to 0 characters.
max_group_length: max length of characters that if a group of args is joined by " "
On unix, ths defaults to SC_ARG_MAX from sysconf. On Windows the default is On unix, ths defaults to SC_ARG_MAX from sysconf. On Windows the default is
the max usable for CreateProcess (32,768 chars) the max usable for CreateProcess (32,768 chars)
""" """
if max_group_len is None: if max_group_length is None:
max_group_len = 32768 # default to the Windows limit max_group_length = 32768 # default to the Windows limit
if hasattr(os, "sysconf"): if hasattr(os, "sysconf"):
# sysconf is only on unix and returns -1 if an option isn't present # sysconf is only on unix and returns -1 if an option isn't present
sysconf_max = os.sysconf("SC_ARG_MAX") sysconf_max = os.sysconf("SC_ARG_MAX")
if sysconf_max != -1: if sysconf_max != -1:
max_group_len = sysconf_max max_group_length = sysconf_max
group: List[str] = [] group: List[str] = []
grouplen, space = 0, 0 grouplen, space = prefix_length, 0
for i, arg in enumerate(args): for i, arg in enumerate(args):
arglen = len(arg) arglen = len(arg)
if arglen > max_group_len: if arglen > max_group_length:
raise ValueError(f"Argument is longer than the maximum command line size: '{arg}'") raise ValueError(f"Argument is longer than the maximum command line size: '{arg}'")
next_grouplen = grouplen + arglen + space next_grouplen = grouplen + arglen + space
if len(group) == max_group_size or next_grouplen > max_group_len: if len(group) == max_group_size or next_grouplen > max_group_length:
yield group yield group
group, grouplen, space = [], 0, 0 group, grouplen, space = [], prefix_length, 0
group.append(arg) group.append(arg)
grouplen += arglen + space grouplen += arglen + space

3
lib/spack/spack/cmd/pkg.py
View File

@ -186,7 +186,8 @@ def pkg_grep(args, unknown_args):
return 0 # no packages to search return 0 # no packages to search
# set up iterator and save the first group to ensure we don't end up with a group of size 1 # set up iterator and save the first group to ensure we don't end up with a group of size 1
groups = spack.cmd.group_arguments(all_paths) prefix_length = len(" ".join(args.grep_args + unknown_args))
groups = spack.cmd.group_arguments(all_paths, prefix_length=prefix_length)
# You can force GNU grep to show filenames on every line with -H, but not POSIX grep. # You can force GNU grep to show filenames on every line with -H, but not POSIX grep.
# POSIX grep only shows filenames when you're grepping 2 or more files. Since we # POSIX grep only shows filenames when you're grepping 2 or more files. Since we

8
lib/spack/spack/test/cmd/pkg.py
View File

@ -322,7 +322,7 @@ def test_pkg_hash(mock_packages):
@pytest.mark.parametrize( @pytest.mark.parametrize(
["max_group_size", "max_group_len", "lengths", "error"], ["max_group_size", "max_group_length", "lengths", "error"],
[ [
(3, 1, None, ValueError), (3, 1, None, ValueError),
(3, 13, None, ValueError), (3, 13, None, ValueError),
@ -334,9 +334,9 @@ def test_pkg_hash(mock_packages):
(4, 56, [4, 4, 2], None), (4, 56, [4, 4, 2], None),
], ],
) )
def test_group_arguments(mock_packages, max_group_size, max_group_len, lengths, error): def test_group_arguments(mock_packages, max_group_size, max_group_length, lengths, error):
generator = spack.cmd.group_arguments( generator = spack.cmd.group_arguments(
group_args, max_group_size=max_group_size, max_group_len=max_group_len group_args, max_group_size=max_group_size, max_group_length=max_group_length
) )
# just check that error cases raise # just check that error cases raise
@ -349,7 +349,7 @@ def test_group_arguments(mock_packages, max_group_size, max_group_len, lengths,
assert sum(groups, []) == group_args assert sum(groups, []) == group_args
assert [len(group) for group in groups] == lengths assert [len(group) for group in groups] == lengths
assert all( assert all(
sum(len(elt) for elt in group) + (len(group) - 1) <= max_group_len for group in groups sum(len(elt) for elt in group) + (len(group) - 1) <= max_group_length for group in groups
) )