spack external find: allow to search by tags (#21407)

This commit adds an option to the `external find`
command that allows it to search by tags. In this
way group of executables with common purposes can
be grouped under a single name and a simple command
can be used to detect all of them.

As an example introduce the 'build-tools' tag to
search for common development tools on a system

lib/spack/spack/cmd/common/arguments.py

@@ -250,8 +250,8 @@ def very_long():
 @arg
 def tags():
     return Args(
-        '-t', '--tags', action='append',
-        help='filter a package query by tags')
+        '-t', '--tag', action='append', dest='tags', metavar='TAG',
+        help='filter a package query by tag (multiple use allowed)')
 
 
 @arg

lib/spack/spack/cmd/external.py

@@ -16,6 +16,7 @@
 import six
 import spack
 import spack.cmd
+import spack.cmd.common.arguments
 import spack.error
 import spack.util.environment
 import spack.util.spack_yaml as syaml
@@ -42,6 +43,7 @@ def setup_parser(subparser):
         '--scope', choices=scopes, metavar=scopes_metavar,
         default=spack.config.default_modify_scope('packages'),
         help="configuration scope to modify")
+    spack.cmd.common.arguments.add_common_arguments(find_parser, ['tags'])
     find_parser.add_argument('packages', nargs=argparse.REMAINDER)
 
     sp.add_parser(
@@ -148,9 +150,28 @@ def _spec_is_valid(spec):
 
 
 def external_find(args):
+    # Construct the list of possible packages to be detected
+    packages_to_check = []
+
+    # Add the packages that have been required explicitly
     if args.packages:
         packages_to_check = list(spack.repo.get(pkg) for pkg in args.packages)
-    else:
+        if args.tags:
+            allowed = set(spack.repo.path.packages_with_tags(*args.tags))
+            packages_to_check = [x for x in packages_to_check if x in allowed]
+
+    if args.tags and not packages_to_check:
+        # If we arrived here we didn't have any explicit package passed
+        # as argument, which means to search all packages.
+        # Since tags are cached it's much faster to construct what we need
+        # to search directly, rather than filtering after the fact
+        packages_to_check = [
+            spack.repo.get(pkg) for pkg in
+            spack.repo.path.packages_with_tags(*args.tags)
+        ]
+
+    # If the list of packages is empty, search for every possible package
+    if not args.tags and not packages_to_check:
         packages_to_check = spack.repo.path.all_packages()
 
     pkg_to_entries = _get_external_packages(packages_to_check)

lib/spack/spack/test/cmd/external.py

@@ -2,6 +2,8 @@
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
+import pytest
+
 import os
 import os.path
 
@@ -242,3 +244,26 @@ def test_new_entries_are_reported_correctly(
     # has been found
     output = external('find', 'gcc')
     assert 'No new external packages detected' in output
+
+
+@pytest.mark.parametrize('command_args', [
+    ('-t', 'build-tools'),
+    ('-t', 'build-tools', 'cmake'),
+])
+def test_use_tags_for_detection(
+        command_args, mock_executable, mutable_config, monkeypatch
+):
+    # Prepare an environment to detect a fake cmake
+    cmake_exe = mock_executable('cmake', output="echo cmake version 3.19.1")
+    prefix = os.path.dirname(cmake_exe)
+    monkeypatch.setenv('PATH', prefix)
+
+    openssl_exe = mock_executable('openssl', output="OpenSSL 2.8.3")
+    prefix = os.path.dirname(openssl_exe)
+    monkeypatch.setenv('PATH', prefix)
+
+    # Test that we detect specs
+    output = external('find', *command_args)
+    assert 'The following specs have been' in output
+    assert 'cmake' in output
+    assert 'openssl' not in output

lib/spack/spack/test/cmd/find.py

@@ -89,7 +89,7 @@ def test_query_arguments():
 @pytest.mark.usefixtures('database', 'mock_display')
 def test_tag1(parser, specs):
 
-    args = parser.parse_args(['--tags', 'tag1'])
+    args = parser.parse_args(['--tag', 'tag1'])
     spack.cmd.find.find(parser, args)
 
     assert len(specs) == 2
@@ -100,7 +100,7 @@ def test_tag1(parser, specs):
 @pytest.mark.db
 @pytest.mark.usefixtures('database', 'mock_display')
 def test_tag2(parser, specs):
-    args = parser.parse_args(['--tags', 'tag2'])
+    args = parser.parse_args(['--tag', 'tag2'])
     spack.cmd.find.find(parser, args)
 
     assert len(specs) == 1
@@ -110,7 +110,7 @@ def test_tag2(parser, specs):
 @pytest.mark.db
 @pytest.mark.usefixtures('database', 'mock_display')
 def test_tag2_tag3(parser, specs):
-    args = parser.parse_args(['--tags', 'tag2', '--tags', 'tag3'])
+    args = parser.parse_args(['--tag', 'tag2', '--tag', 'tag3'])
     spack.cmd.find.find(parser, args)
 
     assert len(specs) == 0

lib/spack/spack/test/cmd/list.py

@@ -33,15 +33,15 @@ def test_list_search_description():
 
 
 def test_list_tags():
-    output = list('--tags', 'proxy-app')
+    output = list('--tag', 'proxy-app')
     assert 'cloverleaf3d' in output
     assert 'hdf5' not in output
 
-    output = list('--tags', 'hpc')
+    output = list('--tag', 'hpc')
     assert 'nek5000' in output
     assert 'mfem' in output
 
-    output = list('--tags', 'HPC')
+    output = list('--tag', 'HPC')
     assert 'nek5000' in output
     assert 'mfem' in output
 

share/spack/spack-completion.bash

@@ -882,7 +882,7 @@ _spack_external() {
 _spack_external_find() {
     if $list_options
     then
-        SPACK_COMPREPLY="-h --help --not-buildable --scope"
+        SPACK_COMPREPLY="-h --help --not-buildable --scope -t --tag"
     else
         _all_packages
     fi
@@ -904,7 +904,7 @@ _spack_fetch() {
 _spack_find() {
     if $list_options
     then
-        SPACK_COMPREPLY="-h --help --format --json -d --deps -p --paths --groups --no-groups -l --long -L --very-long -t --tags -c --show-concretized -f --show-flags --show-full-compiler -x --explicit -X --implicit -u --unknown -m --missing -v --variants --loaded -M --only-missing --deprecated --only-deprecated -N --namespace --start-date --end-date"
+        SPACK_COMPREPLY="-h --help --format --json -d --deps -p --paths --groups --no-groups -l --long -L --very-long -t --tag -c --show-concretized -f --show-flags --show-full-compiler -x --explicit -X --implicit -u --unknown -m --missing -v --variants --loaded -M --only-missing --deprecated --only-deprecated -N --namespace --start-date --end-date"
     else
         _installed_packages
     fi
@@ -1063,7 +1063,7 @@ _spack_license_update_copyright_year() {
 _spack_list() {
     if $list_options
     then
-        SPACK_COMPREPLY="-h --help -d --search-description --format --update -v --virtuals -t --tags"
+        SPACK_COMPREPLY="-h --help -d --search-description --format --update -v --virtuals -t --tag"
     else
         _all_packages
     fi

var/spack/repos/builtin/packages/autoconf/package.py

@@ -31,6 +31,8 @@ class Autoconf(AutotoolsPackage, GNUMirrorPackage):
 
     build_directory = 'spack-build'
 
+    tags = ['build-tools']
+
     executables = [
         '^autoconf$', '^autoheader$', '^autom4te$', '^autoreconf$',
         '^autoscan$', '^autoupdate$', '^ifnames$'

var/spack/repos/builtin/packages/automake/package.py

@@ -25,6 +25,8 @@ class Automake(AutotoolsPackage, GNUMirrorPackage):
 
     build_directory = 'spack-build'
 
+    tags = ['build-tools']
+
     executables = ['^automake$']
 
     @classmethod

var/spack/repos/builtin/packages/cmake/package.py

@@ -14,6 +14,8 @@ class Cmake(Package):
     url = 'https://github.com/Kitware/CMake/releases/download/v3.19.0/cmake-3.19.0.tar.gz'
     maintainers = ['chuckatkins']
 
+    tags = ['build-tools']
+
     executables = ['^cmake$']
 
     version('3.19.2',   sha256='e3e0fd3b23b7fb13e1a856581078e0776ffa2df4e9d3164039c36d3315e0c7f0')

var/spack/repos/builtin/packages/flex/package.py

@@ -14,6 +14,8 @@ class Flex(AutotoolsPackage):
     homepage = "https://github.com/westes/flex"
     url = "https://github.com/westes/flex/releases/download/v2.6.1/flex-2.6.1.tar.gz"
 
+    tags = ['build-tools']
+
     executables = ['^flex$']
 
     version('2.6.4', sha256='e87aae032bf07c26f85ac0ed3250998c37621d95f8bd748b31f15b33c45ee995')

var/spack/repos/builtin/packages/gmake/package.py

@@ -29,6 +29,8 @@ class Gmake(AutotoolsPackage, GNUMirrorPackage):
     patch('https://src.fedoraproject.org/rpms/make/raw/519a7c5bcbead22e6ea2d2c2341d981ef9e25c0d/f/make-4.2.1-glob-fix-2.patch', level=1, sha256='fe5b60d091c33f169740df8cb718bf4259f84528b42435194ffe0dd5b79cd125', when='@4.2.1')
     patch('https://src.fedoraproject.org/rpms/make/raw/519a7c5bcbead22e6ea2d2c2341d981ef9e25c0d/f/make-4.2.1-glob-fix-3.patch', level=1, sha256='ca60bd9c1a1b35bc0dc58b6a4a19d5c2651f7a94a4b22b2c5ea001a1ca7a8a7f', when='@:4.2.1')
 
+    tags = ['build-tools']
+
     executables = ['^g?make$']
 
     @classmethod

var/spack/repos/builtin/packages/libtool/package.py

@@ -30,6 +30,8 @@ class Libtool(AutotoolsPackage, GNUMirrorPackage):
 
     build_directory = 'spack-build'
 
+    tags = ['build-tools']
+
     executables = ['^g?libtool(ize)?$']
 
     @classmethod

var/spack/repos/builtin/packages/m4/package.py

@@ -33,6 +33,8 @@ class M4(AutotoolsPackage, GNUMirrorPackage):
 
     build_directory = 'spack-build'
 
+    tags = ['build-tools']
+
     executables = ['^g?m4$']
 
     @classmethod

var/spack/repos/builtin/packages/pkg-config/package.py

@@ -31,6 +31,8 @@ class PkgConfig(AutotoolsPackage):
 
     parallel = False
 
+    tags = ['build-tools']
+
     executables = ['^pkg-config$']
 
     @classmethod

var/spack/repos/builtin/packages/pkgconf/package.py

@@ -37,6 +37,8 @@ class Pkgconf(AutotoolsPackage):
     # https://github.com/spack/spack/issues/3525
     conflicts('%pgi')
 
+    tags = ['build-tools']
+
     executables = ['^pkgconf$', '^pkg-config$']
 
     @classmethod