spack external find: change default behavior (#29031)

See https://github.com/spack/spack/issues/25353#issuecomment-1041868116

This commit changes the default behavior of
```
$ spack external find
```
from searching all the possible packages Spack knows about to
search only for the ones tagged as being a "build-tool".

It also introduces a `--all` option to restore the old behavior.

lib/spack/spack/cmd/external.py

@@ -39,8 +39,17 @@ def setup_parser(subparser):
         '--scope', choices=scopes, metavar=scopes_metavar,
         default=spack.config.default_modify_scope('packages'),
         help="configuration scope to modify")
+    find_parser.add_argument(
+        '--all', action='store_true',
+        help="search for all packages that Spack knows about"
+    )
     spack.cmd.common.arguments.add_common_arguments(find_parser, ['tags'])
     find_parser.add_argument('packages', nargs=argparse.REMAINDER)
+    find_parser.epilog = (
+        'The search is by default on packages tagged with the "build-tools" or '
+        '"core-packages" tags. Use the --all option to search for every possible '
+        'package Spack knows how to find.'
+    )
 
     sp.add_parser(
         'list', help='list detectable packages, by repository and name'
@@ -48,6 +57,14 @@ def setup_parser(subparser):
 
 
 def external_find(args):
+    # If the user didn't specify anything, search for build tools by default
+    if not args.tags and not args.all and not args.packages:
+        args.tags = ['core-packages', 'build-tools']
+
+    # If the user specified both --all and --tag, then --all has precedence
+    if args.all and args.tags:
+        args.tags = []
+
     # Construct the list of possible packages to be detected
     packages_to_check = []
 
@@ -64,9 +81,10 @@ def external_find(args):
         # Since tags are cached it's much faster to construct what we need
         # to search directly, rather than filtering after the fact
         packages_to_check = [
-            spack.repo.get(pkg) for pkg in
+            spack.repo.get(pkg) for tag in args.tags for pkg in
-            spack.repo.path.packages_with_tags(*args.tags)
+            spack.repo.path.packages_with_tags(tag)
         ]
+        packages_to_check = list(set(packages_to_check))
 
     # If the list of packages is empty, search for every possible package
     if not args.tags and not packages_to_check:

lib/spack/spack/test/cmd/external.py

@@ -124,7 +124,7 @@ def test_find_external_cmd_not_buildable(
 
 def test_find_external_cmd_full_repo(
         mutable_config, working_env, mock_executable, mutable_mock_repo):
-    """Test invoking 'spack external find' with no additional arguments, which
+    """Test invoking 'spack external find --all' with no additional arguments
     iterates through each package in the repository.
     """
 
@@ -134,7 +134,7 @@ def test_find_external_cmd_full_repo(
     prefix = os.path.dirname(os.path.dirname(exe_path1))
 
     os.environ['PATH'] = ':'.join([os.path.dirname(exe_path1)])
-    external('find')
+    external('find', '--all')
 
     pkgs_cfg = spack.config.get('packages')
     pkg_cfg = pkgs_cfg['find-externals1']

share/spack/spack-completion.bash

@@ -1024,7 +1024,7 @@ _spack_external() {
 _spack_external_find() {
     if $list_options
     then
-        SPACK_COMPREPLY="-h --help --not-buildable --scope -t --tag"
+        SPACK_COMPREPLY="-h --help --not-buildable --scope --all -t --tag"
     else
         _all_packages
     fi

var/spack/repos/builtin/packages/bazel/package.py

@@ -17,6 +17,9 @@ class Bazel(Package):
     url      = "https://github.com/bazelbuild/bazel/releases/download/3.1.0/bazel-3.1.0-dist.zip"
 
     maintainers = ['adamjstewart']
+
+    tags = ['build-tools']
+
     version('4.0.0',  sha256='d350f80e70654932db252db380d2ec0144a00e86f8d9f2b4c799ffdb48e9cdd1')
     version('3.7.2',  sha256='de255bb42163a915312df9f4b86e5b874b46d9e8d4b72604b5123c3a845ed9b1')
     version('3.7.1',  sha256='c9244e5905df6b0190113e26082c72d58b56b1b0dec66d076f083ce4089b0307')

var/spack/repos/builtin/packages/binutils/package.py

@@ -13,6 +13,8 @@ class Binutils(AutotoolsPackage, GNUMirrorPackage):
 
     maintainers = ['alalazo']
 
+    tags = ['build-tools', 'core-packages']
+
     executables = ['^nm$', '^readelf$']
 
     version('2.37', sha256='67fc1a4030d08ee877a4867d3dcab35828148f87e1fd05da6db585ed5a166bd4')

var/spack/repos/builtin/packages/bison/package.py

@@ -18,6 +18,8 @@ class Bison(AutotoolsPackage, GNUMirrorPackage):
     homepage = "https://www.gnu.org/software/bison/"
     gnu_mirror_path = "bison/bison-3.6.4.tar.gz"
 
+    tags = ['build-tools']
+
     executables = ['^bison$']
 
     version('3.8.2', sha256='06c9e13bdf7eb24d4ceb6b59205a4f67c2c7e7213119644430fe82fbd14a0abb')

var/spack/repos/builtin/packages/ccache/package.py

@@ -17,6 +17,8 @@ class Ccache(CMakePackage):
     url      = "https://github.com/ccache/ccache/releases/download/v4.2.1/ccache-4.2.1.tar.gz"
     maintainers = ['haampie']
 
+    tags = ['build-tools']
+
     executables = ['^ccache$']
 
     version('4.5.1', sha256='f0d3cff5d555d6868f14a7d05696f0370074e475304fd5aa152b98f892364981')

var/spack/repos/builtin/packages/coreutils/package.py

@@ -16,6 +16,8 @@ class Coreutils(AutotoolsPackage, GNUMirrorPackage):
     homepage = 'https://www.gnu.org/software/coreutils/'
     gnu_mirror_path = 'coreutils/coreutils-8.26.tar.xz'
 
+    tags = ['core-packages']
+
     version('8.32', sha256='4458d8de7849df44ccab15e16b1548b285224dbba5f08fac070c1c0e0bcc4cfa')
     version('8.31', sha256='ff7a9c918edce6b4f4b2725e3f9b37b0c4d193531cac49a48b56c4d0d3a9e9fd')
     version('8.30', sha256='e831b3a86091496cdba720411f9748de81507798f6130adeaef872d206e1b057')

var/spack/repos/builtin/packages/cvs/package.py

@@ -20,6 +20,8 @@ class Cvs(AutotoolsPackage, GNUMirrorPackage):
     patch('https://gentoofan.org/gentoo/poly-c_overlay/dev-vcs/cvs/files/cvs-1.12.13.1-fix-gnulib-SEGV-vasnprintf.patch',
           sha256='e13db2acebad3ca5be5d8e0fa97f149b0f9661e4a9a731965c8226290c6413c0', when='@1.12.13')
 
+    tags = ['build-tools']
+
     parallel = False
     executables = [r'^cvs$']
 

var/spack/repos/builtin/packages/diffutils/package.py

@@ -12,6 +12,8 @@ class Diffutils(AutotoolsPackage, GNUMirrorPackage):
     """GNU Diffutils is a package of several programs related to finding
     differences between files."""
 
+    tags = ['core-packages']
+
     executables = [r'^diff$']
 
     homepage = "https://www.gnu.org/software/diffutils/"

var/spack/repos/builtin/packages/findutils/package.py

@@ -12,6 +12,8 @@ class Findutils(AutotoolsPackage, GNUMirrorPackage):
     """The GNU Find Utilities are the basic directory searching
        utilities of the GNU operating system."""
 
+    tags = ['core-packages']
+
     homepage = "https://www.gnu.org/software/findutils/"
     gnu_mirror_path = "findutils/findutils-4.8.0.tar.xz"
 

var/spack/repos/builtin/packages/gawk/package.py

@@ -24,7 +24,7 @@ class Gawk(AutotoolsPackage, GNUMirrorPackage):
 
     executables = ['^gawk$']
 
-    tags = ['build-tools']
+    tags = ['build-tools', 'core-packages']
 
     version('5.1.1', sha256='d87629386e894bbea11a5e00515fc909dc9b7249529dad9e6a3a2c77085f7ea2')
     version('5.1.0', sha256='cf5fea4ac5665fd5171af4716baab2effc76306a9572988d5ba1078f196382bd')

var/spack/repos/builtin/packages/git/package.py

@@ -19,6 +19,8 @@ class Git(AutotoolsPackage):
     homepage = "http://git-scm.com"
     url      = "https://mirrors.edge.kernel.org/pub/software/scm/git/git-2.12.0.tar.gz"
 
+    tags = ['build-tools']
+
     executables = ['^git$']
 
     # In order to add new versions here, add a new list entry with:

var/spack/repos/builtin/packages/groff/package.py

@@ -17,6 +17,8 @@ class Groff(AutotoolsPackage, GNUMirrorPackage):
     homepage = "https://www.gnu.org/software/groff/"
     gnu_mirror_path = "groff/groff-1.22.3.tar.gz"
 
+    tags = ['build-tools']
+
     version('1.22.4', sha256='e78e7b4cb7dec310849004fa88847c44701e8d133b5d4c13057d876c1bad0293')
     version('1.22.3', sha256='3a48a9d6c97750bfbd535feeb5be0111db6406ddb7bb79fc680809cda6d828a5')
 

var/spack/repos/builtin/packages/meson/package.py

@@ -14,6 +14,8 @@ class Meson(PythonPackage):
     homepage = "https://mesonbuild.com/"
     url      = "https://github.com/mesonbuild/meson/archive/0.49.0.tar.gz"
 
+    tags = ['build-tools']
+
     maintainers = ['michaelkuhn']
 
     version('0.61.2', sha256='33cd555314a94d52acfbb3f6f44d4e61c4ad0bfec7acf4301be7e40bb969b3a8')

var/spack/repos/builtin/packages/ninja/package.py

@@ -14,9 +14,9 @@ class Ninja(Package):
     url      = "https://github.com/ninja-build/ninja/archive/v1.7.2.tar.gz"
     git      = "https://github.com/ninja-build/ninja.git"
 
-    executables = ['^ninja$']
+    tags = ['build-tools', 'e4s']
 
-    tags = ['e4s']
+    executables = ['^ninja$']
 
     version('kitware', branch='features-for-fortran', git='https://github.com/Kitware/ninja.git')
     version('master', branch='master')

var/spack/repos/builtin/packages/openssh/package.py

@@ -21,6 +21,8 @@ class Openssh(AutotoolsPackage):
     homepage = "https://www.openssh.com/"
     url      = "https://mirrors.sonic.net/pub/OpenBSD/OpenSSH/portable/openssh-8.7p1.tar.gz"
 
+    tags = ['core-packages']
+
     version('8.8p1', sha256='4590890ea9bb9ace4f71ae331785a3a5823232435161960ed5fc86588f331fe9')
     version('8.7p1', sha256='7ca34b8bb24ae9e50f33792b7091b3841d7e1b440ff57bc9fabddf01e2ed1e24')
     version('8.6p1', sha256='c3e6e4da1621762c850d03b47eed1e48dff4cc9608ddeb547202a234df8ed7ae')

var/spack/repos/builtin/packages/openssl/package.py

@@ -23,6 +23,8 @@ class Openssl(Package):   # Uses Fake Autotools, should subclass Package
     list_url = "https://www.openssl.org/source/old/"
     list_depth = 1
 
+    tags = ['core-packages']
+
     executables = ['openssl']
 
     version('3.0.1', sha256='c311ad853353bce796edad01a862c50a8a587f62e7e2100ef465ab53ec9b06d1')

var/spack/repos/builtin/packages/patch/package.py

@@ -15,6 +15,8 @@ class Patch(AutotoolsPackage, GNUMirrorPackage):
     homepage = "https://savannah.gnu.org/projects/patch/"
     gnu_mirror_path = "patch/patch-2.7.6.tar.xz"
 
+    tags = ['core-packages']
+
     version('2.7.6', sha256='ac610bda97abe0d9f6b7c963255a11dcb196c25e337c61f94e4778d632f1d8fd')
     version('2.7.5', sha256='fd95153655d6b95567e623843a0e77b81612d502ecf78a489a4aed7867caa299')
 

var/spack/repos/builtin/packages/scons/package.py

@@ -12,6 +12,8 @@ class Scons(PythonPackage):
     homepage = "https://scons.org"
     pypi = "scons/scons-3.1.1.tar.gz"
 
+    tags = ['build-tools']
+
     version('3.1.2', sha256='8aaa483c303efeb678e6f7c776c8444a482f8ddc3ad891f8b6cdd35264da9a1f')
     version('3.1.1', sha256='fd44f8f2a4562e7e5bc8c63c82b01e469e8115805a3e9c2923ee54cdcd6678b3')
     version('3.1.0', sha256='94e0d0684772d3e6d9368785296716e0ed6ce757270b3ed814e5aa72d3163890')

var/spack/repos/builtin/packages/subversion/package.py

@@ -17,6 +17,8 @@ class Subversion(AutotoolsPackage):
         'https://downloads.apache.org/subversion/subversion-1.13.0.tar.gz'
     ]
 
+    tags = ['build-tools']
+
     version('1.14.1', sha256='dee2796abaa1f5351e6cc2a60b1917beb8238af548b20d3e1ec22760ab2f0cad')
     version('1.14.0', sha256='ef3d1147535e41874c304fb5b9ea32745fbf5d7faecf2ce21d4115b567e937d0')
     version('1.13.0', sha256='daad440c03b8a86fcca804ea82217bb1902cfcae1b7d28c624143c58dcb96931')

var/spack/repos/builtin/packages/tar/package.py

@@ -17,6 +17,8 @@ class Tar(AutotoolsPackage, GNUMirrorPackage):
 
     executables = [r'^tar$']
 
+    tag = ['core-packages']
+
     version('1.34', sha256='03d908cf5768cfe6b7ad588c921c6ed21acabfb2b79b788d1330453507647aed')
     version('1.32', sha256='b59549594d91d84ee00c99cf2541a3330fed3a42c440503326dab767f2fbb96c')
     version('1.31', sha256='b471be6cb68fd13c4878297d856aebd50551646f4e3074906b1a74549c40d5a2')

var/spack/repos/builtin/packages/texinfo/package.py

@@ -21,6 +21,8 @@ class Texinfo(AutotoolsPackage, GNUMirrorPackage):
 
     executables = ['^info$']
 
+    tags = ['build-tools']
+
     version('6.5', sha256='d34272e4042c46186ddcd66bd5d980c0ca14ff734444686ccf8131f6ec8b1427')
     version('6.3', sha256='300a6ba4958c2dd4a6d5ce60f0a335daf7e379f5374f276f6ba31a221f02f606')
     version('6.0', sha256='83d3183290f34e7f958d209d0b20022c6fe9e921eb6fe94c27d988827d4878d2')