Add Changelog for v0.17.3, bump version number

Fixes missing chgrp on symlinks in package installations, and errors on
symlinks referencing non-existent or non-writable locations.

Note: `os.chown(.., follow_symlinks=False)` is python3 only, but
`os.lchown` exists in both versions.

.github/workflows/bootstrap.yml

@@ -29,14 +29,20 @@ jobs:
               bzip2 curl file gcc-c++ gcc gcc-gfortran git gnupg2 gzip \
               make patch unzip which xz python3 python3-devel tree \
               cmake bison bison-devel libstdc++-static
-      - uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579 # @v2
-      - name: Setup repo and non-root user
+      - name: Checkout
+        uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b
+      - name: Setup non-root user
+        run: |
+          # See [1] below
+          git config --global --add safe.directory /__w/spack/spack
+          useradd spack-test && mkdir -p ~spack-test
+          chown -R spack-test . ~spack-test
+      - name: Setup repo
+        shell: runuser -u spack-test -- bash {0}
         run: |
           git --version
           git fetch --unshallow
           . .github/workflows/setup_git.sh
-          useradd spack-test
-          chown -R spack-test .
       - name: Bootstrap clingo
         shell: runuser -u spack-test -- bash {0}
         run: |
@@ -59,22 +65,20 @@ jobs:
               bzip2 curl file g++ gcc gfortran git gnupg2 gzip \
               make patch unzip xz-utils python3 python3-dev tree \
               cmake bison
-      - name: Work around CVE-2022-24765
+      - name: Checkout
+        uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b
+      - name: Setup non-root user
         run: |
-          # Apparently Ubuntu patched git v2.25.1 with a security patch that introduces 
-          # a breaking behavior. See:
-          # - https://github.blog/2022-04-12-git-security-vulnerability-announced/
-          # - https://github.com/actions/checkout/issues/760
-          # - http://changelogs.ubuntu.com/changelogs/pool/main/g/git/git_2.25.1-1ubuntu3.3/changelog
+          # See [1] below
           git config --global --add safe.directory /__w/spack/spack
-      - uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579 # @v2
-      - name: Setup repo and non-root user
+          useradd spack-test && mkdir -p ~spack-test
+          chown -R spack-test . ~spack-test
+      - name: Setup repo
+        shell: runuser -u spack-test -- bash {0}
         run: |
           git --version
           git fetch --unshallow
           . .github/workflows/setup_git.sh
-          useradd -m spack-test
-          chown -R spack-test .
       - name: Bootstrap clingo
         shell: runuser -u spack-test -- bash {0}
         run: |
@@ -95,9 +99,12 @@ jobs:
               bzip2 curl file gcc-c++ gcc gcc-fortran tar git gpg2 gzip \
               make patch unzip which xz python3 python3-devel tree \
               cmake bison
-      - uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579 # @v2
-      - name: Setup repo and non-root user
+      - name: Checkout
+        uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b
+      - name: Setup repo
         run: |
+          # See [1] below
+          git config --global --add safe.directory /__w/spack/spack
           git --version
           git fetch --unshallow
           . .github/workflows/setup_git.sh
@@ -115,7 +122,8 @@ jobs:
       - name: Install dependencies
         run: |
           brew install cmake bison@2.7 tree
-      - uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579 # @v2
+      - name: Checkout
+        uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b
       - name: Bootstrap clingo
         run: |
           source share/spack/setup-env.sh
@@ -134,8 +142,9 @@ jobs:
       - name: Install dependencies
         run: |
           brew install tree
-      - uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579 # @v2
-      - uses: actions/setup-python@dc73133d4da04e56a135ae2246682783cc7c7cb6 # @v2
+      - name: Checkout
+        uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b
+      - uses: actions/setup-python@98f2ad02fd48d057ee3b4d4f66525b231c3e52b6
         with:
           python-version: ${{ matrix.python-version }}
       - name: Bootstrap clingo
@@ -151,11 +160,12 @@ jobs:
       matrix:
         python-version: ['2.7', '3.5', '3.6', '3.7', '3.8', '3.9']
     steps:
-      - uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579 # @v2
-      - uses: actions/setup-python@dc73133d4da04e56a135ae2246682783cc7c7cb6 # @v2
+      - name: Checkout
+        uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b
+      - uses: actions/setup-python@98f2ad02fd48d057ee3b4d4f66525b231c3e52b6
         with:
           python-version: ${{ matrix.python-version }}
-      - name: Setup repo and non-root user
+      - name: Setup repo
         run: |
           git --version
           git fetch --unshallow
@@ -179,22 +189,20 @@ jobs:
           apt-get install -y \
               bzip2 curl file g++ gcc patchelf gfortran git gzip \
               make patch unzip xz-utils python3 python3-dev tree
-      - name: Work around CVE-2022-24765
+      - name: Checkout
+        uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b
+      - name: Setup non-root user
         run: |
-          # Apparently Ubuntu patched git v2.25.1 with a security patch that introduces 
-          # a breaking behavior. See:
-          # - https://github.blog/2022-04-12-git-security-vulnerability-announced/
-          # - https://github.com/actions/checkout/issues/760
-          # - http://changelogs.ubuntu.com/changelogs/pool/main/g/git/git_2.25.1-1ubuntu3.3/changelog
+          # See [1] below
           git config --global --add safe.directory /__w/spack/spack
-      - uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579
-      - name: Setup repo and non-root user
+          useradd spack-test && mkdir -p ~spack-test
+          chown -R spack-test . ~spack-test
+      - name: Setup repo
+        shell: runuser -u spack-test -- bash {0}
         run: |
           git --version
           git fetch --unshallow
           . .github/workflows/setup_git.sh
-          useradd -m spack-test
-          chown -R spack-test .
       - name: Bootstrap GnuPG
         shell: runuser -u spack-test -- bash {0}
         run: |
@@ -216,22 +224,20 @@ jobs:
               bzip2 curl file g++ gcc patchelf gfortran git gzip \
               make patch unzip xz-utils python3 python3-dev tree \
               gawk
-      - name: Work around CVE-2022-24765
+      - name: Checkout
+        uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b
+      - name: Setup non-root user
         run: |
-          # Apparently Ubuntu patched git v2.25.1 with a security patch that introduces 
-          # a breaking behavior. See:
-          # - https://github.blog/2022-04-12-git-security-vulnerability-announced/
-          # - https://github.com/actions/checkout/issues/760
-          # - http://changelogs.ubuntu.com/changelogs/pool/main/g/git/git_2.25.1-1ubuntu3.3/changelog
+          # See [1] below
           git config --global --add safe.directory /__w/spack/spack
-      - uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579
-      - name: Setup repo and non-root user
+          useradd spack-test && mkdir -p ~spack-test
+          chown -R spack-test . ~spack-test
+      - name: Setup repo
+        shell: runuser -u spack-test -- bash {0}
         run: |
           git --version
           git fetch --unshallow
           . .github/workflows/setup_git.sh
-          useradd -m spack-test
-          chown -R spack-test .
       - name: Bootstrap GnuPG
         shell: runuser -u spack-test -- bash {0}
         run: |
@@ -249,7 +255,8 @@ jobs:
           brew install tree
           # Remove GnuPG since we want to bootstrap it
           sudo rm -rf /usr/local/bin/gpg
-      - uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579
+      - name: Checkout
+        uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b
       - name: Bootstrap GnuPG
         run: |
           source share/spack/setup-env.sh
@@ -265,7 +272,8 @@ jobs:
           brew install gawk tree
           # Remove GnuPG since we want to bootstrap it
           sudo rm -rf /usr/local/bin/gpg
-      - uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579
+      - name: Checkout
+        uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b
       - name: Bootstrap GnuPG
         run: |
           source share/spack/setup-env.sh
@@ -273,3 +281,11 @@ jobs:
           spack bootstrap untrust github-actions
           spack -d gpg list
           tree ~/.spack/bootstrap/store/
+
+
+# [1] Distros that have patched git to resolve CVE-2022-24765 (e.g. Ubuntu patching v2.25.1)
+#     introduce breaking behaviorso we have to set `safe.directory` in gitconfig ourselves.
+#     See:
+#     - https://github.blog/2022-04-12-git-security-vulnerability-announced/
+#     - https://github.com/actions/checkout/issues/760
+#     - http://changelogs.ubuntu.com/changelogs/pool/main/g/git/git_2.25.1-1ubuntu3.3/changelog

CHANGELOG.md

@@ -1,3 +1,13 @@
+# v0.17.3 (2022-07-14)
+
+### Spack bugfixes
+
+* Fix missing chgrp on symlinks in package installations (#30743)
+* Allow having non-existing upstreams (#30744, #30746)
+* Fix `spack stage` with custom paths (#30448)
+* Fix failing call for `spack buildcache save-specfile` (#30637)
+* Fix globbing in compiler wrapper (#30699)
+
 # v0.17.2 (2022-04-13)
 
 ### Spack bugfixes

lib/spack/env/cc

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/sh -f
 # shellcheck disable=SC2034  # evals in this script fool shellcheck
 #
 # Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
@@ -768,7 +768,9 @@ if [ "$SPACK_DEBUG" = TRUE ]; then
     input_log="$SPACK_DEBUG_LOG_DIR/spack-cc-$SPACK_DEBUG_LOG_ID.in.log"
     output_log="$SPACK_DEBUG_LOG_DIR/spack-cc-$SPACK_DEBUG_LOG_ID.out.log"
     echo "[$mode] $command $input_command" >> "$input_log"
-    echo "[$mode] ${full_command_list}" >> "$output_log"
+    IFS="$lsep"
+    echo "[$mode] "$full_command_list >> "$output_log"
+    unset IFS
 fi
 
 # Execute the full command, preserving spaces with IFS set

lib/spack/llnl/util/filesystem.py

@@ -302,13 +302,16 @@ def group_ids(uid=None):
     return [g.gr_gid for g in grp.getgrall() if user in g.gr_mem]
 
 
-def chgrp(path, group):
+def chgrp(path, group, follow_symlinks=True):
     """Implement the bash chgrp function on a single path"""
     if isinstance(group, six.string_types):
         gid = grp.getgrnam(group).gr_gid
     else:
         gid = group
-    os.chown(path, -1, gid)
+    if follow_symlinks:
+        os.chown(path, -1, gid)
+    else:
+        os.lchown(path, -1, gid)
 
 
 def chmod_x(entry, perms):

lib/spack/spack/__init__.py

@@ -4,7 +4,7 @@
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
 #: major, minor, patch version for Spack, in a tuple
-spack_version_info = (0, 17, 2)
+spack_version_info = (0, 17, 3)
 
 #: String containing Spack version joined with .'s
 spack_version = '.'.join(str(v) for v in spack_version_info)

lib/spack/spack/cmd/buildcache.py

@@ -754,11 +754,12 @@ def save_specfiles(args):
     if args.root_specfile:
         with open(args.root_specfile) as fd:
             root_spec_as_json = fd.read()
+        spec_format = 'yaml' if args.root_specfile.endswith('yaml') else 'json'
     else:
         root_spec = Spec(args.root_spec)
         root_spec.concretize()
         root_spec_as_json = root_spec.to_json(hash=ht.build_hash)
-    spec_format = 'yaml' if args.root_specfile.endswith('yaml') else 'json'
+        spec_format = 'json'
     save_dependency_specfiles(
         root_spec_as_json, args.specfile_dir, args.specs.split(), spec_format)
 

lib/spack/spack/cmd/stage.py

@@ -27,12 +27,6 @@ def setup_parser(subparser):
 
 
 def stage(parser, args):
-    # We temporarily modify the working directory when setting up a stage, so we need to
-    # convert this to an absolute path here in order for it to remain valid later.
-    custom_path = os.path.abspath(args.path) if args.path else None
-    if custom_path:
-        spack.stage.create_stage_root(custom_path)
-
     if not args.specs:
         env = ev.active_environment()
         if env:
@@ -54,6 +48,10 @@ def stage(parser, args):
 
     specs = spack.cmd.parse_specs(args.specs, concretize=False)
 
+    # We temporarily modify the working directory when setting up a stage, so we need to
+    # convert this to an absolute path here in order for it to remain valid later.
+    custom_path = os.path.abspath(args.path) if args.path else None
+
     # prevent multiple specs from extracting in the same folder
     if len(specs) > 1 and custom_path:
         tty.die("`--path` requires a single spec, but multiple were provided")

lib/spack/spack/database.py

@@ -355,10 +355,10 @@ def __init__(self, root, db_dir=None, upstream_dbs=None,
         self.prefix_fail_path = os.path.join(self._db_dir, 'prefix_failures')
 
         # Create needed directories and files
-        if not os.path.exists(self._db_dir):
+        if not is_upstream and not os.path.exists(self._db_dir):
             fs.mkdirp(self._db_dir)
 
-        if not os.path.exists(self._failure_dir) and not is_upstream:
+        if not is_upstream and not os.path.exists(self._failure_dir):
             fs.mkdirp(self._failure_dir)
 
         self.is_upstream = is_upstream
@@ -1033,9 +1033,7 @@ def _read(self):
                 self._read_from_file(self._index_path)
             return
         elif self.is_upstream:
-            raise UpstreamDatabaseLockingError(
-                "No database index file is present, and upstream"
-                " databases cannot generate an index file")
+            tty.warn('upstream not found: {0}'.format(self._index_path))
 
     def _add(
             self,

lib/spack/spack/test/cmd/stage.py

@@ -39,7 +39,6 @@ def check_stage_path(monkeypatch, tmpdir):
 
     def fake_stage(pkg, mirror_only=False):
         assert pkg.path == expected_path
-        assert os.path.isdir(expected_path), expected_path
 
     monkeypatch.setattr(spack.package.PackageBase, 'do_stage', fake_stage)
 

lib/spack/spack/test/installer.py

@@ -834,7 +834,7 @@ def test_setup_install_dir_grp(install_mockery, monkeypatch, capfd):
     def _get_group(spec):
         return mock_group
 
-    def _chgrp(path, group):
+    def _chgrp(path, group, follow_symlinks=True):
         tty.msg(mock_chgrp_msg.format(path, group))
 
     monkeypatch.setattr(prefs, 'get_package_group', _get_group)

lib/spack/spack/util/file_permissions.py

@@ -44,7 +44,7 @@ def set_permissions(path, perms, group=None):
     fs.chmod_x(path, perms)
 
     if group:
-        fs.chgrp(path, group)
+        fs.chgrp(path, group, follow_symlinks=False)
 
 
 class InvalidPermissionsError(SpackError):