Don't add .spack/binary_distribution twice to the tarball when re-distributing (#39042)

Previously, spack would list the ./spack/binary_distribution file twice when pushing
a package that was installed from a binary tarball itself.

lib/spack/spack/binary_distribution.py

@@ -1208,9 +1208,17 @@ def tar_add_metadata(tar: tarfile.TarFile, path: str, data: dict):
     tar.addfile(deterministic_tarinfo(tarinfo), io.BytesIO(bstring))
 
 
-def _do_create_tarball(tarfile_path, binaries_dir, pkg_dir, buildinfo):
+def deterministic_tarinfo_without_buildinfo(tarinfo: tarfile.TarInfo):
+    """Skip buildinfo file when creating a tarball, and normalize other tarinfo fields."""
+    if tarinfo.name.endswith("/.spack/binary_distribution"):
+        return None
+
+    return deterministic_tarinfo(tarinfo)
+
+
+def _do_create_tarball(tarfile_path: str, binaries_dir: str, pkg_dir: str, buildinfo: dict):
     with gzip_compressed_tarfile(tarfile_path) as tar:
-        tar.add(name=binaries_dir, arcname=pkg_dir, filter=deterministic_tarinfo)
+        tar.add(name=binaries_dir, arcname=pkg_dir, filter=deterministic_tarinfo_without_buildinfo)
         tar_add_metadata(tar, buildinfo_file_name(pkg_dir), buildinfo)
 
 

lib/spack/spack/test/bindist.py

@@ -28,6 +28,7 @@
 import spack.repo
 import spack.store
 import spack.util.gpg
+import spack.util.spack_yaml as syaml
 import spack.util.url as url_util
 import spack.util.web as web_util
 from spack.binary_distribution import get_buildfile_manifest
@@ -856,6 +857,39 @@ def urlopen(request: urllib.request.Request):
         fetcher.conditional_fetch()
 
 
+def test_tarball_doesnt_include_buildinfo_twice(tmpdir):
+    """When tarballing a package that was installed from a buildcache, make
+    sure that the buildinfo file is not included twice in the tarball."""
+    p = tmpdir.mkdir("prefix")
+    p.mkdir(".spack")
+
+    # Create a binary_distribution file in the .spack folder
+    with open(p.join(".spack", "binary_distribution"), "w") as f:
+        f.write(syaml.dump({"metadata", "old"}))
+
+    # Now create a tarball, which should include a new binary_distribution file
+    tarball = str(tmpdir.join("prefix.tar.gz"))
+
+    bindist._do_create_tarball(
+        tarfile_path=tarball,
+        binaries_dir=str(p),
+        pkg_dir="my-pkg-prefix",
+        buildinfo={"metadata": "new"},
+    )
+
+    # Verify we don't have a repeated binary_distribution file,
+    # and that the tarball contains the new one, not the old one.
+    with tarfile.open(tarball) as tar:
+        assert syaml.load(tar.extractfile("my-pkg-prefix/.spack/binary_distribution")) == {
+            "metadata": "new"
+        }
+        assert tar.getnames() == [
+            "my-pkg-prefix",
+            "my-pkg-prefix/.spack",
+            "my-pkg-prefix/.spack/binary_distribution",
+        ]
+
+
 def test_reproducible_tarball_is_reproducible(tmpdir):
     p = tmpdir.mkdir("prefix")
     p.mkdir("bin")