From f32b5e572a299d48eb102cd374a63a2f231430a3 Mon Sep 17 00:00:00 2001 From: Ryan Krattiger <80296582+kwryankrattiger@users.noreply.github.com> Date: Sat, 15 Mar 2025 03:41:25 -0500 Subject: [PATCH] ci: remove --keep-stage flag (#49467) logs are now copied from the install dir --- lib/spack/spack/ci/__init__.py | 34 +++++++++++-------- lib/spack/spack/cmd/ci.py | 5 ++- lib/spack/spack/test/cmd/ci.py | 2 +- .../configs/darwin/aarch64/config.yaml | 2 -- .../cloud_pipelines/configs/darwin/ci.yaml | 8 ----- 5 files changed, 22 insertions(+), 29 deletions(-) diff --git a/lib/spack/spack/ci/__init__.py b/lib/spack/spack/ci/__init__.py index 0f33ba7f8ad..c5726590972 100644 --- a/lib/spack/spack/ci/__init__.py +++ b/lib/spack/spack/ci/__init__.py @@ -6,6 +6,7 @@ import codecs import json import os +import pathlib import re import shutil import stat @@ -23,7 +24,6 @@ import spack import spack.binary_distribution as bindist -import spack.builder import spack.concretize import spack.config as cfg import spack.environment as ev @@ -33,6 +33,7 @@ import spack.paths import spack.repo import spack.spec +import spack.store import spack.util.git import spack.util.gpg as gpg_util import spack.util.spack_yaml as syaml @@ -581,22 +582,25 @@ def copy_stage_logs_to_artifacts(job_spec: spack.spec.Spec, job_log_dir: str) -> tty.debug(f"job spec: {job_spec}") try: - pkg_cls = spack.repo.PATH.get_pkg_class(job_spec.name) - job_pkg = pkg_cls(job_spec) - tty.debug(f"job package: {job_pkg}") - except AssertionError: - msg = f"Cannot copy stage logs: job spec ({job_spec}) must be concrete" - tty.error(msg) + package_metadata_root = pathlib.Path(spack.store.STORE.layout.metadata_path(job_spec)) + except spack.error.SpackError as e: + tty.error(f"Cannot copy logs: {str(e)}") return - stage_dir = job_pkg.stage.path - tty.debug(f"stage dir: {stage_dir}") - for file in [ - job_pkg.log_path, - job_pkg.env_mods_path, - *spack.builder.create(job_pkg).archive_files, - ]: - copy_files_to_artifacts(file, job_log_dir) + # Get the package's archived files + archive_files = [] + archive_root = package_metadata_root / "archived-files" + if archive_root.is_dir(): + archive_files = [f for f in archive_root.rglob("*") if f.is_file()] + else: + msg = "Cannot copy package archived files: archived-files must be a directory" + tty.warn(msg) + + build_log_zipped = package_metadata_root / "spack-build-out.txt.gz" + build_env_mods = package_metadata_root / "spack-build-env.txt" + + for f in [build_log_zipped, build_env_mods, *archive_files]: + copy_files_to_artifacts(str(f), job_log_dir) def copy_test_logs_to_artifacts(test_stage, job_test_dir): diff --git a/lib/spack/spack/cmd/ci.py b/lib/spack/spack/cmd/ci.py index d0d04b95ee6..1cd4a1127d3 100644 --- a/lib/spack/spack/cmd/ci.py +++ b/lib/spack/spack/cmd/ci.py @@ -427,7 +427,7 @@ def ci_rebuild(args): # Arguments when installing the root from sources deps_install_args = install_args + ["--only=dependencies"] - root_install_args = install_args + ["--keep-stage", "--only=package"] + root_install_args = install_args + ["--only=package"] if cdash_handler: # Add additional arguments to `spack install` for CDash reporting. @@ -464,8 +464,7 @@ def ci_rebuild(args): job_spec.to_dict(hash=ht.dag_hash), ) - # We generated the "spack install ..." command to "--keep-stage", copy - # any logs from the staging directory to artifacts now + # Copy logs and archived files from the install metadata (.spack) directory to artifacts now spack_ci.copy_stage_logs_to_artifacts(job_spec, job_log_dir) # If the installation succeeded and we're running stand-alone tests for diff --git a/lib/spack/spack/test/cmd/ci.py b/lib/spack/spack/test/cmd/ci.py index e2058efc4ed..8afe70dc981 100644 --- a/lib/spack/spack/test/cmd/ci.py +++ b/lib/spack/spack/test/cmd/ci.py @@ -867,7 +867,7 @@ def test_push_to_build_cache( logs_dir = scratch / "logs_dir" logs_dir.mkdir() ci.copy_stage_logs_to_artifacts(concrete_spec, str(logs_dir)) - assert "spack-build-out.txt" in os.listdir(logs_dir) + assert "spack-build-out.txt.gz" in os.listdir(logs_dir) dl_dir = scratch / "download_dir" buildcache_cmd("download", "--spec-file", json_path, "--path", str(dl_dir)) diff --git a/share/spack/gitlab/cloud_pipelines/configs/darwin/aarch64/config.yaml b/share/spack/gitlab/cloud_pipelines/configs/darwin/aarch64/config.yaml index cbf84c3baab..fcbe195a4f4 100644 --- a/share/spack/gitlab/cloud_pipelines/configs/darwin/aarch64/config.yaml +++ b/share/spack/gitlab/cloud_pipelines/configs/darwin/aarch64/config.yaml @@ -1,5 +1,3 @@ config: - build_stage: - - $spack/tmp/stage install_tree: root: $spack/opt/spack diff --git a/share/spack/gitlab/cloud_pipelines/configs/darwin/ci.yaml b/share/spack/gitlab/cloud_pipelines/configs/darwin/ci.yaml index 0340fa7694e..7e2296fb4af 100644 --- a/share/spack/gitlab/cloud_pipelines/configs/darwin/ci.yaml +++ b/share/spack/gitlab/cloud_pipelines/configs/darwin/ci.yaml @@ -8,13 +8,5 @@ ci: tags: [spack] - build-job: tags: [ "macos-sequoia", "apple-clang-16", "aarch64-macos" ] - - # after_script intended to ensure all stage files are properly cleaned up, - # including those that may have been created as read-only by `go mod` - # as part of installation of a golang package - # see: https://github.com/spack/spack/issues/49147 - after_script-: - - - if [[ -d tmp ]] ; then chmod -R u+w tmp ; else echo tmp not found ; fi - - ./bin/spack clean -a - build-job-remove: image:: macos-run-on-metal