Ci reactivate darwin pipelines (#48453)

* ci: darwin stacks: update tags following system updates

* disable SPACK_CI_DISABLE_STACKS; only enable *darwin* stacks for testing

* manually chmod u+w tmp/ before cleanup due to issue#49147

* comment out failing specs for now

* re-enable logic for disabling stacks

* add explanatory comment for darwin after_script additions

* remove more darwin-only targetting

* restore build_stage to default location

* move build-job-remove out of individual darwin stacks into darwin top level config

* keep build_stage in $spack/tmp for now

share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml

@@ -426,7 +426,7 @@ developer-tools-aarch64-linux-gnu-build:
     SPACK_CI_STACK_NAME: developer-tools-darwin
 
 developer-tools-darwin-generate:
-  tags: [ "macos-sonoma", "apple-clang-16", "aarch64-macos" ]
+  tags: [ "macos-sequoia", "apple-clang-16", "aarch64-macos" ]
   extends: [ ".developer-tools-darwin", ".generate-base"]
 
 developer-tools-darwin-build:
@@ -686,7 +686,7 @@ ml-linux-aarch64-cuda-build:
     SPACK_CI_STACK_NAME: ml-darwin-aarch64-mps
 
 ml-darwin-aarch64-mps-generate:
-  tags: [ "macos-sonoma", "apple-clang-16", "aarch64-macos" ]
+  tags: [ "macos-sequoia", "apple-clang-16", "aarch64-macos" ]
   extends: [ ".ml-darwin-aarch64-mps", ".generate-base"]
 
 ml-darwin-aarch64-mps-build:
@@ -910,7 +910,7 @@ bootstrap-x86_64-linux-gnu-build:
     SPACK_CI_STACK_NAME: bootstrap-aarch64-darwin
 
 bootstrap-aarch64-darwin-generate:
-  tags: [ "macos-sonoma", "apple-clang-16", "aarch64-macos" ]
+  tags: [ "macos-sequoia", "apple-clang-16", "aarch64-macos" ]
   extends: [.bootstrap-aarch64-darwin, .generate-base]
 
 bootstrap-aarch64-darwin-build:

share/spack/gitlab/cloud_pipelines/configs/darwin/aarch64/config.yaml

@@ -1,3 +1,5 @@
 config:
+  build_stage:
+  - $spack/tmp/stage
   install_tree:
     root: $spack/opt/spack

share/spack/gitlab/cloud_pipelines/configs/darwin/ci.yaml

@@ -1,6 +1,19 @@
 ci:
+  broken-tests-packages:
+  - mpich
+  - openmpi
   pipeline-gen:
+  - build-job-remove:
+      tags: [spack]
   - build-job:
-      tags: [ "macos-sonoma", "apple-clang-16", "aarch64-macos" ]
+      tags: [ "macos-sequoia", "apple-clang-16", "aarch64-macos" ]
+
+      # after_script intended to ensure all stage files are properly cleaned up,
+      # including those that may have been created as read-only by `go mod`
+      # as part of installation of a golang package
+      # see: https://github.com/spack/spack/issues/49147
+      after_script-:
+      - - if [[ -d tmp ]] ; then chmod -R u+w tmp ; else echo tmp not found ; fi
+        - ./bin/spack clean -a
   - build-job-remove:
       image:: macos-run-on-metal

share/spack/gitlab/cloud_pipelines/stacks/bootstrap-aarch64-darwin/spack.yaml

@@ -25,8 +25,6 @@ spack:
 
   ci:
     pipeline-gen:
-    - build-job-remove:
-        tags: [spack, public]
     - build-job:
         variables:
           CI_GPG_KEY_ROOT: /etc/protected-runner

share/spack/gitlab/cloud_pipelines/stacks/developer-tools-darwin/spack.yaml

@@ -9,7 +9,7 @@ spack:
     reuse: false
   specs:
       # editors
-    - neovim~no_luajit
+    #- neovim~no_luajit # build fails: https://github.com/spack/spack/pull/48453#issuecomment-2624788262
     - py-pynvim
     - emacs+json~native+treesitter   # TODO native not supported until gcc builds on darwin
       # - tree-sitter is a dep, should also have cli but no package
@@ -64,8 +64,6 @@ spack:
 
   ci:
     pipeline-gen:
-    - build-job-remove:
-        tags: [ spack, public ]
     - build-job:
         variables:
           CI_GPG_KEY_ROOT: /etc/protected-runner

share/spack/gitlab/cloud_pipelines/stacks/ml-darwin-aarch64-mps/spack.yaml

@@ -9,13 +9,13 @@ spack:
       - ~cuda
       - ~rocm
     mpi:
-      require: openmpi
+      require: mpich
     openblas:
       require: ~fortran
 
   specs:
   # Horovod
-  - py-horovod
+  # - py-horovod # https://github.com/spack/spack/pull/48453#issuecomment-2676023970
 
   # Hugging Face
   - py-transformers
@@ -45,18 +45,18 @@ spack:
   - py-segmentation-models-pytorch
   - py-timm
   - py-torch
-  - py-torch-cluster
+  # - py-torch-cluster      # https://github.com/spack/spack/pull/48453#issuecomment-2676023970
   - py-torch-geometric
-  - py-torch-nvidia-apex
-  - py-torch-scatter
-  - py-torch-sparse
-  - py-torch-spline-conv
+  # - py-torch-nvidia-apex  # https://github.com/spack/spack/pull/48453#issuecomment-2676023970
+  # - py-torch-scatter      # https://github.com/spack/spack/pull/48453#issuecomment-2676023970
+  # - py-torch-sparse       # https://github.com/spack/spack/pull/48453#issuecomment-2676023970
+  # - py-torch-spline-conv  # https://github.com/spack/spack/pull/48453#issuecomment-2676023970
   - py-torchaudio
   - py-torchdata
   - py-torchfile
   - py-torchgeo
   - py-torchmetrics
-  - py-torchtext
+  # - py-torchtext          # https://github.com/spack/spack/pull/48453#issuecomment-2676023970
   - py-torchvision
   - py-vector-quantize-pytorch
 
@@ -84,8 +84,6 @@ spack:
 
   ci:
     pipeline-gen:
-    - build-job-remove:
-        tags: [ spack, public ]
     - build-job:
         variables:
           CI_GPG_KEY_ROOT: /etc/protected-runner