Pcluster local buildcache (#37852)

* [pcluster pipeline] Use local buildcache instead of upstream spack

Spack currently does not relocate compiler references from upstream spack
installations. When using a buildcache we don't need an upstream spack.

* gcc needs to be installed via postinstall to get correct deps

* quantum-espresso@gcc@12.3.0 returns ICE on neoverse_{n,v}1

* Force gitlab to pull the new container

* Revert "Force gitlab to pull the new container"

This reverts commit 3af5f4cd88.

Seems the gitlab version does not yet support "pull_policy" in .gitlab-ci.yml

* Gitlab keeps picking up wrong container. Renaming

* Update containers once more after failed build

share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml

@@ -788,20 +788,17 @@ deprecated-ci-build:
 ########################################
 
 .aws-pcluster-generate-image:
-  image: { "name": "ghcr.io/spack/pcluster-amazonlinux-2:latest", "entrypoint": [""] }
+  image: { "name": "ghcr.io/spack/pcluster-amazonlinux-2:v2023-05-25", "entrypoint": [""] }
 
 .aws-pcluster-generate:
   before_script:
-      # Setup postinstall Spack as upstream installation
+    # Use gcc from local container buildcache
     - - . "./share/spack/setup-env.sh"
       - . /etc/profile.d/modules.sh
-      - if [[ -f /bootstrap/spack/etc/spack/packages.yaml ]]; then cp /bootstrap/spack/etc/spack/packages.yaml ./etc/spack/; fi
+      - spack mirror add local-cache /bootstrap/local-cache
-      - if [[ -f /bootstrap/spack/etc/spack/config.yaml ]]; then cp /bootstrap/spack/etc/spack/config.yaml ./etc/spack/; fi
+      - spack gpg trust /bootstrap/public-key
-      - if [[ -f /bootstrap/spack/etc/spack/modules.xyaml ]]; then cp /bootstrap/spack/etc/spack/modules.yaml ./etc/spack/; fi
-      - if [[ -f /bootstrap/spack/etc/spack/mirrors.yaml ]]; then cp /bootstrap/spack/etc/spack/mirrors.yaml ./etc/spack/; fi
-      - if [[ -d /bootstrap/spack/opt/spack ]]; then spack config add "upstreams:postinstall:install_tree:/bootstrap/spack/opt/spack"; fi
       - cd "${CI_PROJECT_DIR}" && curl -sOL https://raw.githubusercontent.com/spack/spack-configs/main/AWS/parallelcluster/postinstall.sh
-      - sed -i -e '/nohup/s/&$//' -e 's/nohup//' -e "s/spack arch -t/echo ${SPACK_TARGET_ARCH}/g" postinstall.sh
+      - sed -i -e "s/spack arch -t/echo ${SPACK_TARGET_ARCH}/g" postinstall.sh
       - /bin/bash postinstall.sh -fg
       - spack config --scope site add "packages:all:target:\"target=${SPACK_TARGET_ARCH}\""
   after_script:

share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-icelake/spack.yaml

@@ -35,18 +35,15 @@ spack:
   ci:
     pipeline-gen:
     - build-job:
-        image: { "name": "ghcr.io/spack/pcluster-amazonlinux-2:latest", "entrypoint": [""] }
+        image: { "name": "ghcr.io/spack/pcluster-amazonlinux-2:v2023-05-25", "entrypoint": [""] }
         before_script:
         - - . "./share/spack/setup-env.sh"
           - . /etc/profile.d/modules.sh
           - spack --version
           - spack arch
-        # Setup postinstall Spack as upstream installation
+        # Use gcc from local container buildcache
-        - - if [[ -f /bootstrap/spack/etc/spack/packages.yaml ]]; then cp /bootstrap/spack/etc/spack/packages.yaml ./etc/spack/; fi
+        - - spack mirror add local-cache /bootstrap/local-cache
-          - if [[ -f /bootstrap/spack/etc/spack/config.yaml ]]; then cp /bootstrap/spack/etc/spack/config.yaml ./etc/spack/; fi
+          - spack gpg trust /bootstrap/public-key
-          - if [[ -f /bootstrap/spack/etc/spack/modules.yaml ]]; then cp /bootstrap/spack/etc/spack/modules.yaml ./etc/spack/; fi
-          - if [[ -f /bootstrap/spack/etc/spack/mirrors.yaml ]]; then cp /bootstrap/spack/etc/spack/mirrors.yaml ./etc/spack/; fi
-          - if [[ -d /bootstrap/spack/opt/spack ]]; then spack config add "upstreams:postinstall:install_tree:/bootstrap/spack/opt/spack"; fi
         - - /bin/bash "${SPACK_ARTIFACTS_ROOT}/postinstall.sh" -fg
           - spack config --scope site add "packages:all:target:\"target=${SPACK_TARGET_ARCH}\""
     - signing-job:

share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-neoverse_n1/spack.yaml

@@ -16,7 +16,7 @@ spack:
     - openfoam
     - palace
     # - py-devito
-    - quantum-espresso
+    # - quantum-espresso
     # - wrf
 
   - optimized_libs:
@@ -36,19 +36,16 @@ spack:
   ci:
     pipeline-gen:
     - build-job:
-        image: { "name": "ghcr.io/spack/pcluster-amazonlinux-2:latest", "entrypoint": [""] }
+        image: { "name": "ghcr.io/spack/pcluster-amazonlinux-2:v2023-05-25", "entrypoint": [""] }
         tags: ["aarch64"]
         before_script:
         - - . "./share/spack/setup-env.sh"
           - . /etc/profile.d/modules.sh
           - spack --version
           - spack arch
-        # Setup postinstall Spack as upstream installation
+        # Use gcc from local container buildcache
-        - - if [[ -f /bootstrap/spack/etc/spack/packages.yaml ]]; then cp /bootstrap/spack/etc/spack/packages.yaml ./etc/spack/; fi
+        - - spack mirror add local-cache /bootstrap/local-cache
-          - if [[ -f /bootstrap/spack/etc/spack/config.yaml ]]; then cp /bootstrap/spack/etc/spack/config.yaml ./etc/spack/; fi
+          - spack gpg trust /bootstrap/public-key
-          - if [[ -f /bootstrap/spack/etc/spack/modules.yaml ]]; then cp /bootstrap/spack/etc/spack/modules.yaml ./etc/spack/; fi
-          - if [[ -f /bootstrap/spack/etc/spack/mirrors.yaml ]]; then cp /bootstrap/spack/etc/spack/mirrors.yaml ./etc/spack/; fi
-          - if [[ -d /bootstrap/spack/opt/spack ]]; then spack config add "upstreams:postinstall:install_tree:/bootstrap/spack/opt/spack"; fi
         - - /bin/bash "${SPACK_ARTIFACTS_ROOT}/postinstall.sh" -fg
           - spack config --scope site add "packages:all:target:\"target=${SPACK_TARGET_ARCH}\""
     - signing-job:

share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-neoverse_v1/spack.yaml

@@ -16,7 +16,7 @@ spack:
     - openfoam
     - palace
     # - py-devito
-    - quantum-espresso
+    # - quantum-espresso
     # - wrf
 
   - optimized_libs:
@@ -36,19 +36,16 @@ spack:
   ci:
     pipeline-gen:
     - build-job:
-        image: { "name": "ghcr.io/spack/pcluster-amazonlinux-2:latest", "entrypoint": [""] }
+        image: { "name": "ghcr.io/spack/pcluster-amazonlinux-2:v2023-05-25", "entrypoint": [""] }
         tags: ["aarch64"]
         before_script:
         - - . "./share/spack/setup-env.sh"
           - . /etc/profile.d/modules.sh
           - spack --version
           - spack arch
-        # Setup postinstall Spack as upstream installation
+        # Use gcc from local container buildcache
-        - - if [[ -f /bootstrap/spack/etc/spack/packages.yaml ]]; then cp /bootstrap/spack/etc/spack/packages.yaml ./etc/spack/; fi
+        - - spack mirror add local-cache /bootstrap/local-cache
-          - if [[ -f /bootstrap/spack/etc/spack/config.yaml ]]; then cp /bootstrap/spack/etc/spack/config.yaml ./etc/spack/; fi
+          - spack gpg trust /bootstrap/public-key
-          - if [[ -f /bootstrap/spack/etc/spack/modules.yaml ]]; then cp /bootstrap/spack/etc/spack/modules.yaml ./etc/spack/; fi
-          - if [[ -f /bootstrap/spack/etc/spack/mirrors.yaml ]]; then cp /bootstrap/spack/etc/spack/mirrors.yaml ./etc/spack/; fi
-          - if [[ -d /bootstrap/spack/opt/spack ]]; then spack config add "upstreams:postinstall:install_tree:/bootstrap/spack/opt/spack"; fi
         - - /bin/bash "${SPACK_ARTIFACTS_ROOT}/postinstall.sh" -fg
           - spack config --scope site add "packages:all:target:\"target=${SPACK_TARGET_ARCH}\""
     - signing-job:

share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-skylake/spack.yaml

@@ -35,18 +35,15 @@ spack:
   ci:
     pipeline-gen:
     - build-job:
-        image: { "name": "ghcr.io/spack/pcluster-amazonlinux-2:latest", "entrypoint": [""] }
+        image: { "name": "ghcr.io/spack/pcluster-amazonlinux-2:v2023-05-25", "entrypoint": [""] }
         before_script:
         - - . "./share/spack/setup-env.sh"
           - . /etc/profile.d/modules.sh
           - spack --version
           - spack arch
-        # Setup postinstall Spack as upstream installation
+        # Use gcc from local container buildcache
-        - - if [[ -f /bootstrap/spack/etc/spack/packages.yaml ]]; then cp /bootstrap/spack/etc/spack/packages.yaml ./etc/spack/; fi
+        - - spack mirror add local-cache /bootstrap/local-cache
-          - if [[ -f /bootstrap/spack/etc/spack/config.yaml ]]; then cp /bootstrap/spack/etc/spack/config.yaml ./etc/spack/; fi
+          - spack gpg trust /bootstrap/public-key
-          - if [[ -f /bootstrap/spack/etc/spack/modules.yaml ]]; then cp /bootstrap/spack/etc/spack/modules.yaml ./etc/spack/; fi
-          - if [[ -f /bootstrap/spack/etc/spack/mirrors.yaml ]]; then cp /bootstrap/spack/etc/spack/mirrors.yaml ./etc/spack/; fi
-          - if [[ -d /bootstrap/spack/opt/spack ]]; then spack config add "upstreams:postinstall:install_tree:/bootstrap/spack/opt/spack"; fi
         - - /bin/bash "${SPACK_ARTIFACTS_ROOT}/postinstall.sh" -fg
           - spack config --scope site add "packages:all:target:\"target=${SPACK_TARGET_ARCH}\""
     - signing-job: