E4S Cray CI Stack (#37837)

* e4s cray ci stack

* e4s ci: add cray

* add zen4 tag

* WIP: new defintions just for cray

* updates

* remove ci signing job overrride, not necessary

* echo $PATH and show modules loaded

* add mirror

* add external def for cray-libsci

* comment out quantum-espresso

* use /etc/protected-runner as key path

* cray ci stack: do not remove tags: [spack, public]

* make cray stack composable

* generate job should run on public tagged runner, override default config:install_tree:root

* CI: Use relative path in default script

* CI: Use relative includes paths for shell runners

* Use concrete_env_dir for relpath

* ml-darwin-aarch64-mps: jax has bazel codesign issue

---------

Co-authored-by: Scott Wittenburg <scott.wittenburg@kitware.com>
Co-authored-by: Ryan Krattiger <ryan.krattiger@kitware.com>

share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml

@@ -4,6 +4,11 @@ default:
   image: { "name": "ghcr.io/spack/e4s-ubuntu-18.04:v2021-10-18", "entrypoint": [""] }
 
 # CI Platform-Arch
+.cray_zen4:
+  variables:
+    SPACK_TARGET_PLATFORM: "cray"
+    SPACK_TARGET_ARCH: "zen4"
+
 .darwin_x86_64:
   variables:
     SPACK_TARGET_PLATFORM: "darwin"
@@ -815,3 +820,89 @@ aws-pcluster-build-neoverse_v1:
   needs:
     - artifacts: True
       job: aws-pcluster-generate-neoverse_v1
+
+# Cray definitions
+.base-cray-job:
+  variables:
+    SPACK_BUILDCACHE_DESTINATION: "s3://spack-binaries-cray/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}"
+    AWS_ACCESS_KEY_ID: ${CRAY_MIRRORS_AWS_ACCESS_KEY_ID}
+    AWS_SECRET_ACCESS_KEY: ${CRAY_MIRRORS_AWS_SECRET_ACCESS_KEY}
+  rules:
+    - if: $CI_COMMIT_REF_NAME == "develop"
+      # Pipelines on develop only rebuild what is missing from the mirror
+      when: always
+      variables:
+        SPACK_PIPELINE_TYPE: "spack_protected_branch"
+    - if: $CI_COMMIT_REF_NAME =~ /^pr[\d]+_.*$/
+      # Pipelines on PR branches rebuild only what's missing, and do extra pruning
+      when: always
+      variables:
+        SPACK_PIPELINE_TYPE: "spack_pull_request"
+        SPACK_BUILDCACHE_DESTINATION: "s3://spack-binaries-cray/prs/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}"
+        SPACK_PRUNE_UNTOUCHED: "True"
+        SPACK_PRUNE_UNTOUCHED_DEPENDENT_DEPTH: "1"
+
+.generate-cray:
+  tags: [ "cce@15.0.1", "cray-zen4", "public" ]
+  extends: [ ".base-cray-job" ]
+  stage: generate
+  script:
+    - echo $PATH
+    - module avail
+    - module list
+    - export SPACK_DISABLE_LOCAL_CONFIG=1
+    - export SPACK_USER_CACHE_PATH=$(pwd)/_user_cache
+    - uname -a || true
+    - grep -E 'vendor|model name' /proc/cpuinfo 2>/dev/null | sort -u || head -n10 /proc/cpuinfo 2>/dev/null || true
+    - nproc || true
+    - . "./share/spack/setup-env.sh"
+    - spack --version
+    - cd share/spack/gitlab/cloud_pipelines/stacks/${SPACK_CI_STACK_NAME}
+    - spack env activate --without-view .
+    - export SPACK_CI_CONFIG_ROOT="${SPACK_ROOT}/share/spack/gitlab/cloud_pipelines/configs"
+    - spack
+      --config-scope "${SPACK_CI_CONFIG_ROOT}"
+      --config-scope "${SPACK_CI_CONFIG_ROOT}/${SPACK_TARGET_PLATFORM}"
+      --config-scope "${SPACK_CI_CONFIG_ROOT}/${SPACK_TARGET_PLATFORM}/${SPACK_TARGET_ARCH}"
+      ${CI_STACK_CONFIG_SCOPES}
+      ci generate --check-index-only
+      --buildcache-destination "${SPACK_BUILDCACHE_DESTINATION}"
+      --artifacts-root "${CI_PROJECT_DIR}/jobs_scratch_dir"
+      --output-file "${CI_PROJECT_DIR}/jobs_scratch_dir/cloud-ci-pipeline.yml"
+  after_script:
+    - cat /proc/loadavg || true
+  artifacts:
+    paths:
+      - "${CI_PROJECT_DIR}/jobs_scratch_dir"
+  interruptible: true
+  timeout: 60 minutes
+  retry:
+    max: 2
+    when:
+      - always
+
+.build-cray:
+  extends: [ ".base-cray-job" ]
+  stage: build
+
+#######################################
+# E4S - Cray
+#######################################
+.e4s-cray:
+  extends: [ ".cray_zen4" ]
+  variables:
+    SPACK_CI_STACK_NAME: e4s-cray
+
+e4s-cray-generate:
+  extends: [ ".generate-cray", ".e4s-cray" ]
+
+e4s-cray-build:
+  extends: [ ".build-cray", ".e4s-cray" ]
+  trigger:
+    include:
+      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
+        job: e4s-cray-generate
+    strategy: depend
+  needs:
+    - artifacts: True
+      job: e4s-cray-generate