Add macOS ML CI stacks (#36586)

* Add macOS ML CI stacks * torchmeta is no longer maintained and requires ancient PyTorch * Add MXNet * update darwin aarch64 stacks * add darwin-aarch64 scoped config.yaml * remove unnecessary cleanup job * fix specifications * fix labels * fix labels * fix indent on tags specification * no tags for trigger jobs * try overriding tags in stack spack.yaml * do not use CI_STACK_CONFIG_SCOPES * incorporate config:install_tree:root: overrides and compiler defs * copy relevant ci-scoped config settings directly into stack spack.yaml * remove build-job-remove * spack ci generate: add debug flag * include cdash config directly in stack spack.yaml * customize build-job script section to avoid absolute paths * add any-job specification * tags: use aarch64-macos instead of aarch64 * generate tags: use aarch64-macos instead of aarch64 * do not add morepadding * use shared mirror; comment out known failures * remove any-job * nproc || true * comment out specs failing due to bazel from cache codesign issue --------- Co-authored-by: eugeneswalker <eugenesunsetwalker@gmail.com>
2023-05-25 00:12:54 -05:00
parent 06f9bcf734
commit 2f8cea2792
6 changed files with 223 additions and 172 deletions
--- a/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml
+++ b/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml
@@ -128,6 +128,38 @@ default:
  extends: [ ".base-job", ".generate-base" ]
  tags: ["spack", "public", "medium", "x86_64"]

+.darwin-generate-base:
+  stage: generate
+  script:
+  - export SPACK_DISABLE_LOCAL_CONFIG=1
+  - export SPACK_USER_CACHE_PATH=$(pwd)/_user_cache
+  - uname -a || true
+  - grep -E 'vendor|model name' /proc/cpuinfo 2>/dev/null | sort -u || head -n10 /proc/cpuinfo 2>/dev/null || true
+  - nproc || true
+  - . "./share/spack/setup-env.sh"
+  - spack --version
+  - cd share/spack/gitlab/cloud_pipelines/stacks/${SPACK_CI_STACK_NAME}
+  - spack env activate --without-view .
+  - spack -d ci generate --check-index-only
+      --buildcache-destination "${SPACK_BUILDCACHE_DESTINATION}"
+      --artifacts-root "${CI_PROJECT_DIR}/jobs_scratch_dir"
+      --output-file "${CI_PROJECT_DIR}/jobs_scratch_dir/cloud-ci-pipeline.yml"
+  after_script:
+  - cat /proc/loadavg || true
+  artifacts:
+    paths:
+    - "${CI_PROJECT_DIR}/jobs_scratch_dir"
+  interruptible: true
+  timeout: 60 minutes
+  retry:
+    max: 2
+    when:
+    - always
+
+.darwin-generate:
+  extends: [ ".base-job", ".darwin-generate-base" ]
+
+
 .generate-deprecated:
  extends: [ ".base-job" ]
  stage: generate
@@ -236,124 +268,6 @@ protected-publish:
 #     - artifacts: True
 #       job: my-super-cool-stack-generate

-########################################
-#          E4S Mac Stack
-#
-# With no near-future plans to have
-# protected aws runners running mac
-# builds, it seems best to decouple
-# them from the rest of the stacks for
-# the time being.  This way they can
-# still run on UO runners and be signed
-# using the previous approach.
-########################################
-# .e4s-mac:
-#   variables:
-#     SPACK_CI_STACK_NAME: e4s-mac
-#   allow_failure: True
-
-# .mac-pr:
-#   only:
-#   - /^pr[\d]+_.*$/
-#   - /^github\/pr[\d]+_.*$/
-#   variables:
-#     SPACK_BUILDCACHE_DESTINATION: "s3://spack-binaries-prs/${CI_COMMIT_REF_NAME}"
-#     SPACK_PRUNE_UNTOUCHED: "True"
-
-# .mac-protected:
-#   only:
-#   - /^develop$/
-#   - /^releases\/v.*/
-#   - /^v.*/
-#   - /^github\/develop$/
-#   variables:
-#     SPACK_BUILDCACHE_DESTINATION: "s3://spack-binaries/${CI_COMMIT_REF_NAME}/${SPACK_CI_STACK_NAME}"
-
-# .mac-pr-build:
-#   extends: [ ".mac-pr", ".build" ]
-#   variables:
-#     AWS_ACCESS_KEY_ID: ${PR_MIRRORS_AWS_ACCESS_KEY_ID}
-#     AWS_SECRET_ACCESS_KEY: ${PR_MIRRORS_AWS_SECRET_ACCESS_KEY}
-# .mac-protected-build:
-#   extends: [ ".mac-protected", ".build" ]
-#   variables:
-#     AWS_ACCESS_KEY_ID: ${PROTECTED_MIRRORS_AWS_ACCESS_KEY_ID}
-#     AWS_SECRET_ACCESS_KEY: ${PROTECTED_MIRRORS_AWS_SECRET_ACCESS_KEY}
-#     SPACK_SIGNING_KEY: ${PACKAGE_SIGNING_KEY}
-
-# e4s-mac-pr-generate:
-#   extends: [".e4s-mac", ".mac-pr"]
-#   stage: generate
-#   script:
-#     - tmp="$(mktemp -d)"; export SPACK_USER_CONFIG_PATH="$tmp"; export SPACK_USER_CACHE_PATH="$tmp"
-#     - . "./share/spack/setup-env.sh"
-#     - spack --version
-#     - cd share/spack/gitlab/cloud_pipelines/stacks/${SPACK_CI_STACK_NAME}
-#     - spack env activate --without-view .
-#     - spack ci generate --check-index-only
-#       --buildcache-destination "${SPACK_BUILDCACHE_DESTINATION}"
-#       --artifacts-root "${CI_PROJECT_DIR}/jobs_scratch_dir"
-#       --output-file "${CI_PROJECT_DIR}/jobs_scratch_dir/cloud-ci-pipeline.yml"
-#   artifacts:
-#     paths:
-#       - "${CI_PROJECT_DIR}/jobs_scratch_dir"
-#   tags:
-#   - lambda
-#   interruptible: true
-#   retry:
-#     max: 2
-#     when:
-#       - runner_system_failure
-#       - stuck_or_timeout_failure
-#   timeout: 60 minutes
-
-# e4s-mac-protected-generate:
-#   extends: [".e4s-mac", ".mac-protected"]
-#   stage: generate
-#   script:
-#     - tmp="$(mktemp -d)"; export SPACK_USER_CONFIG_PATH="$tmp"; export SPACK_USER_CACHE_PATH="$tmp"
-#     - . "./share/spack/setup-env.sh"
-#     - spack --version
-#     - cd share/spack/gitlab/cloud_pipelines/stacks/${SPACK_CI_STACK_NAME}
-#     - spack env activate --without-view .
-#     - spack ci generate --check-index-only
-#       --artifacts-root "${CI_PROJECT_DIR}/jobs_scratch_dir"
-#       --output-file "${CI_PROJECT_DIR}/jobs_scratch_dir/cloud-ci-pipeline.yml"
-#   artifacts:
-#     paths:
-#       - "${CI_PROJECT_DIR}/jobs_scratch_dir"
-#   tags:
-#   - omicron
-#   interruptible: true
-#   retry:
-#     max: 2
-#     when:
-#       - runner_system_failure
-#       - stuck_or_timeout_failure
-#   timeout: 60 minutes
-
-# e4s-mac-pr-build:
-#   extends: [ ".e4s-mac", ".mac-pr-build" ]
-#   trigger:
-#     include:
-#       - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
-#         job: e4s-mac-pr-generate
-#     strategy: depend
-#   needs:
-#     - artifacts: True
-#       job: e4s-mac-pr-generate
-
-# e4s-mac-protected-build:
-#   extends: [ ".e4s-mac", ".mac-protected-build" ]
-#   trigger:
-#     include:
-#       - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
-#         job: e4s-mac-protected-generate
-#     strategy: depend
-#   needs:
-#     - artifacts: True
-#       job: e4s-mac-protected-generate
-
 ########################################
 # E4S pipeline
 ########################################
@@ -762,6 +676,28 @@ ml-linux-x86_64-rocm-build:
    - artifacts: True
      job: ml-linux-x86_64-rocm-generate

+########################################
+# Machine Learning - Darwin aarch64 (MPS)
+########################################
+.ml-darwin-aarch64-mps:
+  variables:
+    SPACK_CI_STACK_NAME: ml-darwin-aarch64-mps
+
+ml-darwin-aarch64-mps-generate:
+  tags: [ "macos-ventura", "apple-clang-14", "aarch64-macos" ]
+  extends: [ ".ml-darwin-aarch64-mps", ".darwin-generate"]
+
+ml-darwin-aarch64-mps-build:
+  extends: [ ".ml-darwin-aarch64-mps", ".build" ]
+  trigger:
+    include:
+      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
+        job: ml-darwin-aarch64-mps-generate
+    strategy: depend
+  needs:
+    - artifacts: True
+      job: ml-darwin-aarch64-mps-generate
+
 ########################################
 # Deprecated CI testing
 ########################################