ci: require x86_64_v3 everywhere (#36158)

2023-03-29 15:58:48 +02:00 · 2023-03-29 15:58:48 +02:00 · dba57ff113
commit dba57ff113
parent 7579eaf75a
13 changed files with 232 additions and 262 deletions
--- a/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml
+++ b/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml
@ -9,15 +9,10 @@ default:
    SPACK_TARGET_PLATFORM: "darwin"
    SPACK_TARGET_ARCH: "x86_64"

-.linux_x86_64:
+.linux_x86_64_v3:
  variables:
    SPACK_TARGET_PLATFORM: "linux"
-    SPACK_TARGET_ARCH: "x86_64"
-
-.linux_x86_64_v4:
-  variables:
-    SPACK_TARGET_PLATFORM: "linux"
-    SPACK_TARGET_ARCH: "x86_64_v4"
+    SPACK_TARGET_ARCH: "x86_64_v3"

 .linux_aarch64:
  variables:
@ -169,7 +164,7 @@ protected-publish:
 # My Super Cool Pipeline
 ########################################
 # .my-super-cool-stack:
-#   extends: [ ".linux_x86_64" ]
+#   extends: [ ".linux_x86_64_v3" ]
 #   variables:
 #     SPACK_CI_STACK_NAME: my-super-cool-stack
 #     tags: [ "all", "tags", "your", "job", "needs"]
@ -324,7 +319,7 @@ protected-publish:
 # E4S pipeline
 ########################################
 .e4s:
-  extends: [ ".linux_x86_64" ]
+  extends: [ ".linux_x86_64_v3" ]
  variables:
    SPACK_CI_STACK_NAME: e4s

@ -362,7 +357,7 @@ e4s-protected-build:
 # GPU Testing Pipeline
 ########################################
 .gpu-tests:
-  extends: [ ".linux_x86_64" ]
+  extends: [ ".linux_x86_64_v3" ]
  variables:
    SPACK_CI_STACK_NAME: gpu-tests

@ -400,7 +395,7 @@ gpu-tests-protected-build:
 # E4S OneAPI Pipeline
 ########################################
 .e4s-oneapi:
-  extends: [ ".linux_x86_64" ]
+  extends: [ ".linux_x86_64_v3" ]
  variables:
    SPACK_CI_STACK_NAME: e4s-oneapi

@ -478,7 +473,7 @@ e4s-power-protected-build:
 # Build tests for different build-systems
 #########################################
 .build_systems:
-  extends: [ ".linux_x86_64" ]
+  extends: [ ".linux_x86_64_v3" ]
  variables:
    SPACK_CI_STACK_NAME: build_systems

@ -514,7 +509,7 @@ build_systems-protected-build:
 # RADIUSS
 #########################################
 .radiuss:
-  extends: [ ".linux_x86_64" ]
+  extends: [ ".linux_x86_64_v3" ]
  variables:
    SPACK_CI_STACK_NAME: radiuss

@ -562,7 +557,7 @@ radiuss-protected-build:
  image: { "name": "ghcr.io/spack/e4s-amazonlinux-2:v2023-03-09", "entrypoint": [""] }

 .radiuss-aws:
-  extends: [ ".linux_x86_64_v4" ]
+  extends: [ ".linux_x86_64_v3" ]
  variables:
    SPACK_CI_STACK_NAME: radiuss-aws

@ -634,7 +629,7 @@ radiuss-aws-aarch64-protected-build:
 # ECP Data & Vis SDK
 ########################################
 .data-vis-sdk:
-  extends: [ ".linux_x86_64" ]
+  extends: [ ".linux_x86_64_v3" ]
  variables:
    SPACK_CI_STACK_NAME: data-vis-sdk

@ -679,7 +674,7 @@ data-vis-sdk-protected-build:
  image: { "name": "ghcr.io/spack/e4s-amazonlinux-2:v2023-03-09", "entrypoint": [""] }

 .aws-ahug:
-  extends: [ ".linux_x86_64_v4" ]
+  extends: [ ".linux_x86_64_v3" ]
  variables:
    SPACK_CI_STACK_NAME: aws-ahug

@ -757,7 +752,7 @@ aws-ahug-aarch64-protected-build:
  image: { "name": "ghcr.io/spack/e4s-amazonlinux-2:v2023-03-09", "entrypoint": [""] }

 .aws-isc:
-  extends: [ ".linux_x86_64_v4" ]
+  extends: [ ".linux_x86_64_v3" ]
  variables:
    SPACK_CI_STACK_NAME: aws-isc

@ -830,7 +825,7 @@ aws-isc-aarch64-protected-build:
 # Spack Tutorial
 ########################################
 .tutorial:
-  extends: [ ".linux_x86_64" ]
+  extends: [ ".linux_x86_64_v3" ]
  variables:
    SPACK_CI_STACK_NAME: tutorial

@ -866,7 +861,7 @@ tutorial-protected-build:
 # Machine Learning - Linux x86_64 (CPU)
 #######################################
 .ml-linux-x86_64-cpu:
-  extends: [ ".linux_x86_64_v4" ]
+  extends: [ ".linux_x86_64_v3" ]
  variables:
    SPACK_CI_STACK_NAME: ml-linux-x86_64-cpu

@ -906,7 +901,7 @@ ml-linux-x86_64-cpu-protected-build:
 # Machine Learning - Linux x86_64 (CUDA)
 ########################################
 .ml-linux-x86_64-cuda:
-  extends: [ ".linux_x86_64_v4" ]
+  extends: [ ".linux_x86_64_v3" ]
  variables:
    SPACK_CI_STACK_NAME: ml-linux-x86_64-cuda

@ -946,7 +941,7 @@ ml-linux-x86_64-cuda-protected-build:
 # Machine Learning - Linux x86_64 (ROCm)
 ########################################
 .ml-linux-x86_64-rocm:
-  extends: [ ".linux_x86_64_v4" ]
+  extends: [ ".linux_x86_64_v3" ]
  variables:
    SPACK_CI_STACK_NAME: ml-linux-x86_64-rocm

--- a/share/spack/gitlab/cloud_pipelines/configs/linux/x86_64_v3/ci.yaml
+++ b/share/spack/gitlab/cloud_pipelines/configs/linux/x86_64_v3/ci.yaml
@ -5,4 +5,4 @@ ci:
      - - curl -LfsS "https://github.com/JuliaBinaryWrappers/GNUMake_jll.jl/releases/download/GNUMake-v4.3.0+1/GNUMake.v4.3.0.x86_64-linux-gnu.tar.gz" -o gmake.tar.gz
        - printf "fef1f59e56d2d11e6d700ba22d3444b6e583c663d6883fd0a4f63ab8bd280f0f gmake.tar.gz" | sha256sum --check --strict --quiet
        - tar -xzf gmake.tar.gz -C /usr bin/make 2> /dev/null
-      tags: ["x86_64"]
+      tags: ["x86_64_v3"]
--- a/share/spack/gitlab/cloud_pipelines/configs/linux/x86_64_v4/ci.yaml
+++ b/share/spack/gitlab/cloud_pipelines/configs/linux/x86_64_v4/ci.yaml
@ -1,8 +0,0 @@
-ci:
-  pipeline-gen:
-  - build-job:
-      before_script:
-      - - curl -LfsS "https://github.com/JuliaBinaryWrappers/GNUMake_jll.jl/releases/download/GNUMake-v4.3.0+1/GNUMake.v4.3.0.x86_64-linux-gnu.tar.gz" -o gmake.tar.gz
-        - printf "fef1f59e56d2d11e6d700ba22d3444b6e583c663d6883fd0a4f63ab8bd280f0f gmake.tar.gz" | sha256sum --check --strict --quiet
-        - tar -xzf gmake.tar.gz -C /usr bin/make 2> /dev/null
-      tags: ["x86_64_v4"]
--- a/share/spack/gitlab/cloud_pipelines/stacks/build_systems/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/build_systems/spack.yaml
@ -1,5 +1,8 @@
 spack:
  view: false
+  packages:
+    all:
+      require: target=x86_64_v3
  definitions:
    - default_specs:
      - 'uncrustify build_system=autotools'
@ -11,7 +14,7 @@ spack:
      - r-rcpp  # RPackage
      - ruby-rake  # RubyPackage
    - arch:
-      - '%gcc target=x86_64'
+      - '%gcc'

  specs:
    - matrix:
--- a/share/spack/gitlab/cloud_pipelines/stacks/data-vis-sdk/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/data-vis-sdk/spack.yaml
@ -10,7 +10,7 @@ spack:
    libglx:
      require: ^mesa +glx
    all:
-      target: [x86_64]
+      require: target=x86_64_v3

  definitions:
  - paraview_specs:
--- a/share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml
@ -6,7 +6,7 @@ spack:
      providers:
        blas: [openblas]
        mpi: [mpich]
-      target: [x86_64]
+      require: target=x86_64_v3
      variants: +mpi amdgpu_target=gfx90a cuda_arch=80
    tbb:
      require: "intel-tbb"
--- a/share/spack/gitlab/cloud_pipelines/stacks/gpu-tests/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/gpu-tests/spack.yaml
@ -7,7 +7,7 @@ spack:
      providers:
        blas: [openblas]
        mpi: [mpich]
-      target: [x86_64]
+      require: target=x86_64_v3
      variants: +mpi amdgpu_target=gfx90a cuda_arch=80
    tbb:
      require: "intel-tbb"
--- a/share/spack/gitlab/cloud_pipelines/stacks/ml-linux-x86_64-cpu/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/ml-linux-x86_64-cpu/spack.yaml
@ -2,84 +2,77 @@ spack:
  view: false
  packages:
    all:
-      target: [x86_64_v3]
+      require: target=x86_64_v3
      variants: ~cuda~rocm
-
-  definitions:
-    - packages:
-      # Horovod
-      - py-horovod
-
-      # Hugging Face
-      - py-transformers
-
-      # JAX
-      - py-jax
-      - py-jaxlib
-
-      # Keras
-      - py-keras
-      - py-keras-applications
-      - py-keras-preprocessing
-      - py-keras2onnx
-
-      # PyTorch
-      - py-botorch
-      - py-efficientnet-pytorch
-      - py-gpytorch
-      - py-kornia
-      - py-lightning
-      - py-pytorch-gradual-warmup-lr
-      - py-pytorch-lightning
-      - py-segmentation-models-pytorch
-      - py-timm
-      - py-torch
-      - py-torch-cluster
-      - py-torch-geometric
-      - py-torch-nvidia-apex
-      - py-torch-scatter
-      - py-torch-sparse
-      - py-torch-spline-conv
-      - py-torchaudio
-      - py-torchdata
-      - py-torchfile
-      - py-torchgeo
-      - py-torchmeta
-      - py-torchmetrics
-      - py-torchtext
-      - py-torchvision
-      - py-vector-quantize-pytorch
-
-      # scikit-learn
-      - py-scikit-learn
-      - py-scikit-learn-extra
-
-      # TensorBoard
-      - py-tensorboard
-      - py-tensorboard-data-server
-      - py-tensorboard-plugin-wit
-      - py-tensorboardx
-
-      # TensorFlow
-      - py-tensorflow
-      - py-tensorflow-datasets
-      - py-tensorflow-estimator
-      - py-tensorflow-hub
-      - py-tensorflow-metadata
-      - py-tensorflow-probability
-
-      # XGBoost
-      - py-xgboost
-      # - r-xgboost
-      - xgboost
-
-    - arch:
-      - target=x86_64_v3
+    mpi:
+      require: openmpi

  specs:
-    - matrix:
-      - [$packages]
-      - [$arch]
+    # Horovod
+    - py-horovod
+
+    # Hugging Face
+    - py-transformers
+
+    # JAX
+    - py-jax
+    - py-jaxlib
+
+    # Keras
+    - py-keras
+    - py-keras-applications
+    - py-keras-preprocessing
+    - py-keras2onnx
+
+    # PyTorch
+    - py-botorch
+    - py-efficientnet-pytorch
+    - py-gpytorch
+    - py-kornia
+    - py-lightning
+    - py-pytorch-gradual-warmup-lr
+    - py-pytorch-lightning
+    - py-segmentation-models-pytorch
+    - py-timm
+    - py-torch
+    - py-torch-cluster
+    - py-torch-geometric
+    - py-torch-nvidia-apex
+    - py-torch-scatter
+    - py-torch-sparse
+    - py-torch-spline-conv
+    - py-torchaudio
+    - py-torchdata
+    - py-torchfile
+    - py-torchgeo
+    - py-torchmeta
+    - py-torchmetrics
+    - py-torchtext
+    - py-torchvision
+    - py-vector-quantize-pytorch
+
+    # scikit-learn
+    - py-scikit-learn
+    - py-scikit-learn-extra
+
+    # TensorBoard
+    - py-tensorboard
+    - py-tensorboard-data-server
+    - py-tensorboard-plugin-wit
+    - py-tensorboardx
+
+    # TensorFlow
+    - py-tensorflow
+    - py-tensorflow-datasets
+    - py-tensorflow-estimator
+    - py-tensorflow-hub
+    - py-tensorflow-metadata
+    - py-tensorflow-probability
+
+    # XGBoost
+    - py-xgboost
+    # - r-xgboost
+    - xgboost

  mirrors: { "mirror": "s3://spack-binaries/develop/ml-linux-x86_64-cpu" }

--- a/share/spack/gitlab/cloud_pipelines/stacks/ml-linux-x86_64-cuda/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/ml-linux-x86_64-cuda/spack.yaml
@ -2,87 +2,80 @@ spack:
  view: false
  packages:
    all:
-      target: [x86_64_v3]
+      require: target=x86_64_v3
      variants: ~rocm+cuda cuda_arch=80
    llvm:
      # https://github.com/spack/spack/issues/27999
      require: ~cuda
-
-  definitions:
-    - packages:
-      # Horovod
-      - py-horovod
-
-      # Hugging Face
-      - py-transformers
-
-      # JAX
-      - py-jax
-      - py-jaxlib
-
-      # Keras
-      - py-keras
-      - py-keras-applications
-      - py-keras-preprocessing
-      - py-keras2onnx
-
-      # PyTorch
-      - py-botorch
-      - py-efficientnet-pytorch
-      - py-gpytorch
-      - py-kornia
-      - py-lightning
-      - py-pytorch-gradual-warmup-lr
-      - py-pytorch-lightning
-      - py-segmentation-models-pytorch
-      - py-timm
-      - py-torch
-      - py-torch-cluster
-      - py-torch-geometric
-      - py-torch-nvidia-apex
-      - py-torch-scatter
-      - py-torch-sparse
-      - py-torch-spline-conv
-      - py-torchaudio
-      - py-torchdata
-      - py-torchfile
-      - py-torchgeo
-      - py-torchmeta
-      - py-torchmetrics
-      - py-torchtext
-      - py-torchvision
-      - py-vector-quantize-pytorch
-
-      # scikit-learn
-      - py-scikit-learn
-      - py-scikit-learn-extra
-
-      # TensorBoard
-      - py-tensorboard
-      - py-tensorboard-data-server
-      - py-tensorboard-plugin-wit
-      - py-tensorboardx
-
-      # TensorFlow
-      - py-tensorflow
-      - py-tensorflow-datasets
-      - py-tensorflow-estimator
-      - py-tensorflow-hub
-      - py-tensorflow-metadata
-      - py-tensorflow-probability
-
-      # XGBoost
-      - py-xgboost
-      # - r-xgboost
-      - xgboost
-
-    - arch:
-      - target=x86_64_v3
+    mpi:
+      require: openmpi

  specs:
-    - matrix:
-      - [$packages]
-      - [$arch]
+    # Horovod
+    - py-horovod
+
+    # Hugging Face
+    - py-transformers
+
+    # JAX
+    - py-jax
+    - py-jaxlib
+
+    # Keras
+    - py-keras
+    - py-keras-applications
+    - py-keras-preprocessing
+    - py-keras2onnx
+
+    # PyTorch
+    - py-botorch
+    - py-efficientnet-pytorch
+    - py-gpytorch
+    - py-kornia
+    - py-lightning
+    - py-pytorch-gradual-warmup-lr
+    - py-pytorch-lightning
+    - py-segmentation-models-pytorch
+    - py-timm
+    - py-torch
+    - py-torch-cluster
+    - py-torch-geometric
+    - py-torch-nvidia-apex
+    - py-torch-scatter
+    - py-torch-sparse
+    - py-torch-spline-conv
+    - py-torchaudio
+    - py-torchdata
+    - py-torchfile
+    - py-torchgeo
+    - py-torchmeta
+    - py-torchmetrics
+    - py-torchtext
+    - py-torchvision
+    - py-vector-quantize-pytorch
+
+    # scikit-learn
+    - py-scikit-learn
+    - py-scikit-learn-extra
+
+    # TensorBoard
+    - py-tensorboard
+    - py-tensorboard-data-server
+    - py-tensorboard-plugin-wit
+    - py-tensorboardx
+
+    # TensorFlow
+    - py-tensorflow
+    - py-tensorflow-datasets
+    - py-tensorflow-estimator
+    - py-tensorflow-hub
+    - py-tensorflow-metadata
+    - py-tensorflow-probability
+
+    # XGBoost
+    - py-xgboost
+    # - r-xgboost
+    - xgboost

  mirrors: { "mirror": "s3://spack-binaries/develop/ml-linux-x86_64-cuda" }

--- a/share/spack/gitlab/cloud_pipelines/stacks/ml-linux-x86_64-rocm/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/ml-linux-x86_64-rocm/spack.yaml
@ -2,90 +2,83 @@ spack:
  view: false
  packages:
    all:
-      target: [x86_64_v3]
+      require: target=x86_64_v3
      variants: ~cuda+rocm amdgpu_target=gfx90a
    gl:
      require: "osmesa"
    py-torch:
      # Does not yet support Spack-installed ROCm
      require: ~rocm
-
-  definitions:
-    - packages:
-      # Horovod
-      - py-horovod
-
-      # Hugging Face
-      - py-transformers
-
-      # JAX
-      - py-jax
-      - py-jaxlib
-
-      # Keras
-      - py-keras
-      - py-keras-applications
-      - py-keras-preprocessing
-      - py-keras2onnx
-
-      # PyTorch
-      # Does not yet support Spack-install ROCm
-      # - py-botorch
-      # - py-efficientnet-pytorch
-      # - py-gpytorch
-      # - py-kornia
-      # - py-lightning
-      # - py-pytorch-gradual-warmup-lr
-      # - py-pytorch-lightning
-      # - py-segmentation-models-pytorch
-      # - py-timm
-      # - py-torch
-      # - py-torch-cluster
-      # - py-torch-geometric
-      # - py-torch-nvidia-apex
-      # - py-torch-scatter
-      # - py-torch-sparse
-      # - py-torch-spline-conv
-      # - py-torchaudio
-      # - py-torchdata
-      # - py-torchfile
-      # - py-torchgeo
-      # - py-torchmeta
-      # - py-torchmetrics
-      # - py-torchtext
-      # - py-torchvision
-      # - py-vector-quantize-pytorch
-
-      # scikit-learn
-      - py-scikit-learn
-      - py-scikit-learn-extra
-
-      # TensorBoard
-      - py-tensorboard
-      - py-tensorboard-data-server
-      - py-tensorboard-plugin-wit
-      - py-tensorboardx
-
-      # TensorFlow
-      - py-tensorflow
-      - py-tensorflow-datasets
-      - py-tensorflow-estimator
-      - py-tensorflow-hub
-      - py-tensorflow-metadata
-      - py-tensorflow-probability
-
-      # XGBoost
-      - py-xgboost
-      # - r-xgboost
-      - xgboost
-
-    - arch:
-      - target=x86_64_v3
+    mpi:
+      require: openmpi

  specs:
-    - matrix:
-      - [$packages]
-      - [$arch]
+    # Horovod
+    - py-horovod
+
+    # Hugging Face
+    - py-transformers
+
+    # JAX
+    - py-jax
+    - py-jaxlib
+
+    # Keras
+    - py-keras
+    - py-keras-applications
+    - py-keras-preprocessing
+    - py-keras2onnx
+
+    # PyTorch
+    # Does not yet support Spack-install ROCm
+    # - py-botorch
+    # - py-efficientnet-pytorch
+    # - py-gpytorch
+    # - py-kornia
+    # - py-lightning
+    # - py-pytorch-gradual-warmup-lr
+    # - py-pytorch-lightning
+    # - py-segmentation-models-pytorch
+    # - py-timm
+    # - py-torch
+    # - py-torch-cluster
+    # - py-torch-geometric
+    # - py-torch-nvidia-apex
+    # - py-torch-scatter
+    # - py-torch-sparse
+    # - py-torch-spline-conv
+    # - py-torchaudio
+    # - py-torchdata
+    # - py-torchfile
+    # - py-torchgeo
+    # - py-torchmeta
+    # - py-torchmetrics
+    # - py-torchtext
+    # - py-torchvision
+    # - py-vector-quantize-pytorch
+
+    # scikit-learn
+    - py-scikit-learn
+    - py-scikit-learn-extra
+
+    # TensorBoard
+    - py-tensorboard
+    - py-tensorboard-data-server
+    - py-tensorboard-plugin-wit
+    - py-tensorboardx
+
+    # TensorFlow
+    - py-tensorflow
+    - py-tensorflow-datasets
+    - py-tensorflow-estimator
+    - py-tensorflow-hub
+    - py-tensorflow-metadata
+    - py-tensorflow-probability
+
+    # XGBoost
+    - py-xgboost
+    # - r-xgboost
+    - xgboost

  mirrors: { "mirror": "s3://spack-binaries/develop/ml-linux-x86_64-rocm" }

--- a/share/spack/gitlab/cloud_pipelines/stacks/radiuss-aws/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/radiuss-aws/spack.yaml
@ -11,6 +11,7 @@ spack:
          - openmpi
          - mpich
      variants: +mpi cuda_arch=70
+      require: target=x86_64_v3

  definitions:
  - radiuss:
--- a/share/spack/gitlab/cloud_pipelines/stacks/radiuss/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/radiuss/spack.yaml
@ -2,7 +2,7 @@ spack:
  view: false
  packages:
    all:
-      target: [x86_64]
+      require: target=x86_64_v3

      providers:
        mpi: [mvapich2]
--- a/share/spack/gitlab/cloud_pipelines/stacks/tutorial/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/tutorial/spack.yaml
@ -7,7 +7,7 @@ spack:
  view: false
  packages:
    all:
-      target: [x86_64]
+      require: target=x86_64_v3
    tbb:
      require: intel-tbb
  definitions: