Add aws-plcuster[-aarch64] stacks (#37627)

Add aws-plcuster[-aarch64] stacks. These stacks build packages defined in https://github.com/spack/spack-configs/tree/main/AWS/parallelcluster They use a custom container from https://github.com/spack/gitlab-runners which includes necessary ParallelCluster software to link and build as well as an upstream spack installation with current GCC and dependencies. Intel and ARM software is installed and used during the build stage but removed from the buildcache before the signing stage. Files `configs/linux/{arch}/ci.yaml` select the necessary providers in order to build for specific architectures (icelake, skylake, neoverse_{n,v}1).
2023-05-18 00:21:10 +02:00 · 2023-05-18 00:21:10 +02:00 · 125c20bc06
commit 125c20bc06
parent f7696a4480
9 changed files with 394 additions and 0 deletions
--- a/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml
+++ b/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml
@ -14,6 +14,26 @@ default:
    SPACK_TARGET_PLATFORM: "linux"
    SPACK_TARGET_ARCH: "x86_64_v3"

+.linux_skylake:
+  variables:
+    SPACK_TARGET_PLATFORM: "linux"
+    SPACK_TARGET_ARCH: "skylake_avx512"
+
+.linux_icelake:
+  variables:
+    SPACK_TARGET_PLATFORM: "linux"
+    SPACK_TARGET_ARCH: "icelake"
+
+.linux_neoverse_n1:
+  variables:
+    SPACK_TARGET_PLATFORM: "linux"
+    SPACK_TARGET_ARCH: "neoverse_n1"
+
+.linux_neoverse_v1:
+  variables:
+    SPACK_TARGET_PLATFORM: "linux"
+    SPACK_TARGET_ARCH: "neoverse_v1"
+
 .linux_aarch64:
  variables:
    SPACK_TARGET_PLATFORM: "linux"
@ -762,3 +782,103 @@ deprecated-ci-build:
  needs:
    - artifacts: True
      job: deprecated-ci-generate
+
+########################################
+# AWS PCLUSTER
+########################################
+
+.aws-pcluster-generate-image:
+  image: { "name": "ghcr.io/spack/pcluster-amazonlinux-2:latest", "entrypoint": [""] }
+
+.aws-pcluster-generate:
+  before_script:
+      # Setup postinstall Spack as upstream installation
+    - - . "./share/spack/setup-env.sh"
+      - . /etc/profile.d/modules.sh
+      - if [[ -f /bootstrap/spack/etc/spack/packages.yaml ]]; then cp /bootstrap/spack/etc/spack/packages.yaml ./etc/spack/; fi
+      - if [[ -f /bootstrap/spack/etc/spack/config.yaml ]]; then cp /bootstrap/spack/etc/spack/config.yaml ./etc/spack/; fi
+      - if [[ -f /bootstrap/spack/etc/spack/modules.xyaml ]]; then cp /bootstrap/spack/etc/spack/modules.yaml ./etc/spack/; fi
+      - if [[ -f /bootstrap/spack/etc/spack/mirrors.yaml ]]; then cp /bootstrap/spack/etc/spack/mirrors.yaml ./etc/spack/; fi
+      - if [[ -d /bootstrap/spack/opt/spack ]]; then spack config add "upstreams:postinstall:install_tree:/bootstrap/spack/opt/spack"; fi
+      - cd "${CI_PROJECT_DIR}" && curl -sOL https://raw.githubusercontent.com/spack/spack-configs/main/AWS/parallelcluster/postinstall.sh
+      - sed -i -e '/nohup/s/&$//' -e 's/nohup//' -e "s/spack arch -t/echo ${SPACK_TARGET_ARCH}/g" postinstall.sh
+      - /bin/bash postinstall.sh -fg
+      - spack config --scope site add "packages:all:target:\"target=${SPACK_TARGET_ARCH}\""
+  after_script:
+    - - mv "${CI_PROJECT_DIR}/postinstall.sh" "${CI_PROJECT_DIR}/jobs_scratch_dir/"
+
+# Icelake (one pipeline per target)
+.aws-pcluster-icelake:
+  variables:
+    SPACK_CI_STACK_NAME: aws-pcluster-icelake
+
+aws-pcluster-generate-icelake:
+  extends: [ ".linux_icelake", ".aws-pcluster-icelake", ".generate", ".tags-x86_64_v4", ".aws-pcluster-generate", ".aws-pcluster-generate-image" ]
+
+aws-pcluster-build-icelake:
+  extends: [ ".linux_icelake", ".aws-pcluster-icelake", ".build" ]
+  trigger:
+    include:
+      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
+        job: aws-pcluster-generate-icelake
+    strategy: depend
+  needs:
+    - artifacts: True
+      job: aws-pcluster-generate-icelake
+
+# Skylake_avx512 (one pipeline per target)
+.aws-pcluster-skylake:
+  variables:
+    SPACK_CI_STACK_NAME: aws-pcluster-skylake
+
+aws-pcluster-generate-skylake:
+  extends: [ ".linux_skylake", ".aws-pcluster-skylake", ".generate", ".tags-x86_64_v4", ".aws-pcluster-generate", ".aws-pcluster-generate-image" ]
+
+aws-pcluster-build-skylake:
+  extends: [ ".linux_skylake", ".aws-pcluster-skylake", ".build" ]
+  trigger:
+    include:
+      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
+        job: aws-pcluster-generate-skylake
+    strategy: depend
+  needs:
+    - artifacts: True
+      job: aws-pcluster-generate-skylake
+
+# Neoverse_n1 (one pipeline per target)
+.aws-pcluster-neoverse_n1:
+  variables:
+    SPACK_CI_STACK_NAME: aws-pcluster-neoverse_n1
+
+aws-pcluster-generate-neoverse_n1:
+  extends: [ ".linux_neoverse_n1", ".aws-pcluster-neoverse_n1", ".generate-aarch64", ".aws-pcluster-generate", ".aws-pcluster-generate-image" ]
+
+aws-pcluster-build-neoverse_n1:
+  extends: [  ".linux_neoverse_n1", ".aws-pcluster-neoverse_n1", ".build" ]
+  trigger:
+    include:
+      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
+        job: aws-pcluster-generate-neoverse_n1
+    strategy: depend
+  needs:
+    - artifacts: True
+      job: aws-pcluster-generate-neoverse_n1
+
+# Neoverse_v1 (one pipeline per target)
+.aws-pcluster-neoverse_v1:
+  variables:
+    SPACK_CI_STACK_NAME: aws-pcluster-neoverse_v1
+
+aws-pcluster-generate-neoverse_v1:
+  extends: [ ".linux_neoverse_v1", ".aws-pcluster-neoverse_v1", ".generate-aarch64", ".aws-pcluster-generate", ".aws-pcluster-generate-image" ]
+
+aws-pcluster-build-neoverse_v1:
+  extends: [  ".linux_neoverse_v1", ".aws-pcluster-neoverse_v1", ".build" ]
+  trigger:
+    include:
+      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
+        job: aws-pcluster-generate-neoverse_v1
+    strategy: depend
+  needs:
+    - artifacts: True
+      job: aws-pcluster-generate-neoverse_v1
--- a/share/spack/gitlab/cloud_pipelines/configs/linux/icelake/ci.yaml
+++ b/share/spack/gitlab/cloud_pipelines/configs/linux/icelake/ci.yaml
@ -0,0 +1,11 @@
+ci:
+  pipeline-gen:
+  - any-job:
+      variables:
+        SPACK_TARGET_ARCH: icelake
+  - build-job:
+      before_script:
+      - - curl -LfsS "https://github.com/JuliaBinaryWrappers/GNUMake_jll.jl/releases/download/GNUMake-v4.3.0+1/GNUMake.v4.3.0.x86_64-linux-gnu.tar.gz" -o gmake.tar.gz
+        - printf "fef1f59e56d2d11e6d700ba22d3444b6e583c663d6883fd0a4f63ab8bd280f0f gmake.tar.gz" | sha256sum --check --strict --quiet
+        - tar -xzf gmake.tar.gz -C /usr bin/make 2> /dev/null
+      tags: ["x86_64_v4"]
--- a/share/spack/gitlab/cloud_pipelines/configs/linux/neoverse_n1/ci.yaml
+++ b/share/spack/gitlab/cloud_pipelines/configs/linux/neoverse_n1/ci.yaml
@ -0,0 +1,7 @@
+ci:
+  pipeline-gen:
+  - any-job:
+      variables:
+        SPACK_TARGET_ARCH: neoverse_n1
+  - build-job:
+      tags: ["aarch64", "graviton2"]
--- a/share/spack/gitlab/cloud_pipelines/configs/linux/neoverse_v1/ci.yaml
+++ b/share/spack/gitlab/cloud_pipelines/configs/linux/neoverse_v1/ci.yaml
@ -0,0 +1,7 @@
+ci:
+  pipeline-gen:
+  - any-job:
+      variables:
+        SPACK_TARGET_ARCH: neoverse_v1
+  - build-job:
+      tags: ["aarch64", "graviton3"]
--- a/share/spack/gitlab/cloud_pipelines/configs/linux/skylake_avx512/ci.yaml
+++ b/share/spack/gitlab/cloud_pipelines/configs/linux/skylake_avx512/ci.yaml
@ -0,0 +1,11 @@
+ci:
+  pipeline-gen:
+  - any-job:
+      variables:
+        SPACK_TARGET_ARCH: skylake_avx512
+  - build-job:
+      before_script:
+      - - curl -LfsS "https://github.com/JuliaBinaryWrappers/GNUMake_jll.jl/releases/download/GNUMake-v4.3.0+1/GNUMake.v4.3.0.x86_64-linux-gnu.tar.gz" -o gmake.tar.gz
+        - printf "fef1f59e56d2d11e6d700ba22d3444b6e583c663d6883fd0a4f63ab8bd280f0f gmake.tar.gz" | sha256sum --check --strict --quiet
+        - tar -xzf gmake.tar.gz -C /usr bin/make 2> /dev/null
+      tags: ["x86_64_v4"]
--- a/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-icelake/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-icelake/spack.yaml
@ -0,0 +1,58 @@
+spack:
+  view: false
+
+  definitions:
+  - compiler_specs:
+    - gcc
+    - gettext
+
+  - compiler_target:
+    - '%gcc@7.3.1 target=x86_64_v3'
+
+  - optimized_configs:
+    # - gromacs
+    - lammps
+    # - mpas-model
+    - openfoam
+    # - palace
+    # - py-devito
+    # - quantum-espresso
+    # - wrf
+
+  - optimized_libs:
+    - mpich
+    - openmpi
+
+  specs:
+  - matrix:
+    - - $compiler_specs
+    - - $compiler_target
+  - $optimized_configs
+  # - $optimized_libs
+
+  mirrors: { "mirror": "s3://spack-binaries/develop/aws-pcluster-icelake" }
+
+  ci:
+    pipeline-gen:
+    - build-job:
+        image: { "name": "ghcr.io/spack/pcluster-amazonlinux-2:latest", "entrypoint": [""] }
+        before_script:
+        - - . "./share/spack/setup-env.sh"
+          - . /etc/profile.d/modules.sh
+          - spack --version
+          - spack arch
+        # Setup postinstall Spack as upstream installation
+        - - if [[ -f /bootstrap/spack/etc/spack/packages.yaml ]]; then cp /bootstrap/spack/etc/spack/packages.yaml ./etc/spack/; fi
+          - if [[ -f /bootstrap/spack/etc/spack/config.yaml ]]; then cp /bootstrap/spack/etc/spack/config.yaml ./etc/spack/; fi
+          - if [[ -f /bootstrap/spack/etc/spack/modules.yaml ]]; then cp /bootstrap/spack/etc/spack/modules.yaml ./etc/spack/; fi
+          - if [[ -f /bootstrap/spack/etc/spack/mirrors.yaml ]]; then cp /bootstrap/spack/etc/spack/mirrors.yaml ./etc/spack/; fi
+          - if [[ -d /bootstrap/spack/opt/spack ]]; then spack config add "upstreams:postinstall:install_tree:/bootstrap/spack/opt/spack"; fi
+        - - /bin/bash "${SPACK_ARTIFACTS_ROOT}/postinstall.sh" -fg
+          - spack config --scope site add "packages:all:target:\"target=${SPACK_TARGET_ARCH}\""
+    - signing-job:
+        before_script:
+          # Do not distribute Intel & ARM binaries
+          - - for i in $(aws s3 ls --recursive ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/ | grep intel-oneapi | awk '{print $4}' | sed -e 's?^.*build_cache/??g'); do aws s3 rm ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/$i; done
+            - for i in $(aws s3 ls --recursive ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/ | grep armpl | awk '{print $4}' | sed -e 's?^.*build_cache/??g'); do aws s3 rm ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/$i; done
+  cdash:
+    build-group: AWS Packages
--- a/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-neoverse_n1/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-neoverse_n1/spack.yaml
@ -0,0 +1,61 @@
+spack:
+  view: false
+
+  definitions:
+  - compiler_specs:
+    - gcc
+    - gettext
+
+  - compiler_target:
+    - '%gcc@7.3.1 target=aarch64'
+
+  - optimized_configs:
+    - gromacs
+    # - lammps
+    # - mpas-model
+    - openfoam
+    - palace
+    # - py-devito
+    - quantum-espresso
+    # - wrf
+
+  - optimized_libs:
+    - mpich
+    - openmpi
+
+  specs:
+  - matrix:
+    - - $compiler_specs
+    - - $compiler_target
+  - $optimized_configs
+  - $optimized_libs
+
+
+  mirrors: { "mirror": "s3://spack-binaries/develop/aws-pcluster-neoverse_n1" }
+
+  ci:
+    pipeline-gen:
+    - build-job:
+        image: { "name": "ghcr.io/spack/pcluster-amazonlinux-2:latest", "entrypoint": [""] }
+        tags: ["aarch64"]
+        before_script:
+        - - . "./share/spack/setup-env.sh"
+          - . /etc/profile.d/modules.sh
+          - spack --version
+          - spack arch
+        # Setup postinstall Spack as upstream installation
+        - - if [[ -f /bootstrap/spack/etc/spack/packages.yaml ]]; then cp /bootstrap/spack/etc/spack/packages.yaml ./etc/spack/; fi
+          - if [[ -f /bootstrap/spack/etc/spack/config.yaml ]]; then cp /bootstrap/spack/etc/spack/config.yaml ./etc/spack/; fi
+          - if [[ -f /bootstrap/spack/etc/spack/modules.yaml ]]; then cp /bootstrap/spack/etc/spack/modules.yaml ./etc/spack/; fi
+          - if [[ -f /bootstrap/spack/etc/spack/mirrors.yaml ]]; then cp /bootstrap/spack/etc/spack/mirrors.yaml ./etc/spack/; fi
+          - if [[ -d /bootstrap/spack/opt/spack ]]; then spack config add "upstreams:postinstall:install_tree:/bootstrap/spack/opt/spack"; fi
+        - - /bin/bash "${SPACK_ARTIFACTS_ROOT}/postinstall.sh" -fg
+          - spack config --scope site add "packages:all:target:\"target=${SPACK_TARGET_ARCH}\""
+    - signing-job:
+        before_script:
+          # Do not distribute Intel & ARM binaries
+          - - for i in $(aws s3 ls --recursive ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/ | grep intel-oneapi | awk '{print $4}' | sed -e 's?^.*build_cache/??g'); do aws s3 rm ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/$i; done
+            - for i in $(aws s3 ls --recursive ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/ | grep armpl | awk '{print $4}' | sed -e 's?^.*build_cache/??g'); do aws s3 rm ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/$i; done
+
+  cdash:
+    build-group: AWS Packages
--- a/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-neoverse_v1/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-neoverse_v1/spack.yaml
@ -0,0 +1,61 @@
+spack:
+  view: false
+
+  definitions:
+  - compiler_specs:
+    - gcc
+    - gettext
+
+  - compiler_target:
+    - '%gcc@7.3.1 target=aarch64'
+
+  - optimized_configs:
+    - gromacs
+    # - lammps
+    # - mpas-model
+    - openfoam
+    - palace
+    # - py-devito
+    - quantum-espresso
+    # - wrf
+
+  - optimized_libs:
+    - mpich
+    - openmpi
+
+  specs:
+  - matrix:
+    - - $compiler_specs
+    - - $compiler_target
+  - $optimized_configs
+  - $optimized_libs
+
+
+  mirrors: { "mirror": "s3://spack-binaries/develop/aws-pcluster-neoverse_v1" }
+
+  ci:
+    pipeline-gen:
+    - build-job:
+        image: { "name": "ghcr.io/spack/pcluster-amazonlinux-2:latest", "entrypoint": [""] }
+        tags: ["aarch64"]
+        before_script:
+        - - . "./share/spack/setup-env.sh"
+          - . /etc/profile.d/modules.sh
+          - spack --version
+          - spack arch
+        # Setup postinstall Spack as upstream installation
+        - - if [[ -f /bootstrap/spack/etc/spack/packages.yaml ]]; then cp /bootstrap/spack/etc/spack/packages.yaml ./etc/spack/; fi
+          - if [[ -f /bootstrap/spack/etc/spack/config.yaml ]]; then cp /bootstrap/spack/etc/spack/config.yaml ./etc/spack/; fi
+          - if [[ -f /bootstrap/spack/etc/spack/modules.yaml ]]; then cp /bootstrap/spack/etc/spack/modules.yaml ./etc/spack/; fi
+          - if [[ -f /bootstrap/spack/etc/spack/mirrors.yaml ]]; then cp /bootstrap/spack/etc/spack/mirrors.yaml ./etc/spack/; fi
+          - if [[ -d /bootstrap/spack/opt/spack ]]; then spack config add "upstreams:postinstall:install_tree:/bootstrap/spack/opt/spack"; fi
+        - - /bin/bash "${SPACK_ARTIFACTS_ROOT}/postinstall.sh" -fg
+          - spack config --scope site add "packages:all:target:\"target=${SPACK_TARGET_ARCH}\""
+    - signing-job:
+        before_script:
+          # Do not distribute Intel & ARM binaries
+          - - for i in $(aws s3 ls --recursive ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/ | grep intel-oneapi | awk '{print $4}' | sed -e 's?^.*build_cache/??g'); do aws s3 rm ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/$i; done
+            - for i in $(aws s3 ls --recursive ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/ | grep armpl | awk '{print $4}' | sed -e 's?^.*build_cache/??g'); do aws s3 rm ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/$i; done
+
+  cdash:
+    build-group: AWS Packages
--- a/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-skylake/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-skylake/spack.yaml
@ -0,0 +1,58 @@
+spack:
+  view: false
+
+  definitions:
+  - compiler_specs:
+    - gcc
+    - gettext
+
+  - compiler_target:
+    - '%gcc@7.3.1 target=x86_64_v3'
+
+  - optimized_configs:
+    # - gromacs
+    - lammps
+    # - mpas-model
+    - openfoam
+    # - palace
+    # - py-devito
+    # - quantum-espresso
+    # - wrf
+
+  - optimized_libs:
+    - mpich
+    - openmpi
+
+  specs:
+  - matrix:
+    - - $compiler_specs
+    - - $compiler_target
+  - $optimized_configs
+  # - $optimized_libs
+
+  mirrors: { "mirror": "s3://spack-binaries/develop/aws-pcluster-skylake" }
+
+  ci:
+    pipeline-gen:
+    - build-job:
+        image: { "name": "ghcr.io/spack/pcluster-amazonlinux-2:latest", "entrypoint": [""] }
+        before_script:
+        - - . "./share/spack/setup-env.sh"
+          - . /etc/profile.d/modules.sh
+          - spack --version
+          - spack arch
+        # Setup postinstall Spack as upstream installation
+        - - if [[ -f /bootstrap/spack/etc/spack/packages.yaml ]]; then cp /bootstrap/spack/etc/spack/packages.yaml ./etc/spack/; fi
+          - if [[ -f /bootstrap/spack/etc/spack/config.yaml ]]; then cp /bootstrap/spack/etc/spack/config.yaml ./etc/spack/; fi
+          - if [[ -f /bootstrap/spack/etc/spack/modules.yaml ]]; then cp /bootstrap/spack/etc/spack/modules.yaml ./etc/spack/; fi
+          - if [[ -f /bootstrap/spack/etc/spack/mirrors.yaml ]]; then cp /bootstrap/spack/etc/spack/mirrors.yaml ./etc/spack/; fi
+          - if [[ -d /bootstrap/spack/opt/spack ]]; then spack config add "upstreams:postinstall:install_tree:/bootstrap/spack/opt/spack"; fi
+        - - /bin/bash "${SPACK_ARTIFACTS_ROOT}/postinstall.sh" -fg
+          - spack config --scope site add "packages:all:target:\"target=${SPACK_TARGET_ARCH}\""
+    - signing-job:
+        before_script:
+          # Do not distribute Intel & ARM binaries
+          - - for i in $(aws s3 ls --recursive ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/ | grep intel-oneapi | awk '{print $4}' | sed -e 's?^.*build_cache/??g'); do aws s3 rm ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/$i; done
+            - for i in $(aws s3 ls --recursive ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/ | grep armpl | awk '{print $4}' | sed -e 's?^.*build_cache/??g'); do aws s3 rm ${SPACK_REMOTE_MIRROR_OVERRIDE}/build_cache/$i; done
+  cdash:
+    build-group: AWS Packages