From 12fe7aef65ecd47267feacd6ed7b9657f272ee0a Mon Sep 17 00:00:00 2001 From: Massimiliano Culpo Date: Thu, 27 Feb 2025 12:13:34 +0100 Subject: [PATCH] pipelines: extract changes from compiler as nodes (#49222) * Split requirements to get better error messages in case of unsat solves. * use list requirements instead of string * activate static_analysis in a few pipelines Signed-off-by: Massimiliano Culpo --- .../aws-pcluster-neoverse_v1/spack.yaml | 13 +- .../stacks/aws-pcluster-x86_64_v4/spack.yaml | 6 + .../stacks/build_systems/spack.yaml | 28 ++-- .../spack.yaml | 132 +++++++++--------- .../spack.yaml | 131 ++++++++--------- .../stacks/e4s-cray-rhel/spack.yaml | 39 +++++- .../stacks/e4s-neoverse-v2/spack.yaml | 4 +- .../stacks/e4s-oneapi/spack.yaml | 37 +++-- .../stacks/e4s-rocm-external/spack.yaml | 7 +- .../cloud_pipelines/stacks/e4s/spack.yaml | 3 +- .../cloud_pipelines/stacks/hep/spack.yaml | 4 +- .../cloud_pipelines/stacks/radiuss/spack.yaml | 66 ++++----- .../stacks/windows-vis/spack.yaml | 2 +- 13 files changed, 249 insertions(+), 223 deletions(-) diff --git a/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-neoverse_v1/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-neoverse_v1/spack.yaml index e91665c398a..045c89de2dc 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-neoverse_v1/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-neoverse_v1/spack.yaml @@ -52,10 +52,12 @@ spack: packages: acfl: require: - - '%gcc target=aarch64' + - "%gcc" + - "target=aarch64" gromacs: require: - - gromacs@2024.3 %gcc ^armpl-gcc ^openmpi + - gromacs@2024.3 ^armpl-gcc ^openmpi + - "%gcc" libfabric: buildable: true externals: @@ -67,13 +69,14 @@ spack: variants: ~lldb mpas-model: require: - - precision=single %gcc ^parallelio+pnetcdf + - precision=single ^parallelio+pnetcdf + - "%gcc" mpich: require: - mpich pmi=pmi2 device=ch4 netmod=ofi +slurm nvhpc: require: - - nvhpc %gcc target=aarch64 + - "target=aarch64" openfoam: require: - openfoam ^scotch@6.0.9 @@ -85,7 +88,7 @@ spack: # require: # - one_of: ["palace cxxflags=\"-include cstdint\" ^fmt@9.1.0"] pmix: - require: 'pmix@3:' + require: "pmix@3:" quantum-espresso: require: - quantum-espresso@6.6 %gcc ^armpl-gcc diff --git a/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-x86_64_v4/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-x86_64_v4/spack.yaml index fd2d4f2138c..d4da712cd75 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-x86_64_v4/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/aws-pcluster-x86_64_v4/spack.yaml @@ -82,6 +82,12 @@ spack: require: - lammps_sizes=bigbig +molecule +kspace +rigid +asphere +opt +openmp +openmp-package fft=mkl ^intel-oneapi-mkl - one_of: [+intel target=x86_64_v4, target=x86_64_v3] + bison: + require: + - "%gcc" + boost: + require: + - "%gcc" libfabric: buildable: true externals: diff --git a/share/spack/gitlab/cloud_pipelines/stacks/build_systems/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/build_systems/spack.yaml index 3d0419e7318..248c699fa95 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/build_systems/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/build_systems/spack.yaml @@ -2,25 +2,19 @@ spack: view: false packages: all: - require: target=x86_64_v3 - definitions: - - default_specs: - - 'uncrustify build_system=autotools' - - 'uncrustify build_system=cmake' - - lz4 # MakefilePackage - - mpich~fortran # AutotoolsPackage - - py-setuptools # PythonPackage - - openjpeg # CMakePackage - - r-rcpp # RPackage - - ruby-rake # RubyPackage - - perl-data-dumper # PerlPackage - - arch: - - '%gcc' + require: + - target=x86_64_v3 specs: - - matrix: - - - $default_specs - - - $arch + - 'uncrustify build_system=autotools' + - 'uncrustify build_system=cmake' + - lz4 # MakefilePackage + - mpich~fortran # AutotoolsPackage + - py-setuptools # PythonPackage + - openjpeg # CMakePackage + - r-rcpp # RPackage + - ruby-rake # RubyPackage + - perl-data-dumper # PerlPackage cdash: build-group: Build Systems diff --git a/share/spack/gitlab/cloud_pipelines/stacks/developer-tools-aarch64-linux-gnu/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/developer-tools-aarch64-linux-gnu/spack.yaml index 4e2356c520e..aa152e1ff98 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/developer-tools-aarch64-linux-gnu/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/developer-tools-aarch64-linux-gnu/spack.yaml @@ -2,79 +2,75 @@ spack: view: false packages: all: - require: target=aarch64 + require: + - target=aarch64 + prefer: + - '%gcc' + concretizer: unify: true reuse: false - definitions: - - default_specs: - # editors - - neovim~no_luajit - - py-pynvim - - emacs+json+native+treesitter # note, pulls in gcc - # - tree-sitter is a dep, should also have cli but no package - - nano # just in case - # tags and scope search helpers - - universal-ctags # only maintained ctags, works better with c++ - - direnv - # runtimes and compilers - - python - - llvm+link_llvm_dylib~lld~lldb~polly+python build_type=MinSizeRel # for clangd, clang-format - - node-js # for editor plugins etc., pyright language server - - npm - - cmake - - libtool - - go # to build fzf, gh, hub - - rust+dev # fd, ripgrep, hyperfine, exa, rust-analyzer - - binutils+ld+gold+plugins # support linking with built gcc - # styling and lints - - astyle - - cppcheck - - uncrustify - - py-fprettify - - py-fortran-language-server - - py-python-lsp-server - # cli dev tools - - ripgrep - - gh - - fd - - bfs - - fzf - - tree - - jq - - py-yq - - hub - - ncdu - - eza - - lsd - - hyperfine - - htop - - tmux - - ccache - # ensure we can use a jobserver build and do this fast - - gmake - - ninja # should be @kitware, can't be because of meson requirement - - openssl certs=system # must be this, system external does not work - - libtree - - patchelf - - sed - - which - - elfutils - - fontconfig - - font-util - - gdb - - flex - - graphviz - - doxygen - - meson - - - arch: - - '%gcc target=aarch64' specs: - - matrix: - - - $default_specs - - - $arch + # editors + - neovim~no_luajit + - py-pynvim + - emacs+json+native+treesitter # note, pulls in gcc + # - tree-sitter is a dep, should also have cli but no package + - nano # just in case + # tags and scope search helpers + - universal-ctags # only maintained ctags, works better with c++ + - direnv + # runtimes and compilers + - python + - llvm+link_llvm_dylib~lld~lldb~polly+python build_type=MinSizeRel # for clangd, clang-format + - node-js # for editor plugins etc., pyright language server + - npm + - cmake + - libtool + - go # to build fzf, gh, hub + - rust+dev # fd, ripgrep, hyperfine, exa, rust-analyzer + - binutils+ld+gold+plugins # support linking with built gcc + # styling and lints + - astyle + - cppcheck + - uncrustify + - py-fprettify + - py-fortran-language-server + - py-python-lsp-server + # cli dev tools + - ripgrep + - gh + - fd + - bfs + - fzf + - tree + - jq + - py-yq + - hub + - ncdu + - eza + - lsd + - hyperfine + - htop + - tmux + - ccache + # ensure we can use a jobserver build and do this fast + - gmake + - ninja # should be @kitware, can't be because of meson requirement + - openssl certs=system # must be this, system external does not work + - libtree + - patchelf + - sed + - which + - elfutils + - fontconfig + - font-util + - gdb + - flex + - graphviz + - doxygen + - meson ci: pipeline-gen: diff --git a/share/spack/gitlab/cloud_pipelines/stacks/developer-tools-x86_64_v3-linux-gnu/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/developer-tools-x86_64_v3-linux-gnu/spack.yaml index 48337e9cd46..c69cd35fc41 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/developer-tools-x86_64_v3-linux-gnu/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/developer-tools-x86_64_v3-linux-gnu/spack.yaml @@ -1,86 +1,79 @@ spack: view: false + packages: all: require: - target=x86_64_v3 - ~cuda - ~rocm - + prefer: + - "%gcc" concretizer: unify: true reuse: false static_analysis: true - definitions: - - default_specs: - # editors - - neovim~no_luajit - - py-pynvim - - emacs+json+native+treesitter # note, pulls in gcc - # - tree-sitter is a dep, should also have cli but no package - - nano # just in case - # tags and scope search helpers - - universal-ctags # only maintained ctags, works better with c++ - - direnv - # runtimes and compilers - - python - - llvm+link_llvm_dylib~lld~lldb~polly+python build_type=MinSizeRel # for clangd, clang-format - - node-js # for editor plugins etc., pyright language server - - npm - - cmake - - libtool - - go # to build fzf, gh, hub - - rust+dev # fd, ripgrep, hyperfine, exa, rust-analyzer - - binutils+ld+gold+plugins # support linking with built gcc - # styling and lints - - astyle - - cppcheck - - uncrustify - - py-fprettify - - py-fortran-language-server - - py-python-lsp-server - # cli dev tools - - ripgrep - - gh - - fd - - bfs - - fzf - - tree - - jq - - py-yq - - hub - - ncdu - - eza - - lsd - - hyperfine - - htop - - tmux - - ccache - # ensure we can use a jobserver build and do this fast - - gmake - - ninja # should be @kitware, can't be because of meson requirement - - openssl certs=system # must be this, system external does not work - - libtree - - patchelf - - sed - - which - - elfutils - - fontconfig - - font-util - - gdb - - flex - - graphviz - - doxygen - - meson - - - arch: - - '%gcc target=x86_64_v3' - specs: - - matrix: - - - $default_specs - - - $arch + # editors + - neovim~no_luajit + - py-pynvim + - emacs+json+native+treesitter # note, pulls in gcc + # - tree-sitter is a dep, should also have cli but no package + - nano # just in case + # tags and scope search helpers + - universal-ctags # only maintained ctags, works better with c++ + - direnv + # runtimes and compilers + - python + - llvm+link_llvm_dylib~lld~lldb~polly+python build_type=MinSizeRel # for clangd, clang-format + - node-js # for editor plugins etc., pyright language server + - npm + - cmake + - libtool + - go # to build fzf, gh, hub + - rust+dev # fd, ripgrep, hyperfine, exa, rust-analyzer + - binutils+ld+gold+plugins # support linking with built gcc + # styling and lints + - astyle + - cppcheck + - uncrustify + - py-fprettify + - py-fortran-language-server + - py-python-lsp-server + # cli dev tools + - ripgrep + - gh + - fd + - bfs + - fzf + - tree + - jq + - py-yq + - hub + - ncdu + - eza + - lsd + - hyperfine + - htop + - tmux + - ccache + # ensure we can use a jobserver build and do this fast + - gmake + - ninja # should be @kitware, can't be because of meson requirement + - openssl certs=system # must be this, system external does not work + - libtree + - patchelf + - sed + - which + - elfutils + - fontconfig + - font-util + - gdb + - flex + - graphviz + - doxygen + - meson ci: pipeline-gen: diff --git a/share/spack/gitlab/cloud_pipelines/stacks/e4s-cray-rhel/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/e4s-cray-rhel/spack.yaml index 9cd1ef7afdd..19eafbc6a75 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/e4s-cray-rhel/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s-cray-rhel/spack.yaml @@ -5,6 +5,7 @@ spack: view: false concretizer: + static_analysis: true reuse: false unify: false @@ -14,8 +15,9 @@ spack: packages: all: - require: "%cce@18.0.0 target=x86_64_v3" - compiler: [cce] + require: + - target=x86_64_v3 + - "%cce" providers: blas: [cray-libsci] lapack: [cray-libsci] @@ -23,6 +25,21 @@ spack: tbb: [intel-tbb] scalapack: [netlib-scalapack] variants: +mpi + + # Virtuals + blas: + require: + - cray-libsci + lapack: + require: + - cray-libsci + mpi: + require: + - cray-mpich + scalapack: + require: + - netlib-scalapack + ncurses: require: +termlib ldflags=-Wl,--undefined-version tbb: @@ -33,21 +50,28 @@ spack: variants: +python +filesystem +iostreams +system elfutils: variants: ~nls - require: "%gcc" + require: + - target=x86_64_v3 + - "%gcc" gcc-runtime: - require: "%gcc" + require: + - target=x86_64_v3 + - "%gcc" hdf5: variants: +fortran +hl +shared libfabric: variants: fabrics=sockets,tcp,udp,rxm mgard: require: + - target=x86_64_v3 - "@2023-01-10:" mpich: variants: ~wrapperrpath paraview: # Don't build GUI support or GLX rendering for HPC/container deployments - require: "~qt ^[virtuals=gl] osmesa" + require: + - "~qt ^[virtuals=gl] osmesa" + - target=x86_64_v3 trilinos: require: - one_of: [+amesos +amesos2 +anasazi +aztec +boost +epetra +epetraext +ifpack @@ -58,6 +82,7 @@ spack: - one_of: [~ml ~muelu ~zoltan2 ~teko, +ml +muelu +zoltan2 +teko] - one_of: [+superlu-dist, ~superlu-dist] - one_of: [+shylu, ~shylu] + - target=x86_64_v3 specs: # CPU @@ -140,8 +165,8 @@ spack: # - gptune # py-scipy: meson.build:82:0: ERROR: Unknown compiler(s): [['/home/gitlab-runner-3/builds/dWfnZWPh/0/spack/spack/lib/spack/env/cce/ftn']] # - hpctoolkit # dyninst requires %gcc # - hpx max_cpu_count=512 networking=mpi # libxcrypt-4.4.35 - # - lammps # lammps-20240829.1: Reversed (or previously applied) patch detected! Assume -R? [n] - # - libpressio +bitgrooming +bzip2 ~cuda ~cusz +fpzip +hdf5 +libdistributed +lua +openmp +python +sz +sz3 +unix +zfp +json +remote +netcdf +mgard # mgard: + # - lammps # lammps-20240829.1: Reversed (or previously applied) patch detected! Assume -R? [n] + # - libpressio +bitgrooming +bzip2 ~cuda ~cusz +fpzip +hdf5 +libdistributed +lua +openmp +python +sz +sz3 +unix +zfp +json +remote +netcdf +mgard # mgard: # - mgard +serial +openmp +timing +unstructured ~cuda # mgard # - nrm # py-scipy: meson.build:82:0: ERROR: Unknown compiler(s): [['/home/gitlab-runner-3/builds/dWfnZWPh/0/spack/spack/lib/spack/env/cce/ftn']] # - nvhpc # requires %gcc diff --git a/share/spack/gitlab/cloud_pipelines/stacks/e4s-neoverse-v2/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/e4s-neoverse-v2/spack.yaml index ae996a4022b..060b091207e 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/e4s-neoverse-v2/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s-neoverse-v2/spack.yaml @@ -7,7 +7,9 @@ spack: packages: all: - require: '%gcc target=neoverse_v2' + require: + - "%gcc" + - target=neoverse_v2 providers: blas: [openblas] mpi: [mpich] diff --git a/share/spack/gitlab/cloud_pipelines/stacks/e4s-oneapi/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/e4s-oneapi/spack.yaml index 2e53eb38b56..49cd6e3e6f8 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/e4s-oneapi/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s-oneapi/spack.yaml @@ -4,12 +4,13 @@ spack: concretizer: reuse: false unify: false + static_analysis: false packages: all: require: - - "target=x86_64_v3" - - "%oneapi" + - target=x86_64_v3 + - '%oneapi' providers: blas: [openblas] tbb: [intel-tbb] @@ -45,28 +46,36 @@ spack: - spec: intel-oneapi-mpi@2021.13.1 prefix: /opt/intel/oneapi unzip: - require: '%gcc target=x86_64_v3' + require: + - '%gcc target=x86_64_v3' binutils: - require: '%gcc target=x86_64_v3' + require: + - '%gcc target=x86_64_v3' variants: +ld +gold +headers +libiberty ~nls llvm: - require: '%gcc target=x86_64_v3' + require: + - '%gcc target=x86_64_v3' ruby: - require: '%gcc target=x86_64_v3' + require: + - '%gcc target=x86_64_v3' rust: - require: '%gcc target=x86_64_v3' + require: + - '%gcc target=x86_64_v3' krb5: - require: '%gcc target=x86_64_v3' - papi: - require: '%gcc target=x86_64_v3' + require: + - '%gcc target=x86_64_v3' openssh: - require: '%gcc target=x86_64_v3' + require: + - '%gcc target=x86_64_v3' dyninst: - require: "%gcc target=x86_64_v3" + require: + - '%gcc target=x86_64_v3' bison: - require: '%gcc target=x86_64_v3' + require: + - '%gcc target=x86_64_v3' paraview: - require: "+examples %oneapi target=x86_64_v3" + require: + - +examples target=x86_64_v3 specs: # CPU diff --git a/share/spack/gitlab/cloud_pipelines/stacks/e4s-rocm-external/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/e4s-rocm-external/spack.yaml index d765f8c7ae2..7f224acf24b 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/e4s-rocm-external/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s-rocm-external/spack.yaml @@ -7,7 +7,8 @@ spack: packages: all: - require: '%gcc target=x86_64_v3' + require: + - 'target=x86_64_v3' providers: blas: [openblas] variants: +mpi @@ -21,7 +22,9 @@ spack: variants: threads=openmp paraview: # Don't build GUI support or GLX rendering for HPC/container deployments - require: "@5.11 +examples ~qt ^[virtuals=gl] osmesa %gcc target=x86_64_v3" + require: + - "@5.11 +examples ~qt ^[virtuals=gl] osmesa" + - 'target=x86_64_v3' # ROCm comgr: diff --git a/share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml index deaca891398..af85a1f80b1 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/e4s/spack.yaml @@ -9,7 +9,8 @@ spack: packages: all: require: - - '%gcc target=x86_64_v3' + - "%gcc" + - target=x86_64_v3 variants: +mpi mpi: require: diff --git a/share/spack/gitlab/cloud_pipelines/stacks/hep/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/hep/spack.yaml index ea9d630e604..36ece583439 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/hep/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/hep/spack.yaml @@ -8,7 +8,9 @@ spack: packages: all: - require: '%gcc target=x86_64_v3' + require: + - "%gcc" + - target=x86_64_v3 providers: blas: [openblas] mpi: [mpich] diff --git a/share/spack/gitlab/cloud_pipelines/stacks/radiuss/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/radiuss/spack.yaml index 3aea6fc48a4..867c2ae9a62 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/radiuss/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/radiuss/spack.yaml @@ -2,48 +2,40 @@ spack: view: false packages: all: - require: target=x86_64_v3 + require: + - target=x86_64_v3 + - '%gcc@7.5.0' providers: mpi: [mvapich2] - definitions: - #- compilers: ['%gcc@8.3.1', '%clang@10.0.0'] - - compilers: ['%gcc@7.5.0'] - - # Note skipping spot since no spack package for it - - radiuss: - - ascent # ^conduit@0.6.0 - - axom - - blt - - caliper - - care # ~benchmarks ~examples ~tests - - chai # ~examples - - conduit # ^hdf5+shared - - flux-core - #- flux-sched - - hypre - - lbann - - lvarray ~tests # per Spack issue #23192 # ~examples - - mfem - - py-hatchet - - py-maestrowf - - py-merlin - - py-shroud - - raja # ~examples # ~tests - - raja-perf - - samrai - - scr - - sundials - - umpire # ~openmp - #- visit # ^mesa-glu@9.0.0 - - xbraid - - zfp - specs: - - matrix: - - [$radiuss] - - [$compilers] + - ascent # ^conduit@0.6.0 + - axom + - blt + - caliper + - care # ~benchmarks ~examples ~tests + - chai # ~examples + - conduit # ^hdf5+shared + - flux-core + # - flux-sched + - hypre + - lbann + - lvarray ~tests # per Spack issue #23192 # ~examples + - mfem + - py-hatchet + - py-maestrowf + - py-merlin + - py-shroud + - raja # ~examples # ~tests + - raja-perf + - samrai + - scr + - sundials + - umpire # ~openmp + # - visit # ^mesa-glu@9.0.0 + - xbraid + - zfp cdash: build-group: RADIUSS diff --git a/share/spack/gitlab/cloud_pipelines/stacks/windows-vis/spack.yaml b/share/spack/gitlab/cloud_pipelines/stacks/windows-vis/spack.yaml index 1d2546b6980..ae9758166aa 100644 --- a/share/spack/gitlab/cloud_pipelines/stacks/windows-vis/spack.yaml +++ b/share/spack/gitlab/cloud_pipelines/stacks/windows-vis/spack.yaml @@ -6,7 +6,7 @@ spack: view: false specs: - - vtk~mpi + - "vtk@9: ~mpi" cdash: build-group: Windows Visualization (Kitware)