py-snakemake-{executor,storage}-plugin-{htcondor,xrootd}: new packages

dla-future: add v0.7.3, deprecate v0.7.0 and v0.7.1 (#48215 )
* dla-future: add v0.7.2, deprecate v0.7.0 and v0.7.1 * update
2024-12-20 11:21:28 -06:00 · 2024-12-20 08:08:01 -07:00 · 2024-12-20 06:53:15 -07:00 · 2024-12-20 10:02:02 +01:00 · 2024-12-20 09:39:42 +01:00 · 2024-12-20 09:24:10 +01:00
10 changed files with 210 additions and 24 deletions
--- a/.github/workflows/unit_tests.yaml
+++ b/.github/workflows/unit_tests.yaml
@@ -14,29 +14,29 @@ jobs:
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
-        os: [ubuntu-22.04]
-        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12']
+        os: [ubuntu-latest]
+        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
        on_develop:
        - ${{ github.ref == 'refs/heads/develop' }}
        include:
        - python-version: '3.6'
          os: ubuntu-20.04
          on_develop: ${{ github.ref == 'refs/heads/develop' }}
-        exclude:
        - python-version: '3.7'
          os: ubuntu-22.04
-          on_develop: false
+          on_develop: ${{ github.ref == 'refs/heads/develop' }}
+        exclude:
        - python-version: '3.8'
-          os: ubuntu-22.04
+          os: ubuntu-latest
          on_develop: false
        - python-version: '3.9'
-          os: ubuntu-22.04
+          os: ubuntu-latest
          on_develop: false
        - python-version: '3.10'
-          os: ubuntu-22.04
+          os: ubuntu-latest
          on_develop: false
        - python-version: '3.11'
-          os: ubuntu-22.04
+          os: ubuntu-latest
          on_develop: false

    steps:
@@ -52,7 +52,13 @@ jobs:
          # Needed for unit tests
          sudo apt-get -y install \
              coreutils cvs gfortran graphviz gnupg2 mercurial ninja-build \
-              cmake bison libbison-dev kcov
+              cmake bison libbison-dev subversion
+    # On ubuntu 24.04, kcov was removed. It may come back in some future Ubuntu
+    - name: Set up Homebrew
+      id: set-up-homebrew
+      uses: Homebrew/actions/setup-homebrew@40e9946c182a64b3db1bf51be0dcb915f7802aa9
+    - name: Install kcov with brew
+      run: "brew install kcov"
    - name: Install Python packages
      run: |
          pip install --upgrade pip setuptools pytest pytest-xdist pytest-cov
@@ -87,7 +93,7 @@ jobs:
        include-hidden-files: true
  # Test shell integration
  shell:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
      with:
@@ -99,7 +105,13 @@ jobs:
      run: |
          sudo apt-get -y update
          # Needed for shell tests
-          sudo apt-get install -y coreutils kcov csh zsh tcsh fish dash bash
+          sudo apt-get install -y coreutils csh zsh tcsh fish dash bash subversion
+    # On ubuntu 24.04, kcov was removed. It may come back in some future Ubuntu
+    - name: Set up Homebrew
+      id: set-up-homebrew
+      uses: Homebrew/actions/setup-homebrew@40e9946c182a64b3db1bf51be0dcb915f7802aa9
+    - name: Install kcov with brew
+      run: "brew install kcov"
    - name: Install Python packages
      run: |
          pip install --upgrade pip setuptools pytest coverage[toml] pytest-xdist
--- a/.github/workflows/valid-style.yml
+++ b/.github/workflows/valid-style.yml
@@ -121,27 +121,27 @@ jobs:
      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
      with:
        repository: haampie/circular-import-fighter
-        ref: 9f60f51bc7134e0be73f27623f1b0357d1718427
+        ref: b5d6ce9be35f602cca7d5a6aa0259fca10639cca
        path: circular-import-fighter
    - name: Install dependencies
      working-directory: circular-import-fighter
      run: make -j dependencies
-    - name: Import cycles before
+    - name: Problematic imports before
      working-directory: circular-import-fighter
-      run: make SPACK_ROOT=../old && cp solution solution.old
-    - name: Import cycles after
+      run: make SPACK_ROOT=../old SUFFIX=.old
+    - name: Problematic imports after
      working-directory: circular-import-fighter
-      run: make clean-graph && make SPACK_ROOT=../new && cp solution solution.new
+      run: make SPACK_ROOT=../new SUFFIX=.new
    - name: Compare import cycles
      working-directory: circular-import-fighter
      run: |
-        edges_before="$(grep -oP 'edges to delete: \K\d+' solution.old)"
-        edges_after="$(grep -oP 'edges to delete: \K\d+' solution.new)"
+        edges_before="$(head -n1 solution.old)"
+        edges_after="$(head -n1 solution.new)"
        if [ "$edges_after" -gt "$edges_before" ]; then
          printf '\033[1;31mImport check failed: %s imports need to be deleted, ' "$edges_after"
          printf 'previously this was %s\033[0m\n'  "$edges_before"
-          printf 'Compare \033[1;97m"Import cycles before"\033[0m and '
-          printf '\033[1;97m"Import cycles after"\033[0m to see problematic imports.\n'
+          printf 'Compare \033[1;97m"Problematic imports before"\033[0m and '
+          printf '\033[1;97m"Problematic imports after"\033[0m.\n'
          exit 1
        else
          printf '\033[1;32mImport check passed: %s <= %s\033[0m\n' "$edges_after" "$edges_before"
--- a/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml
+++ b/share/spack/gitlab/cloud_pipelines/.gitlab-ci.yml
@@ -426,6 +426,30 @@ developer-tools-x86_64_v3-linux-gnu-build:
    - artifacts: True
      job: developer-tools-x86_64_v3-linux-gnu-generate

+###########################################
+# Build tests for different developer tools
+# aarch64
+###########################################
+.developer-tools-aarch64-linux-gnu:
+  extends: [ ".linux_aarch64" ]
+  variables:
+    SPACK_CI_STACK_NAME: developer-tools-aarch64-linux-gnu
+
+developer-tools-aarch64-linux-gnu-generate:
+  extends: [ ".developer-tools-aarch64-linux-gnu", ".generate-aarch64"]
+  image:  ghcr.io/spack/aarch64-linux-gnu:v2024-12-18
+
+developer-tools-aarch64-linux-gnu-build:
+  extends: [ ".developer-tools-aarch64-linux-gnu", ".build" ]
+  trigger:
+    include:
+      - artifact: jobs_scratch_dir/cloud-ci-pipeline.yml
+        job: developer-tools-aarch64-linux-gnu-generate
+    strategy: depend
+  needs:
+    - artifacts: True
+      job: developer-tools-aarch64-linux-gnu-generate
+
 ###########################################
 # Build tests for different developer tools
 # darwin
--- a/share/spack/gitlab/cloud_pipelines/configs/linux/ci.yaml
+++ b/share/spack/gitlab/cloud_pipelines/configs/linux/ci.yaml
@@ -18,6 +18,7 @@ ci:
          KUBERNETES_MEMORY_REQUEST: 48G

    - match:
+      - composable-kernel
      - rust
      build-job:
        tags: [ "spack", "huge" ]
--- a/share/spack/gitlab/cloud_pipelines/stacks/developer-tools-aarch64-linux-gnu/spack.yaml
+++ b/share/spack/gitlab/cloud_pipelines/stacks/developer-tools-aarch64-linux-gnu/spack.yaml
@@ -0,0 +1,85 @@
+spack:
+  view: false
+  packages:
+    all:
+      require: target=aarch64
+  concretizer:
+    unify: true
+    reuse: false
+  definitions:
+  - default_specs:
+      # editors
+    - neovim~no_luajit
+    - py-pynvim
+    - emacs+json+native+treesitter   # note, pulls in gcc
+      # - tree-sitter is a dep, should also have cli but no package
+    - nano   # just in case
+      # tags and scope search helpers
+    - universal-ctags   # only maintained ctags, works better with c++
+    - direnv
+      # runtimes and compilers
+    - python
+    - llvm+link_llvm_dylib~lld~lldb~polly+python build_type=MinSizeRel   # for clangd, clang-format
+    - node-js   # for editor plugins etc., pyright language server
+    - npm
+    - cmake
+    - libtool
+    - go   # to build fzf, gh, hub
+    - rust+dev   # fd, ripgrep, hyperfine, exa, rust-analyzer
+    - binutils+ld+gold+plugins   # support linking with built gcc
+      # styling and lints
+    - astyle
+    - cppcheck
+    - uncrustify
+    - py-fprettify
+    - py-fortran-language-server
+    - py-python-lsp-server
+      # cli dev tools
+    - ripgrep
+    - gh
+    - fd
+    # - bfs # liburing: /usr/include/linux/ipv6.h:19:8: error: redefinition of 'struct in6_pktinfo'
+    - fzf
+    - tree
+    - jq
+    - py-yq
+    - hub
+    - ncdu
+    - eza
+    - lsd
+    - hyperfine
+    - htop
+    - tmux
+    - ccache
+      # ensure we can use a jobserver build and do this fast
+    - gmake
+    - ninja   # should be @kitware, can't be because of meson requirement
+    - openssl certs=system     # must be this, system external does not work
+    - libtree
+    - patchelf
+    - sed
+    - which
+    - elfutils
+    - fontconfig
+    - font-util
+    - gdb
+    - flex
+    - graphviz
+    - doxygen
+    - meson
+
+  - arch:
+    - '%gcc target=aarch64'
+
+  specs:
+  - matrix:
+    - - $default_specs
+    - - $arch
+
+  ci:
+    pipeline-gen:
+    - build-job:
+        image: ghcr.io/spack/aarch64-linux-gnu:v2024-12-18
+
+  cdash:
+    build-group: Developer Tools aarch64-linux-gnu
--- a/share/spack/qa/run-shell-tests
+++ b/share/spack/qa/run-shell-tests
@@ -36,7 +36,7 @@ export QA_DIR=$(realpath $QA_DIR)
 cd "$SPACK_ROOT"

 # Run bash tests with coverage enabled, but pipe output to /dev/null
-# because it seems that kcov seems to undo the script's redirection
+# because it seems that kcov undoes the script's redirection
 if [ "$COVERAGE" = true ]; then
    kcov "$SPACK_ROOT/coverage" "$QA_DIR/setup-env-test.sh" &> /dev/null
    kcov "$SPACK_ROOT/coverage" "$QA_DIR/completion-test.sh" &> /dev/null
--- a/var/spack/repos/builtin/packages/dla-future/package.py
+++ b/var/spack/repos/builtin/packages/dla-future/package.py
@@ -12,12 +12,21 @@ class DlaFuture(CMakePackage, CudaPackage, ROCmPackage):
    homepage = "https://github.com/eth-cscs/DLA-Future"
    url = "https://github.com/eth-cscs/DLA-Future/archive/v0.0.0.tar.gz"
    git = "https://github.com/eth-cscs/DLA-Future.git"
-    maintainers = ["rasolca", "albestro", "msimberg", "aurianer"]
+    maintainers = ["rasolca", "albestro", "msimberg", "aurianer", "RMeli"]

    license("BSD-3-Clause")

-    version("0.7.1", sha256="651129686b7fb04178f230c763b371192f9cb91262ddb9959f722449715bdfe8")
-    version("0.7.0", sha256="40a62bc70b0a06246a16348ce6701ccfab1f0c1ace99684de4bfc6c90776f8c6")
+    version("0.7.3", sha256="8c829b72f4ea9c924abdb6fe2ac7489304be4056ab76b8eba226c33ce7b7dc0e")
+    version(
+        "0.7.1",
+        sha256="651129686b7fb04178f230c763b371192f9cb91262ddb9959f722449715bdfe8",
+        deprecated=True,
+    )
+    version(
+        "0.7.0",
+        sha256="40a62bc70b0a06246a16348ce6701ccfab1f0c1ace99684de4bfc6c90776f8c6",
+        deprecated=True,
+    )
    version("0.6.0", sha256="85dfcee36ff28fa44da3134408c40ebd611bccff8a295982a7c78eaf982524d9")
    version("0.5.0", sha256="f964ee2a96bb58b3f0ee4563ae65fcd136e409a7c0e66beda33f926fc9515a8e")
    version("0.4.1", sha256="ba95f26475ad68da1f3a24d091dc1b925525e269e4c83c1eaf1d37d29b526666")
--- a/var/spack/repos/builtin/packages/py-snakemake-executor-plugin-htcondor/package.py
+++ b/var/spack/repos/builtin/packages/py-snakemake-executor-plugin-htcondor/package.py
@@ -0,0 +1,27 @@
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PySnakemakeExecutorPluginHtcondor(PythonPackage):
+    """A Snakemake executor plugin for submitting jobs to a HTCondor cluster."""
+
+    homepage = "https://github.com/jannisspeer/snakemake-executor-plugin-htcondor"
+    pypi = "snakemake_executor_plugin_htcondor/snakemake_executor_plugin_htcondor-0.1.2.tar.gz"
+
+    maintainers("wdconinc")
+
+    license("MIT", checked_by="wdconinc")
+
+    version("0.1.2", sha256="c5268807ecb6810d1852cbf7ccde0b54394e1a824eb64baa55538ccf55328f45")
+
+    depends_on("htcondor@23.4.0:23", type=("build", "run"))
+
+    depends_on("py-snakemake-interface-common@1.15:1", type=("build", "run"))
+    depends_on("py-snakemake-interface-executor-plugins@9", type=("build", "run"))
+
+    depends_on("python@3.11:3", type=("build", "run"))
+    depends_on("py-poetry-core", type="build")
--- a/var/spack/repos/builtin/packages/py-snakemake-storage-plugin-xrootd/package.py
+++ b/var/spack/repos/builtin/packages/py-snakemake-storage-plugin-xrootd/package.py
@@ -0,0 +1,27 @@
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class PySnakemakeStoragePluginXrootd(PythonPackage):
+    """A Snakemake storage plugin to read and write from XRootD Storage."""
+
+    homepage = "https://github.com/snakemake/snakemake-storage-plugin-xrootd"
+    pypi = "snakemake_storage_plugin_xrootd/snakemake_storage_plugin_xrootd-0.1.4.tar.gz"
+
+    maintainers("wdconinc")
+
+    license("MIT", checked_by="wdconinc")
+
+    version("0.1.4", sha256="61a48b2567fa7f35a29f00f0a74d1efd069a16c0ef7c9f7f440a46088d2c6dbc")
+
+    depends_on("xrootd@5.6.4:5 +python", type=("build", "run"))
+
+    depends_on("py-snakemake-interface-common@1.15:1", type=("build", "run"))
+    depends_on("py-snakemake-interface-storage-plugins@3.3.0:3", type=("build", "run"))
+
+    depends_on("python@3.11:3", type=("build", "run"))
+    depends_on("py-poetry-core", type="build")
--- a/var/spack/repos/builtin/packages/xrandr/package.py
+++ b/var/spack/repos/builtin/packages/xrandr/package.py
@@ -17,6 +17,7 @@ class Xrandr(AutotoolsPackage, XorgPackage):

    maintainers("wdconinc")

+    version("1.5.3", sha256="980befa14b48de2fa74dc07bbb22f746acfd12e1f9dbd539eab9a3c9b434d667")
    version("1.5.2", sha256="efd062cd228dc18a7de26422c81bc4be6a7e62f7f0ad6f9bebdd9ff8385c5668")
    version("1.5.1", sha256="7b99edb7970a1365eaf5bcaf552144e4dfc3ccf510c4abc08569849929fb366e")
    version("1.5.0", sha256="ddfe8e7866149c24ccce8e6aaa0623218ae19130c2859cadcaa4228d8bb4a46d")
Author	SHA1	Message	Date
Wouter Deconinck	7fbb2923cd	py-snakemake-{executor,storage}-plugin-{htcondor,xrootd}: new packages	2024-12-20 11:21:28 -06:00
Rocco Meli	2edbed3a9d	dla-future: add v0.7.3, deprecate v0.7.0 and v0.7.1 (#48215 ) * dla-future: add v0.7.2, deprecate v0.7.0 and v0.7.1 * update	2024-12-20 08:08:01 -07:00
Zack Galbreath	e0035bd658	ci: request 35G of memory for building composable-kernel (#48227 ) This number was determined from the max memory usage recently recorded for this package in our analytics database.	2024-12-20 06:53:15 -07:00
Massimiliano Culpo	64207e8fe8	Use Ubuntu 22.04 to run unit tests against Python 3.7 (#48233 )	2024-12-20 10:02:02 +01:00
Harmen Stoppels	fdc85572f3	import-check: bump and simplify (#48222 )	2024-12-20 09:39:42 +01:00
eugeneswalker	75162be4b6	ci: add developer-tools-aarch64-linux-gnu stack (#48217 )	2024-12-20 09:24:10 +01:00
Todd Gamblin	adbbb91b41	Make unit tests work on ubuntu 24.04 (#48151 ) `kcov` was removed in Ubuntu 24.04, and it is no longer installable via `apt` in our CI images. Instal it via Linuxbrew instead, at least until it comes back to Ubuntu. `subversion` is also not installed on ubuntu 24 by default, so we have to install it manually. - [x] Add linuxbrew to linux tests - [x] Install `kcov` with brew - [x] Install subversion with `apt` Signed-off-by: Todd Gamblin <tgamblin@llnl.gov>	2024-12-20 08:20:07 +01:00
Wouter Deconinck	55eeff7eb0	xrandr: add v1.5.3 (#48178 )	2024-12-19 19:45:17 -07:00