
This PR allows users to configure explicit splicing replacement of an abstract spec in the concretizer. concretizer: splice: explicit: - target: mpi replacement: mpich/abcdef transitive: true This config block would mean "for any spec that concretizes to use mpi, splice in mpich/abcdef in place of the mpi it would naturally concretize to use. See #20262, #26873, #27919, and #46382 for PRs enabling splicing in the Spec object. This PR will be the first place the splice method is used in a user-facing manner. See https://spack.readthedocs.io/en/latest/spack.html#spack.spec.Spec.splice for more information on splicing. This will allow users to reuse generic public binaries while splicing in the performant local mpi implementation on their system. In the config file, the target may be any abstract spec. The replacement must be a spec that includes an abstract hash `/abcdef`. The transitive key is optional, defaulting to true if left out. Two important items to note: 1. When writing explicit splice config, the user is in charge of ensuring that the replacement specs they use are binary compatible with whatever targets they replace. In practice, this will likely require either specific knowledge of what packages will be installed by the user's workflow, or somewhat more specific abstract "target" specs for splicing, to ensure binary compatibility. 2. Explicit splices can cause the output of the concretizer not to satisfy the input. For example, using the config above and consider a package in a binary cache `hdf5/xyzabc` that depends on mvapich2. Then the command `spack install hdf5/xyzabc` will instead install the result of splicing `mpich/abcdef` into `hdf5/xyzabc` in place of whatever mvapich2 spec it previously depended on. When this occurs, a warning message is printed `Warning: explicit splice configuration has caused the the concretized spec {concrete_spec} not to satisfy the input spec {input_spec}". Highlighted technical details of implementation: 1. This PR required modifying the installer to have two separate types of Tasks, `RewireTask` and `BuildTask`. Spliced specs are queued as `RewireTask` and standard specs are queued as `BuildTask`. Each spliced spec retains a pointer to its build_spec for provenance. If a RewireTask is dequeued and the associated `build_spec` is neither available in the install_tree nor from a binary cache, the RewireTask is requeued with a new dependency on a BuildTask for the build_spec, and BuildTasks are queued for the build spec and its dependencies. 2. Relocation is modified so that a spack binary can be simultaneously installed and rewired. This ensures that installing the build_spec is not necessary when splicing from a binary cache. 3. The splicing model is modified to more accurately represent build dependencies -- that is, spliced specs do not have build dependencies, as spliced specs are never built. Their build_specs retain the build dependencies, as they may be built as part of installing the spliced spec. 4. There were vestiges of the compiler bootstrapping logic that were not removed in #46237 because I asked alalazo to leave them in to avoid making the rebase for this PR harder than it needed to be. Those last remains are removed in this PR. Co-authored-by: Nathan Hanford <hanford1@llnl.gov> Co-authored-by: Gregory Becker <becker33@llnl.gov> Co-authored-by: Tamara Dahlgren <dahlgren1@llnl.gov>
149 lines
5.4 KiB
YAML
149 lines
5.4 KiB
YAML
name: style
|
|
|
|
on:
|
|
workflow_call:
|
|
inputs:
|
|
with_coverage:
|
|
required: true
|
|
type: string
|
|
|
|
concurrency:
|
|
group: style-${{github.ref}}-${{github.event.pull_request.number || github.run_number}}
|
|
cancel-in-progress: true
|
|
|
|
|
|
jobs:
|
|
# Validate that the code can be run on all the Python versions
|
|
# supported by Spack
|
|
validate:
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871
|
|
- uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3
|
|
with:
|
|
python-version: '3.11'
|
|
cache: 'pip'
|
|
- name: Install Python Packages
|
|
run: |
|
|
pip install --upgrade pip setuptools
|
|
pip install -r .github/workflows/requirements/style/requirements.txt
|
|
- name: vermin (Spack's Core)
|
|
run: vermin --backport importlib --backport argparse --violations --backport typing -t=3.6- -vvv lib/spack/spack/ lib/spack/llnl/ bin/
|
|
- name: vermin (Repositories)
|
|
run: vermin --backport importlib --backport argparse --violations --backport typing -t=3.6- -vvv var/spack/repos
|
|
# Run style checks on the files that have been changed
|
|
style:
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871
|
|
with:
|
|
fetch-depth: 0
|
|
- uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3
|
|
with:
|
|
python-version: '3.11'
|
|
cache: 'pip'
|
|
- name: Install Python packages
|
|
run: |
|
|
pip install --upgrade pip setuptools
|
|
pip install -r .github/workflows/requirements/style/requirements.txt
|
|
- name: Setup git configuration
|
|
run: |
|
|
# Need this for the git tests to succeed.
|
|
git --version
|
|
. .github/workflows/bin/setup_git.sh
|
|
- name: Run style tests
|
|
run: |
|
|
share/spack/qa/run-style-tests
|
|
audit:
|
|
uses: ./.github/workflows/audit.yaml
|
|
secrets: inherit
|
|
with:
|
|
with_coverage: ${{ inputs.with_coverage }}
|
|
python_version: '3.11'
|
|
# Check that spack can bootstrap the development environment on Python 3.6 - RHEL8
|
|
bootstrap-dev-rhel8:
|
|
runs-on: ubuntu-latest
|
|
container: registry.access.redhat.com/ubi8/ubi
|
|
steps:
|
|
- name: Install dependencies
|
|
run: |
|
|
dnf install -y \
|
|
bzip2 curl file gcc-c++ gcc gcc-gfortran git gnupg2 gzip \
|
|
make patch tcl unzip which xz
|
|
- uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871
|
|
- name: Setup repo and non-root user
|
|
run: |
|
|
git --version
|
|
git config --global --add safe.directory /__w/spack/spack
|
|
git fetch --unshallow
|
|
. .github/workflows/bin/setup_git.sh
|
|
useradd spack-test
|
|
chown -R spack-test .
|
|
- name: Bootstrap Spack development environment
|
|
shell: runuser -u spack-test -- bash {0}
|
|
run: |
|
|
source share/spack/setup-env.sh
|
|
spack debug report
|
|
spack -d bootstrap now --dev
|
|
spack -d style -t black
|
|
spack unit-test -V
|
|
import-check:
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- uses: julia-actions/setup-julia@v2
|
|
with:
|
|
version: '1.10'
|
|
- uses: julia-actions/cache@v2
|
|
|
|
# PR: use the base of the PR as the old commit
|
|
- name: Checkout PR base commit
|
|
if: github.event_name == 'pull_request'
|
|
uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871
|
|
with:
|
|
ref: ${{ github.event.pull_request.base.sha }}
|
|
path: old
|
|
# not a PR: use the previous commit as the old commit
|
|
- name: Checkout previous commit
|
|
if: github.event_name != 'pull_request'
|
|
uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871
|
|
with:
|
|
fetch-depth: 2
|
|
path: old
|
|
- name: Checkout previous commit
|
|
if: github.event_name != 'pull_request'
|
|
run: git -C old reset --hard HEAD^
|
|
|
|
- name: Checkout new commit
|
|
uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871
|
|
with:
|
|
path: new
|
|
- name: Install circular import checker
|
|
uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871
|
|
with:
|
|
repository: haampie/circular-import-fighter
|
|
ref: 555519c6fd5564fd2eb844e7b87e84f4d12602e2
|
|
path: circular-import-fighter
|
|
- name: Install dependencies
|
|
working-directory: circular-import-fighter
|
|
run: make -j dependencies
|
|
- name: Import cycles before
|
|
working-directory: circular-import-fighter
|
|
run: make SPACK_ROOT=../old && cp solution solution.old
|
|
- name: Import cycles after
|
|
working-directory: circular-import-fighter
|
|
run: make clean-graph && make SPACK_ROOT=../new && cp solution solution.new
|
|
- name: Compare import cycles
|
|
working-directory: circular-import-fighter
|
|
run: |
|
|
edges_before="$(grep -oP 'edges to delete: \K\d+' solution.old)"
|
|
edges_after="$(grep -oP 'edges to delete: \K\d+' solution.new)"
|
|
if [ "$edges_after" -gt "$edges_before" ]; then
|
|
printf '\033[1;31mImport check failed: %s imports need to be deleted, ' "$edges_after"
|
|
printf 'previously this was %s\033[0m\n' "$edges_before"
|
|
printf 'Compare \033[1;97m"Import cycles before"\033[0m and '
|
|
printf '\033[1;97m"Import cycles after"\033[0m to see problematic imports.\n'
|
|
exit 1
|
|
else
|
|
printf '\033[1;32mImport check passed: %s <= %s\033[0m\n' "$edges_after" "$edges_before"
|
|
fi
|